Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bfdf17126c | ||
|
|
419ee1c441 | ||
|
|
aaa28f541b | ||
|
|
98d7432077 | ||
|
|
11ac8aee50 | ||
|
|
e7cc0adbdb |
No files matched your search
@@ -13,6 +13,7 @@ require (
|
|||||||
github.com/prometheus/client_golang v1.24.1
|
github.com/prometheus/client_golang v1.24.1
|
||||||
github.com/spf13/viper v1.21.0
|
github.com/spf13/viper v1.21.0
|
||||||
go.uber.org/fx v1.24.0
|
go.uber.org/fx v1.24.0
|
||||||
|
golang.org/x/net v0.60.0
|
||||||
golang.org/x/sys v0.48.0
|
golang.org/x/sys v0.48.0
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -50,7 +51,6 @@ require (
|
|||||||
go.uber.org/zap v1.26.0 // indirect
|
go.uber.org/zap v1.26.0 // indirect
|
||||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||||
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
|
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
|
||||||
golang.org/x/net v0.60.0 // indirect
|
|
||||||
golang.org/x/sync v0.23.0 // indirect
|
golang.org/x/sync v0.23.0 // indirect
|
||||||
golang.org/x/text v0.42.0 // indirect
|
golang.org/x/text v0.42.0 // indirect
|
||||||
google.golang.org/protobuf v1.36.11 // indirect
|
google.golang.org/protobuf v1.36.11 // indirect
|
||||||
|
|||||||
@@ -19,6 +19,8 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
)
|
)
|
||||||
|
|
||||||
// off is the value that switches a timeout, a size limit or a rate limit
|
// off is the value that switches a timeout, a size limit or a rate limit
|
||||||
@@ -40,9 +42,7 @@ var (
|
|||||||
errNotPositive = errors.New("must be more than zero, or off")
|
errNotPositive = errors.New("must be more than zero, or off")
|
||||||
errNotMilliseconds = errors.New(
|
errNotMilliseconds = errors.New(
|
||||||
"is not a whole number of milliseconds above zero, such as 2000")
|
"is not a whole number of milliseconds above zero, such as 2000")
|
||||||
errEmptyItem = errors.New("has an empty item in its list")
|
errEmptyItem = errors.New("has an empty item in its list")
|
||||||
errNotNetblock = errors.New(
|
|
||||||
"is not a netblock such as 10.0.0.0/8, or an address")
|
|
||||||
errNotListenAddr = errors.New(
|
errNotListenAddr = errors.New(
|
||||||
"is not an address to listen on, such as :8080")
|
"is not an address to listen on, such as :8080")
|
||||||
errNotCommand = errors.New("is not rproxy or waf")
|
errNotCommand = errors.New("is not rproxy or waf")
|
||||||
@@ -253,7 +253,7 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
|
|||||||
netblocks := make([]netip.Prefix, 0, len(items))
|
netblocks := make([]netip.Prefix, 0, len(items))
|
||||||
|
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
netblock, err := ParseNetblock(item)
|
netblock, err := reputation.ParseNetblock(item)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -264,27 +264,6 @@ func parseNetblocks(value string) ([]netip.Prefix, error) {
|
|||||||
return netblocks, nil
|
return netblocks, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
|
|
||||||
// address is a netblock of that address alone, a /32 or a /128. A
|
|
||||||
// blocklist's lines are read with it too.
|
|
||||||
func ParseNetblock(value string) (netip.Prefix, error) {
|
|
||||||
if strings.Contains(value, "/") {
|
|
||||||
netblock, err := netip.ParsePrefix(value)
|
|
||||||
if err != nil {
|
|
||||||
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
|
||||||
}
|
|
||||||
|
|
||||||
return netblock.Masked(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
addr, err := netip.ParseAddr(value)
|
|
||||||
if err != nil || addr.Zone() != "" {
|
|
||||||
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
|
||||||
}
|
|
||||||
|
|
||||||
return netip.PrefixFrom(addr, addr.BitLen()), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseListenAddr checks an address to listen on: an optional host and a
|
// parseListenAddr checks an address to listen on: an optional host and a
|
||||||
// port number.
|
// port number.
|
||||||
func parseListenAddr(value string) (string, error) {
|
func parseListenAddr(value string) (string, error) {
|
||||||
|
|||||||
+77
-121
@@ -20,11 +20,26 @@ import (
|
|||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
"golang.org/x/net/http/httpguts"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
"sneak.berlin/go/smallwebwaf/internal/coreruleset"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The names of the settings that waf writes into log lines and ban notes,
|
||||||
|
// as the setting that refused a request or lowered a client's limits.
|
||||||
|
const (
|
||||||
|
ExclusivelyAllowedCountriesSetting = "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES"
|
||||||
|
TagLimitPercentSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
|
||||||
|
ASNBytesLimitPercentSetting = "SWW_WAF_ASN_BYTES_LIMIT_PERCENT"
|
||||||
|
CountryBytesLimitPercentSetting = "SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT"
|
||||||
|
UnknownCountryLimitPercentSetting = "SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT"
|
||||||
|
ASNLimitPercentURLSetting = "SWW_WAF_ASN_LIMIT_PERCENT_URL"
|
||||||
|
BlocklistActionSetting = "SWW_WAF_BLOCKLIST_ACTION"
|
||||||
|
DNSBLAbuseIPDBActionSetting = "SWW_WAF_DNSBL_ABUSEIPDB_ACTION"
|
||||||
)
|
)
|
||||||
|
|
||||||
// WAF is the settings of `sww waf`. A time or size limit of zero is off.
|
// WAF is the settings of `sww waf`. A time or size limit of zero is off.
|
||||||
@@ -100,7 +115,7 @@ type WAF struct {
|
|||||||
// with a tag that percentage of every rate limit and byte limit, from 1
|
// with a tag that percentage of every rate limit and byte limit, from 1
|
||||||
// to 99 (SWW_WAF_TAG_LIMIT_PERCENT). All hold tags as requests carry them: in
|
// to 99 (SWW_WAF_TAG_LIMIT_PERCENT). All hold tags as requests carry them: in
|
||||||
// lower case, a country code in capitals.
|
// lower case, a country code in capitals.
|
||||||
TagLists []TagList
|
TagLists []reputation.TagList
|
||||||
TagDeny []string
|
TagDeny []string
|
||||||
TagLimitPercent map[string]int64
|
TagLimitPercent map[string]int64
|
||||||
// RouteWatchURL is where the routewatch instance is that adds the asn=
|
// RouteWatchURL is where the routewatch instance is that adds the asn=
|
||||||
@@ -497,12 +512,8 @@ var (
|
|||||||
"is not a whole number above zero, such as 60, or off",
|
"is not a whole number above zero, such as 60, or off",
|
||||||
)
|
)
|
||||||
errNotUTF8 = errors.New("is not valid UTF-8")
|
errNotUTF8 = errors.New("is not valid UTF-8")
|
||||||
errNotASN = errors.New("is not an AS number such as AS64496")
|
|
||||||
errNotPercentItem = errors.New(
|
errNotPercentItem = errors.New(
|
||||||
"is not a code, : and a percentage, such as AS64496:50 or cn:25")
|
"is not a code, : and a percentage, such as AS64496:50 or cn:25")
|
||||||
errNotPercent = errors.New(
|
|
||||||
"is not a percentage, a whole number from 0 to 100",
|
|
||||||
)
|
|
||||||
errNotAbuseIPDBThreshold = errors.New(
|
errNotAbuseIPDBThreshold = errors.New(
|
||||||
"is not a whole number from 25 to 100, the scores AbuseIPDB's " +
|
"is not a whole number from 25 to 100, the scores AbuseIPDB's " +
|
||||||
"blacklist can be cut at",
|
"blacklist can be cut at",
|
||||||
@@ -618,24 +629,24 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
|||||||
"800",
|
"800",
|
||||||
),
|
),
|
||||||
ExclusivelyAllowedCountries: env.countries(
|
ExclusivelyAllowedCountries: env.countries(
|
||||||
"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", ""),
|
ExclusivelyAllowedCountriesSetting, ""),
|
||||||
TagLists: env.tagLists("SWW_WAF_TAG_LISTS"),
|
TagLists: env.tagLists("SWW_WAF_TAG_LISTS"),
|
||||||
TagDeny: env.tags("SWW_WAF_TAG_DENY"),
|
TagDeny: env.tags("SWW_WAF_TAG_DENY"),
|
||||||
TagLimitPercent: env.tagPercents("SWW_WAF_TAG_LIMIT_PERCENT"),
|
TagLimitPercent: env.tagPercents(TagLimitPercentSetting),
|
||||||
RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"),
|
RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"),
|
||||||
ASNBytesLimitPercent: env.percents(
|
ASNBytesLimitPercent: env.percents(
|
||||||
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
|
ASNBytesLimitPercentSetting,
|
||||||
ParseASN,
|
reputation.ParseASN,
|
||||||
),
|
),
|
||||||
CountryBytesLimitPercent: env.percents(
|
CountryBytesLimitPercent: env.percents(
|
||||||
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
|
CountryBytesLimitPercentSetting,
|
||||||
parseCountry,
|
parseCountry,
|
||||||
),
|
),
|
||||||
UnknownCountryLimitPercent: env.percent(
|
UnknownCountryLimitPercent: env.percent(
|
||||||
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
|
UnknownCountryLimitPercentSetting,
|
||||||
"100",
|
"100",
|
||||||
),
|
),
|
||||||
ASNLimitPercentURL: env.listURL("SWW_WAF_ASN_LIMIT_PERCENT_URL"),
|
ASNLimitPercentURL: env.listURL(ASNLimitPercentURLSetting),
|
||||||
BlocklistURLs: env.listURLs("SWW_WAF_BLOCKLIST_URLS"),
|
BlocklistURLs: env.listURLs("SWW_WAF_BLOCKLIST_URLS"),
|
||||||
BlocklistRefresh: env.refresh("SWW_WAF_BLOCKLIST_REFRESH", "24h"),
|
BlocklistRefresh: env.refresh("SWW_WAF_BLOCKLIST_REFRESH", "24h"),
|
||||||
DNSBLZones: env.zones("SWW_WAF_DNSBL_ZONES"),
|
DNSBLZones: env.zones("SWW_WAF_DNSBL_ZONES"),
|
||||||
@@ -809,9 +820,9 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) {
|
|||||||
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
|
cfg.AddLookupHeaders = env.boolean("SWW_WAF_ADD_LOOKUP_HEADERS",
|
||||||
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
|
strconv.FormatBool(cfg.LookupSource != LookupSourceOff))
|
||||||
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
cfg.BlocklistAction, cfg.BlocklistLimitPercent = env.action(
|
||||||
"SWW_WAF_BLOCKLIST_ACTION", string(ReputationActionDeny))
|
BlocklistActionSetting, string(ReputationActionDeny))
|
||||||
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
|
cfg.DNSBLAbuseIPDBAction, cfg.DNSBLAbuseIPDBLimitPercent = env.action(
|
||||||
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION", "limit:25")
|
DNSBLAbuseIPDBActionSetting, "limit:25")
|
||||||
|
|
||||||
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
|
env.checkInstanceNameForNtfy(cfg.InstanceName, cfg.AlertNtfyURL != nil)
|
||||||
env.checkLookupDBPath(cfg)
|
env.checkLookupDBPath(cfg)
|
||||||
@@ -1011,7 +1022,7 @@ func (e *environment) percents(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// tagLists reads SWW_WAF_TAG_LISTS, which is empty by default.
|
// tagLists reads SWW_WAF_TAG_LISTS, which is empty by default.
|
||||||
func (e *environment) tagLists(name string) []TagList {
|
func (e *environment) tagLists(name string) []reputation.TagList {
|
||||||
lists, err := parseTagLists(e.value(name, ""))
|
lists, err := parseTagLists(e.value(name, ""))
|
||||||
e.check(name, err)
|
e.check(name, err)
|
||||||
|
|
||||||
@@ -1037,7 +1048,7 @@ func (e *environment) tagPercents(name string) map[string]int64 {
|
|||||||
|
|
||||||
// percent reads a setting that is a percentage, from 0 to 100.
|
// percent reads a setting that is a percentage, from 0 to 100.
|
||||||
func (e *environment) percent(name, defaultValue string) int64 {
|
func (e *environment) percent(name, defaultValue string) int64 {
|
||||||
percent, err := ParsePercent(e.value(name, defaultValue))
|
percent, err := reputation.ParsePercent(e.value(name, defaultValue))
|
||||||
e.check(name, err)
|
e.check(name, err)
|
||||||
|
|
||||||
return percent
|
return percent
|
||||||
@@ -1122,7 +1133,7 @@ func (e *environment) action(
|
|||||||
string(value), string(ReputationActionLimit)+":",
|
string(value), string(ReputationActionLimit)+":",
|
||||||
)
|
)
|
||||||
|
|
||||||
percent, err := ParsePercent(percentText)
|
percent, err := reputation.ParsePercent(percentText)
|
||||||
if !isLimit || err != nil {
|
if !isLimit || err != nil {
|
||||||
e.check(name, fmt.Errorf("%q %w", value, errNotAction))
|
e.check(name, fmt.Errorf("%q %w", value, errNotAction))
|
||||||
}
|
}
|
||||||
@@ -1131,8 +1142,8 @@ func (e *environment) action(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// zones reads the setting that is the list of DNSBL zones. It is empty by
|
// zones reads the setting that is the list of DNSBL zones. It is empty by
|
||||||
// default. The log shows each zone with its key masked, as MaskZoneKey
|
// default. The log shows each zone with its key masked, as
|
||||||
// masks it.
|
// reputation.MaskZoneKey masks it.
|
||||||
func (e *environment) zones(name string) []string {
|
func (e *environment) zones(name string) []string {
|
||||||
value, _ := e.lookup(name)
|
value, _ := e.lookup(name)
|
||||||
zones, err := parseZones(value)
|
zones, err := parseZones(value)
|
||||||
@@ -1140,7 +1151,7 @@ func (e *environment) zones(name string) []string {
|
|||||||
|
|
||||||
logged := make([]string, len(zones))
|
logged := make([]string, len(zones))
|
||||||
for i, zone := range zones {
|
for i, zone := range zones {
|
||||||
logged[i] = MaskZoneKey(zone)
|
logged[i] = reputation.MaskZoneKey(zone)
|
||||||
}
|
}
|
||||||
|
|
||||||
e.settings = append(
|
e.settings = append(
|
||||||
@@ -1220,7 +1231,7 @@ func (e *environment) checkLookupDBPath(cfg *WAF) {
|
|||||||
func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
||||||
for _, country := range cfg.ExclusivelyAllowedCountries {
|
for _, country := range cfg.ExclusivelyAllowedCountries {
|
||||||
if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) {
|
if slices.Contains(cfg.TagDeny, CountryTagPrefix+country) {
|
||||||
e.check("SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
|
e.check(ExclusivelyAllowedCountriesSetting,
|
||||||
fmt.Errorf("%q %w", country, errOnBothLists))
|
fmt.Errorf("%q %w", country, errOnBothLists))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1239,15 +1250,15 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
|||||||
name string
|
name string
|
||||||
set bool
|
set bool
|
||||||
}{
|
}{
|
||||||
{"SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES", len(cfg.ExclusivelyAllowedCountries) > 0},
|
{ExclusivelyAllowedCountriesSetting, len(cfg.ExclusivelyAllowedCountries) > 0},
|
||||||
{"SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
|
{"SWW_WAF_ADD_LOOKUP_HEADERS", cfg.AddLookupHeaders},
|
||||||
{"SWW_WAF_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, onlyLookupGives)},
|
{"SWW_WAF_TAG_DENY", slices.ContainsFunc(cfg.TagDeny, onlyLookupGives)},
|
||||||
{"SWW_WAF_TAG_LIMIT_PERCENT", slices.ContainsFunc(
|
{TagLimitPercentSetting, slices.ContainsFunc(
|
||||||
slices.Collect(maps.Keys(cfg.TagLimitPercent)), onlyLookupGives)},
|
slices.Collect(maps.Keys(cfg.TagLimitPercent)), onlyLookupGives)},
|
||||||
{"SWW_WAF_ASN_BYTES_LIMIT_PERCENT", len(cfg.ASNBytesLimitPercent) > 0},
|
{ASNBytesLimitPercentSetting, len(cfg.ASNBytesLimitPercent) > 0},
|
||||||
{"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT", len(cfg.CountryBytesLimitPercent) > 0},
|
{CountryBytesLimitPercentSetting, len(cfg.CountryBytesLimitPercent) > 0},
|
||||||
{"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT", cfg.UnknownCountryLimitPercent < 100},
|
{UnknownCountryLimitPercentSetting, cfg.UnknownCountryLimitPercent < 100},
|
||||||
{"SWW_WAF_ASN_LIMIT_PERCENT_URL", cfg.ASNLimitPercentURL != ""},
|
{ASNLimitPercentURLSetting, cfg.ASNLimitPercentURL != ""},
|
||||||
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0},
|
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_MINUTE", cfg.AnomalyASN.RequestsPerMinute > 0},
|
||||||
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0},
|
{"SWW_WAF_ANOMALY_ASN_REQUESTS_PER_HOUR", cfg.AnomalyASN.RequestsPerHour > 0},
|
||||||
{"SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0},
|
{"SWW_WAF_ANOMALY_ASN_BYTES_PER_MINUTE", cfg.AnomalyASN.BytesPerMinute > 0},
|
||||||
@@ -1267,7 +1278,7 @@ func (e *environment) checkCountriesAndLookups(cfg *WAF) {
|
|||||||
// blocklist too: the file at a URL is fetched as one list or the other.
|
// blocklist too: the file at a URL is fetched as one list or the other.
|
||||||
func (e *environment) checkASNLimitPercentURL(cfg *WAF) {
|
func (e *environment) checkASNLimitPercentURL(cfg *WAF) {
|
||||||
if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) {
|
if slices.Contains(cfg.BlocklistURLs, cfg.ASNLimitPercentURL) {
|
||||||
e.check("SWW_WAF_ASN_LIMIT_PERCENT_URL",
|
e.check(ASNLimitPercentURLSetting,
|
||||||
fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs))
|
fmt.Errorf("%q %w", cfg.ASNLimitPercentURL, errInBlocklistURLs))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1294,14 +1305,18 @@ func (e *environment) checkTags(cfg *WAF) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for tag := range cfg.TagLimitPercent {
|
for tag := range cfg.TagLimitPercent {
|
||||||
e.check("SWW_WAF_TAG_LIMIT_PERCENT", checkTag(cfg, tag))
|
e.check(TagLimitPercentSetting, checkTag(cfg, tag))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkTag returns an error when tag would never apply, as checkTags says.
|
// checkTag returns an error when tag would never apply, as checkTags says.
|
||||||
func checkTag(cfg *WAF, tag string) error {
|
func checkTag(cfg *WAF, tag string) error {
|
||||||
if !requestCanGetTag(cfg, tag) {
|
if !requestCanGetTag(cfg, tag) {
|
||||||
return fmt.Errorf("%q %w", MaskZoneKey(tag), errNoRequestGetsTag)
|
return fmt.Errorf(
|
||||||
|
"%q %w",
|
||||||
|
reputation.MaskZoneKey(tag),
|
||||||
|
errNoRequestGetsTag,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
@@ -1309,7 +1324,7 @@ func checkTag(cfg *WAF, tag string) error {
|
|||||||
|
|
||||||
// requestCanGetTag reports whether anything cfg sets up can give a request tag.
|
// requestCanGetTag reports whether anything cfg sets up can give a request tag.
|
||||||
func requestCanGetTag(cfg *WAF, tag string) bool {
|
func requestCanGetTag(cfg *WAF, tag string) bool {
|
||||||
isListName := func(list TagList) bool { return list.Name == tag }
|
isListName := func(list reputation.TagList) bool { return list.Name == tag }
|
||||||
if isLookupTag(tag) || slices.ContainsFunc(cfg.TagLists, isListName) {
|
if isLookupTag(tag) || slices.ContainsFunc(cfg.TagLists, isListName) {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
@@ -1333,7 +1348,7 @@ func requestCanGetTag(cfg *WAF, tag string) bool {
|
|||||||
|
|
||||||
return isZoneTag &&
|
return isZoneTag &&
|
||||||
slices.ContainsFunc(cfg.DNSBLZones, func(each string) bool {
|
slices.ContainsFunc(cfg.DNSBLZones, func(each string) bool {
|
||||||
return strings.ToLower(MaskZoneKey(each)) == zone
|
return strings.ToLower(reputation.MaskZoneKey(each)) == zone
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1831,7 +1846,9 @@ func parseNamedNetblocks(value string) ([]anomaly.NamedNetblock, error) {
|
|||||||
return nil, fmt.Errorf("%q %w", item, errNotNamedNetblock)
|
return nil, fmt.Errorf("%q %w", item, errNotNamedNetblock)
|
||||||
}
|
}
|
||||||
|
|
||||||
netblock, err := ParseNetblock(strings.TrimSpace(netblockText))
|
netblock, err := reputation.ParseNetblock(
|
||||||
|
strings.TrimSpace(netblockText),
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -2037,21 +2054,6 @@ func parseCountry(value string) (string, error) {
|
|||||||
return country, nil
|
return country, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseASN reads an AS number such as AS64496, in either case, and
|
|
||||||
// returns it as the lookup gives it: AS and the number, in capitals and
|
|
||||||
// without leading zeros. The file SWW_WAF_ASN_LIMIT_PERCENT_URL names is
|
|
||||||
// read with it too.
|
|
||||||
func ParseASN(value string) (string, error) {
|
|
||||||
digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS")
|
|
||||||
|
|
||||||
number, err := strconv.ParseUint(digits, 10, 32)
|
|
||||||
if !hasAS || err != nil {
|
|
||||||
return "", fmt.Errorf("%q %w", value, errNotASN)
|
|
||||||
}
|
|
||||||
|
|
||||||
return "AS" + strconv.FormatUint(number, 10), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// The tags a request can get that the settings can name, as waf
|
// The tags a request can get that the settings can name, as waf
|
||||||
// writes them. The prefixes are followed by a value.
|
// writes them. The prefixes are followed by a value.
|
||||||
const (
|
const (
|
||||||
@@ -2065,24 +2067,17 @@ const (
|
|||||||
ExploitSignatureTag = "exploitsignature"
|
ExploitSignatureTag = "exploitsignature"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TagList is an address list of SWW_WAF_TAG_LISTS: the tag it adds to the
|
|
||||||
// requests of the clients it holds, and the URL it is fetched from.
|
|
||||||
type TagList struct {
|
|
||||||
Name string
|
|
||||||
URL string
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseTagLists reads a comma-separated list of name=URL pairs, such as
|
// parseTagLists reads a comma-separated list of name=URL pairs, such as
|
||||||
// torexit=https://example.com/tor.txt, and returns each list with its name
|
// torexit=https://example.com/tor.txt, and returns each list with its name
|
||||||
// in lower case. A name can hold a =, as vpn=mullvad does, and ends at the
|
// in lower case. A name can hold a =, as vpn=mullvad does, and ends at the
|
||||||
// = before http. A name or a URL listed twice is an error.
|
// = before http. A name or a URL listed twice is an error.
|
||||||
func parseTagLists(value string) ([]TagList, error) {
|
func parseTagLists(value string) ([]reputation.TagList, error) {
|
||||||
items, err := parseList(value)
|
items, err := parseList(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
lists := make([]TagList, 0, len(items))
|
lists := make([]reputation.TagList, 0, len(items))
|
||||||
|
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
end := strings.Index(item, "=http")
|
end := strings.Index(item, "=http")
|
||||||
@@ -2090,9 +2085,12 @@ func parseTagLists(value string) ([]TagList, error) {
|
|||||||
return nil, fmt.Errorf("%q %w", item, errNotTagList)
|
return nil, fmt.Errorf("%q %w", item, errNotTagList)
|
||||||
}
|
}
|
||||||
|
|
||||||
list := TagList{Name: strings.ToLower(item[:end]), URL: item[end+1:]}
|
list := reputation.TagList{
|
||||||
|
Name: strings.ToLower(item[:end]),
|
||||||
|
URL: item[end+1:],
|
||||||
|
}
|
||||||
|
|
||||||
if slices.ContainsFunc(lists, func(earlier TagList) bool {
|
if slices.ContainsFunc(lists, func(earlier reputation.TagList) bool {
|
||||||
return earlier.Name == list.Name || earlier.URL == list.URL
|
return earlier.Name == list.Name || earlier.URL == list.URL
|
||||||
}) {
|
}) {
|
||||||
return nil, fmt.Errorf("%q %w", item, errListedTwice)
|
return nil, fmt.Errorf("%q %w", item, errListedTwice)
|
||||||
@@ -2105,7 +2103,8 @@ func parseTagLists(value string) ([]TagList, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// parseTags reads a comma-separated list of tags, which parseTag reads. A
|
// parseTags reads a comma-separated list of tags, which parseTag reads. A
|
||||||
// tag listed twice is an error, which shows the tag as MaskZoneKey does.
|
// tag listed twice is an error, which shows the tag as
|
||||||
|
// reputation.MaskZoneKey does.
|
||||||
func parseTags(value string) ([]string, error) {
|
func parseTags(value string) ([]string, error) {
|
||||||
items, err := parseList(value)
|
items, err := parseList(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -2125,7 +2124,7 @@ func parseTags(value string) ([]string, error) {
|
|||||||
if slices.Contains(tags, tag) {
|
if slices.Contains(tags, tag) {
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"%q %w",
|
"%q %w",
|
||||||
MaskZoneKey(item),
|
reputation.MaskZoneKey(item),
|
||||||
errListedTwice,
|
errListedTwice,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -2194,7 +2193,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
|
|||||||
if colon <= 0 {
|
if colon <= 0 {
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"%q %w",
|
"%q %w",
|
||||||
MaskZoneKey(item),
|
reputation.MaskZoneKey(item),
|
||||||
errNotTagPercentItem,
|
errNotTagPercentItem,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -2208,7 +2207,7 @@ func parseTagPercents(value string) (map[string]int64, error) {
|
|||||||
if err != nil || percent < 1 || percent > 99 {
|
if err != nil || percent < 1 || percent > 99 {
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"%q %w",
|
"%q %w",
|
||||||
MaskZoneKey(item[colon+1:]),
|
reputation.MaskZoneKey(item[colon+1:]),
|
||||||
errNotTagPercent,
|
errNotTagPercent,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -2220,7 +2219,11 @@ func parseTagPercents(value string) (map[string]int64, error) {
|
|||||||
if _, listed := percents[tag]; listed {
|
if _, listed := percents[tag]; listed {
|
||||||
// tag, since item[:colon] may end in a space, which MaskZoneKey
|
// tag, since item[:colon] may end in a space, which MaskZoneKey
|
||||||
// would not mask.
|
// would not mask.
|
||||||
return nil, fmt.Errorf("%q %w", MaskZoneKey(tag), errListedTwice)
|
return nil, fmt.Errorf(
|
||||||
|
"%q %w",
|
||||||
|
reputation.MaskZoneKey(tag),
|
||||||
|
errListedTwice,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
percents[tag] = percent
|
percents[tag] = percent
|
||||||
@@ -2254,7 +2257,7 @@ func parsePercents(
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
percent, err := ParsePercent(percentText)
|
percent, err := reputation.ParsePercent(percentText)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -2269,38 +2272,6 @@ func parsePercents(
|
|||||||
return percents, nil
|
return percents, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParsePercent reads a percentage, a whole number from 0 to 100. The file
|
|
||||||
// SWW_WAF_ASN_LIMIT_PERCENT_URL names is read with it too.
|
|
||||||
func ParsePercent(value string) (int64, error) {
|
|
||||||
percent, err := strconv.ParseInt(value, 10, 64)
|
|
||||||
if err != nil || percent < 0 || percent > 100 {
|
|
||||||
return 0, fmt.Errorf("%q %w", value, errNotPercent)
|
|
||||||
}
|
|
||||||
|
|
||||||
return percent, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// headerNameChars are the characters RFC 9110 allows in a header name:
|
|
||||||
// letters, digits and these marks.
|
|
||||||
const headerNameChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" +
|
|
||||||
"0123456789!#$%&'*+-.^_`|~"
|
|
||||||
|
|
||||||
// IsHeaderName reports whether name can be a header name: one or more of
|
|
||||||
// the characters RFC 9110 allows in one.
|
|
||||||
func IsHeaderName(name string) bool {
|
|
||||||
if name == "" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, char := range name {
|
|
||||||
if !strings.ContainsRune(headerNameChars, char) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseHeaderNames reads a comma-separated list of header names in either
|
// parseHeaderNames reads a comma-separated list of header names in either
|
||||||
// case, and returns them in lower case. Host and Transfer-Encoding are
|
// case, and returns them in lower case. Host and Transfer-Encoding are
|
||||||
// refused: Go's HTTP server takes them out of the request's headers.
|
// refused: Go's HTTP server takes them out of the request's headers.
|
||||||
@@ -2313,7 +2284,7 @@ func parseHeaderNames(value string) ([]string, error) {
|
|||||||
headers := make([]string, 0, len(items))
|
headers := make([]string, 0, len(items))
|
||||||
|
|
||||||
for _, item := range items {
|
for _, item := range items {
|
||||||
if !IsHeaderName(item) {
|
if !httpguts.ValidHeaderFieldName(item) {
|
||||||
return nil, fmt.Errorf("%q %w", item, errNotHeaderName)
|
return nil, fmt.Errorf("%q %w", item, errNotHeaderName)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2469,8 +2440,8 @@ const (
|
|||||||
// other name for one that does not exist, so that the zone would list no
|
// other name for one that does not exist, so that the zone would list no
|
||||||
// client. A zone listed twice is an error, whatever the case of its
|
// client. A zone listed twice is an error, whatever the case of its
|
||||||
// letters, which DNS names ignore, and whatever its key, since
|
// letters, which DNS names ignore, and whatever its key, since
|
||||||
// MaskZoneKey shows two keys of one zone alike. An error shows a zone as
|
// reputation.MaskZoneKey shows two keys of one zone alike. An error shows a
|
||||||
// MaskZoneKey does.
|
// zone as it does.
|
||||||
func parseZones(value string) ([]string, error) {
|
func parseZones(value string) ([]string, error) {
|
||||||
zones, err := parseList(value)
|
zones, err := parseList(value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -2480,11 +2451,14 @@ func parseZones(value string) ([]string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for i, zone := range zones {
|
for i, zone := range zones {
|
||||||
shown := MaskZoneKey(zone)
|
shown := reputation.MaskZoneKey(zone)
|
||||||
listedBefore := slices.ContainsFunc(
|
listedBefore := slices.ContainsFunc(
|
||||||
zones[:i],
|
zones[:i],
|
||||||
func(earlier string) bool {
|
func(earlier string) bool {
|
||||||
return strings.EqualFold(MaskZoneKey(earlier), shown)
|
return strings.EqualFold(
|
||||||
|
reputation.MaskZoneKey(earlier),
|
||||||
|
shown,
|
||||||
|
)
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -2501,24 +2475,6 @@ func parseZones(value string) ([]string, error) {
|
|||||||
return zones, nil
|
return zones, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// MaskZoneKey returns zone with ******** in place of its key, if it is a
|
|
||||||
// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net,
|
|
||||||
// such as <key>.xbl.dq.spamhaus.net, whose first label is the key. Any
|
|
||||||
// other zone it returns as it is. A zone is shown so wherever it leaves
|
|
||||||
// the process: in the log, the alerts and the metrics.
|
|
||||||
func MaskZoneKey(zone string) string {
|
|
||||||
// DNS names ignore case, and a name may be written with a dot at its
|
|
||||||
// end.
|
|
||||||
name := strings.TrimSuffix(strings.ToLower(zone), ".")
|
|
||||||
if !strings.HasSuffix(name, ".dq.spamhaus.net") {
|
|
||||||
return zone
|
|
||||||
}
|
|
||||||
|
|
||||||
_, rest, _ := strings.Cut(zone, ".")
|
|
||||||
|
|
||||||
return masked + "." + rest
|
|
||||||
}
|
|
||||||
|
|
||||||
// isZone reports whether each label of zone is as parseZones takes it. It
|
// isZone reports whether each label of zone is as parseZones takes it. It
|
||||||
// also checks the hostnames of SWW_WAF_APPS, which are DNS names too.
|
// also checks the hostnames of SWW_WAF_APPS, which are DNS names too.
|
||||||
func isZone(zone string) bool {
|
func isZone(zone string) bool {
|
||||||
@@ -2574,7 +2530,7 @@ func parseWebhookHeaders(value string) (http.Header, string, error) {
|
|||||||
name, headerValue, found := strings.Cut(item, ":")
|
name, headerValue, found := strings.Cut(item, ":")
|
||||||
name = strings.TrimSpace(name)
|
name = strings.TrimSpace(name)
|
||||||
|
|
||||||
if !found || !IsHeaderName(name) ||
|
if !found || !httpguts.ValidHeaderFieldName(name) ||
|
||||||
strings.ContainsAny(headerValue, "\r\n\x00") {
|
strings.ContainsAny(headerValue, "\r\n\x00") {
|
||||||
return nil, "", fmt.Errorf(
|
return nil, "", fmt.Errorf(
|
||||||
"item %d %w",
|
"item %d %w",
|
||||||
|
|||||||
+29
-20
@@ -18,10 +18,12 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
"golang.org/x/net/http/httpguts"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
"sneak.berlin/go/smallwebwaf/internal/anomaly"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
"sneak.berlin/go/smallwebwaf/internal/config/configtest"
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The settings, by name.
|
// The settings, by name.
|
||||||
@@ -864,6 +866,32 @@ func TestInstanceNameAndLoggedHeadersAsSet(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// config and rules check header names with httpguts.ValidHeaderFieldName.
|
||||||
|
// It takes a name of one or more of the characters RFC 9110 allows in one,
|
||||||
|
// as the check it replaced did: letters, digits and these marks.
|
||||||
|
func TestHeaderNameCheckTakesTheRFC9110Characters(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const marks = "!#$%&'*+-.^_`|~"
|
||||||
|
|
||||||
|
if httpguts.ValidHeaderFieldName("") {
|
||||||
|
t.Error("the empty name is taken")
|
||||||
|
}
|
||||||
|
|
||||||
|
for b := range 256 {
|
||||||
|
char := byte(b)
|
||||||
|
want := 'a' <= char && char <= 'z' || 'A' <= char && char <= 'Z' ||
|
||||||
|
'0' <= char && char <= '9' || strings.IndexByte(marks, char) >= 0
|
||||||
|
|
||||||
|
// string(char) would be the UTF-8 of U+0080 to U+00FF, two bytes,
|
||||||
|
// for a byte above 0x7f.
|
||||||
|
name := string([]byte{char})
|
||||||
|
if got := httpguts.ValidHeaderFieldName(name); got != want {
|
||||||
|
t.Errorf("the name %q is taken: %t, want %t", name, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRemoteLogSettingsDefaults(t *testing.T) {
|
func TestRemoteLogSettingsDefaults(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -1694,7 +1722,7 @@ func TestTagSettingsAsSet(t *testing.T) {
|
|||||||
tagLimitPercent: "vpn=mullvad :50, asn=014061:25, COUNTRY=cn:1, dnsbl=zen.example:99",
|
tagLimitPercent: "vpn=mullvad :50, asn=014061:25, COUNTRY=cn:1, dnsbl=zen.example:99",
|
||||||
})
|
})
|
||||||
|
|
||||||
wantLists := []config.TagList{
|
wantLists := []reputation.TagList{
|
||||||
{Name: torExitTag, URL: torURL},
|
{Name: torExitTag, URL: torURL},
|
||||||
{Name: "vpn=mullvad", URL: "https://example.com/mullvad.txt"},
|
{Name: "vpn=mullvad", URL: "https://example.com/mullvad.txt"},
|
||||||
}
|
}
|
||||||
@@ -2288,25 +2316,6 @@ func TestInvalidDNSBLSettingStopsTheStartSayingWhatIsWrong(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for zone, want := range map[string]string{
|
|
||||||
spamhaus: spamhausMasked,
|
|
||||||
spamhaus + ".": spamhausMasked + ".",
|
|
||||||
"KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET",
|
|
||||||
dronebl: dronebl,
|
|
||||||
"dq.spamhaus.net": "dq.spamhaus.net",
|
|
||||||
spamhaus + ".example": spamhaus + ".example",
|
|
||||||
} {
|
|
||||||
if got := config.MaskZoneKey(zone); got != want {
|
|
||||||
t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDNSBLZoneKeyIsLoggedMaskedAndNeverShown(t *testing.T) {
|
func TestDNSBLZoneKeyIsLoggedMaskedAndNeverShown(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
+47
-14
@@ -99,8 +99,13 @@ type GeoJS struct {
|
|||||||
// waiting are the clients without an answer: those to ask GeoJS about,
|
// waiting are the clients without an answer: those to ask GeoJS about,
|
||||||
// and those it is being asked about.
|
// and those it is being asked about.
|
||||||
waiting map[netip.Prefix]*wait
|
waiting map[netip.Prefix]*wait
|
||||||
// asking is true while a request to GeoJS is under way.
|
// asking is true while a request to GeoJS is under way, and endAsking
|
||||||
asking bool
|
// ends it. stopped is true once StopAsking has been called, and
|
||||||
|
// askGoroutines waits for the goroutine that makes the requests.
|
||||||
|
asking bool
|
||||||
|
endAsking context.CancelFunc
|
||||||
|
stopped bool
|
||||||
|
askGoroutines sync.WaitGroup
|
||||||
// retryDelay is how long GeoJS is left alone after its last failure,
|
// retryDelay is how long GeoJS is left alone after its last failure,
|
||||||
// zero after an answer; retryAt is when it may be asked again.
|
// zero after an answer; retryAt is when it may be asked again.
|
||||||
retryDelay time.Duration
|
retryDelay time.Duration
|
||||||
@@ -210,6 +215,23 @@ func (g *GeoJS) LookUpWithoutWaiting(
|
|||||||
return answer
|
return answer
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StopAsking ends the request to GeoJS under way, and returns once the
|
||||||
|
// goroutine that made it has. A request it ends is no failure: Failed is not
|
||||||
|
// called, and GeoJS is not left alone after it. After StopAsking, GeoJS is
|
||||||
|
// asked nothing.
|
||||||
|
func (g *GeoJS) StopAsking() {
|
||||||
|
g.mu.Lock()
|
||||||
|
|
||||||
|
g.stopped = true
|
||||||
|
if g.asking {
|
||||||
|
g.endAsking()
|
||||||
|
}
|
||||||
|
|
||||||
|
g.mu.Unlock()
|
||||||
|
|
||||||
|
g.askGoroutines.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
// Kept returns client's answer, if one is kept, without asking GeoJS.
|
// Kept returns client's answer, if one is kept, without asking GeoJS.
|
||||||
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
func (g *GeoJS) Kept(client netip.Prefix) (Answer, bool) {
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
@@ -294,7 +316,8 @@ func (g *GeoJS) answerOrWait(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if !g.asking {
|
if !g.asking {
|
||||||
// GeoJS is left alone after a failure, so no answer can come.
|
// GeoJS is left alone after a failure, or asked nothing after
|
||||||
|
// StopAsking, so no answer can come.
|
||||||
w.late = true
|
w.late = true
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -323,22 +346,28 @@ func (g *GeoJS) kept(client netip.Prefix) (Answer, bool) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ask starts asking GeoJS about the waiting clients, unless a request to
|
// ask starts asking GeoJS about the waiting clients, unless a request to
|
||||||
// it is under way or it is left alone after a failure. The requests to
|
// it is under way, it is left alone after a failure, or StopAsking has
|
||||||
// GeoJS are for every client waiting, so they go on when the caller whose
|
// been called. The requests to GeoJS are for every client waiting, so they
|
||||||
// ctx is given stops waiting.
|
// go on when the caller whose ctx is given stops waiting, until StopAsking.
|
||||||
func (g *GeoJS) ask(ctx context.Context) {
|
func (g *GeoJS) ask(ctx context.Context) {
|
||||||
if g.asking || g.now().Before(g.retryAt) {
|
if g.asking || g.stopped || g.now().Before(g.retryAt) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
g.asking = true
|
askCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||||
|
g.asking, g.endAsking = true, end
|
||||||
|
|
||||||
go g.askAboutWaiting(context.WithoutCancel(ctx))
|
g.askGoroutines.Go(func() {
|
||||||
|
defer end()
|
||||||
|
|
||||||
|
g.askAboutWaiting(askCtx)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
|
// askAboutWaiting asks GeoJS about the waiting clients, one request at a
|
||||||
// time, until none is left or GeoJS fails. Each answer kept is given to
|
// time, until none is left, GeoJS fails, or StopAsking ends ctx. Each
|
||||||
// Answered, outside the lock, since Answered takes locks of its own.
|
// answer kept is given to Answered, outside the lock, since Answered takes
|
||||||
|
// locks of its own.
|
||||||
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
||||||
for {
|
for {
|
||||||
clients := g.nextClients()
|
clients := g.nextClients()
|
||||||
@@ -347,6 +376,10 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
given, err := g.request(ctx, clients)
|
given, err := g.request(ctx, clients)
|
||||||
|
if err != nil && ctx.Err() != nil {
|
||||||
|
continue // ended by StopAsking, which is no failure
|
||||||
|
}
|
||||||
|
|
||||||
kept, answered := g.keep(clients, given, err)
|
kept, answered := g.keep(clients, given, err)
|
||||||
|
|
||||||
if g.answered != nil {
|
if g.answered != nil {
|
||||||
@@ -362,13 +395,13 @@ func (g *GeoJS) askAboutWaiting(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// nextClients returns up to maxPerRequest of the waiting clients. When
|
// nextClients returns up to maxPerRequest of the waiting clients. When
|
||||||
// none is waiting, it returns none and notes that no request to GeoJS is
|
// none is waiting, or StopAsking has been called, it returns none and
|
||||||
// under way.
|
// notes that no request to GeoJS is under way.
|
||||||
func (g *GeoJS) nextClients() []netip.Prefix {
|
func (g *GeoJS) nextClients() []netip.Prefix {
|
||||||
g.mu.Lock()
|
g.mu.Lock()
|
||||||
defer g.mu.Unlock()
|
defer g.mu.Unlock()
|
||||||
|
|
||||||
if len(g.waiting) == 0 {
|
if len(g.waiting) == 0 || g.stopped {
|
||||||
g.asking = false
|
g.asking = false
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -318,6 +318,44 @@ func TestCallerWhoseContextEndsEarlyDoesNotStopTheRequest(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStopAskingEndsTheRequestUnderWayAsNoFailureAndAsksNoMore(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
service, _, g, reports := startWithReports()
|
||||||
|
clients := newClients()
|
||||||
|
|
||||||
|
service.set(hanging)
|
||||||
|
g.LookUpWithoutWaiting(t.Context(), clients())
|
||||||
|
waitForRequests(t, service, 1)
|
||||||
|
|
||||||
|
// The request is ended, not waited out, and is no failure.
|
||||||
|
began := time.Now()
|
||||||
|
|
||||||
|
g.StopAsking()
|
||||||
|
|
||||||
|
if waited := time.Since(began); waited != 0 {
|
||||||
|
t.Errorf("StopAsking returned after %s, want at once", waited)
|
||||||
|
}
|
||||||
|
|
||||||
|
reports.wantFailures(t)
|
||||||
|
|
||||||
|
// A new client is then not asked about, and its lookup does not
|
||||||
|
// wait.
|
||||||
|
service.set(answering)
|
||||||
|
wantCountry(t, g, clients(), "")
|
||||||
|
|
||||||
|
if waited := time.Since(began); waited != 0 {
|
||||||
|
t.Errorf("the lookup returned after %s, want at once", waited)
|
||||||
|
}
|
||||||
|
|
||||||
|
waitForRequests(t, service, 1)
|
||||||
|
reports.wantFailures(t)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func TestRequestOver10SecondsIsAFailure(t *testing.T) {
|
func TestRequestOver10SecondsIsAFailure(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ import (
|
|||||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/bans"
|
"sneak.berlin/go/smallwebwaf/internal/bans"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
"sneak.berlin/go/smallwebwaf/internal/ratelimit"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
"sneak.berlin/go/smallwebwaf/internal/remotelog"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
"sneak.berlin/go/smallwebwaf/internal/reputation"
|
||||||
@@ -301,7 +300,7 @@ const sourceLabel = "source"
|
|||||||
|
|
||||||
// AddReputation adds the metrics of the lists fetched from URLs and of the
|
// AddReputation adds the metrics of the lists fetched from URLs and of the
|
||||||
// DNSBL zones, by source, each list's URL or each zone, its key masked as
|
// DNSBL zones, by source, each list's URL or each zone, its key masked as
|
||||||
// config.MaskZoneKey masks it: the requests whose client a blocklist, a
|
// reputation.MaskZoneKey masks it: the requests whose client a blocklist, a
|
||||||
// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts,
|
// zone's verdict or AbuseIPDB's score lists, which ReputationHit counts,
|
||||||
// and, read from lists and dnsbl as the metrics are asked for, for a list,
|
// and, read from lists and dnsbl as the metrics are asked for, for a list,
|
||||||
// the fetches that failed and when the copy in use was fetched, and for a
|
// the fetches that failed and when the copy in use was fetched, and for a
|
||||||
@@ -320,7 +319,9 @@ func (m *Metrics) AddReputation(
|
|||||||
m.registry.MustRegister(m.reputationHits)
|
m.registry.MustRegister(m.reputationHits)
|
||||||
|
|
||||||
for _, zone := range dnsbl.Zones() {
|
for _, zone := range dnsbl.Zones() {
|
||||||
source := prometheus.Labels{sourceLabel: config.MaskZoneKey(zone)}
|
source := prometheus.Labels{
|
||||||
|
sourceLabel: reputation.MaskZoneKey(zone),
|
||||||
|
}
|
||||||
|
|
||||||
m.addReputationQueries(
|
m.addReputationQueries(
|
||||||
source,
|
source,
|
||||||
|
|||||||
@@ -110,8 +110,12 @@ type AbuseIPDB struct {
|
|||||||
// scores are by client. Each is added as it is fetched and never moved
|
// scores are by client. Each is added as it is fetched and never moved
|
||||||
// up, so that the one fetched longest ago is the first dropped.
|
// up, so that the one fetched longest ago is the first dropped.
|
||||||
scores *simplelru.LRU[netip.Prefix, Score]
|
scores *simplelru.LRU[netip.Prefix, Score]
|
||||||
// checking are the clients whose check is under way.
|
// checking are the clients whose check is under way, each with the
|
||||||
checking map[netip.Prefix]bool
|
// function that ends it. stopped is true once StopChecking has been
|
||||||
|
// called, and checkGoroutines waits for the goroutine of each check.
|
||||||
|
checking map[netip.Prefix]context.CancelFunc
|
||||||
|
stopped bool
|
||||||
|
checkGoroutines sync.WaitGroup
|
||||||
// day is the day, in UTC, of the checks spent counts.
|
// day is the day, in UTC, of the checks spent counts.
|
||||||
day civil.Date
|
day civil.Date
|
||||||
spent int
|
spent int
|
||||||
@@ -134,7 +138,7 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
|||||||
params: params,
|
params: params,
|
||||||
httpClient: &http.Client{},
|
httpClient: &http.Client{},
|
||||||
scores: scores,
|
scores: scores,
|
||||||
checking: map[netip.Prefix]bool{},
|
checking: map[netip.Prefix]context.CancelFunc{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -144,11 +148,11 @@ func NewAbuseIPDB(params AbuseIPDBParams) *AbuseIPDB {
|
|||||||
// addresses its request comes from. A client without one is checked in
|
// addresses its request comes from. A client without one is checked in
|
||||||
// the background, by addr, the address its request came from, if
|
// the background, by addr, the address its request came from, if
|
||||||
// offender, if it has committed an offence, unless its check is under
|
// offender, if it has committed an offence, unless its check is under
|
||||||
// way, a check failed less than failureDelay ago, or the day's checks
|
// way, a check failed less than failureDelay ago, the day's checks have
|
||||||
// have used up DailyBudget; Hit never waits for a check. The check that
|
// used up DailyBudget, or StopChecking has been called; Hit never waits
|
||||||
// uses the budget up is logged and raised as a source_failure alert. ctx
|
// for a check. The check that uses the budget up is logged and raised as a
|
||||||
// is the context of the client's request, and a check goes on after the
|
// source_failure alert. ctx is the context of the client's request, and a
|
||||||
// request ends.
|
// check goes on after the request ends, until StopChecking.
|
||||||
func (a *AbuseIPDB) Hit(
|
func (a *AbuseIPDB) Hit(
|
||||||
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
|
ctx context.Context, client netip.Prefix, addr netip.Addr, offender bool,
|
||||||
) (int64, bool) {
|
) (int64, bool) {
|
||||||
@@ -167,14 +171,21 @@ func (a *AbuseIPDB) Hit(
|
|||||||
a.day, a.spent = today, 0
|
a.day, a.spent = today, 0
|
||||||
}
|
}
|
||||||
|
|
||||||
check := offender && !a.checking[client] && !now.Before(a.retryAt) &&
|
_, checking := a.checking[client]
|
||||||
a.spent < a.params.DailyBudget
|
|
||||||
|
check := offender && !checking && !a.stopped &&
|
||||||
|
!now.Before(a.retryAt) && a.spent < a.params.DailyBudget
|
||||||
if check {
|
if check {
|
||||||
a.checking[client] = true
|
checkCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||||
|
a.checking[client] = end
|
||||||
a.checks++
|
a.checks++
|
||||||
a.spent++
|
a.spent++
|
||||||
|
|
||||||
go a.check(context.WithoutCancel(ctx), client, addr)
|
a.checkGoroutines.Go(func() {
|
||||||
|
defer end()
|
||||||
|
|
||||||
|
a.check(checkCtx, client, addr)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
usedUp := check && a.spent == a.params.DailyBudget
|
usedUp := check && a.spent == a.params.DailyBudget
|
||||||
@@ -189,6 +200,24 @@ func (a *AbuseIPDB) Hit(
|
|||||||
return 0, false
|
return 0, false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StopChecking ends the checks under way, and returns once their goroutines
|
||||||
|
// have. A check it ends gives no score and is no failure: it is neither
|
||||||
|
// counted, logged nor raised as an alert, and checks are not left alone
|
||||||
|
// after it. The ended check stays spent from the day's budget. After
|
||||||
|
// StopChecking, Hit checks no client.
|
||||||
|
func (a *AbuseIPDB) StopChecking() {
|
||||||
|
a.mu.Lock()
|
||||||
|
|
||||||
|
a.stopped = true
|
||||||
|
for _, end := range a.checking {
|
||||||
|
end()
|
||||||
|
}
|
||||||
|
|
||||||
|
a.mu.Unlock()
|
||||||
|
|
||||||
|
a.checkGoroutines.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
// Checked returns how many checks were made.
|
// Checked returns how many checks were made.
|
||||||
func (a *AbuseIPDB) Checked() int {
|
func (a *AbuseIPDB) Checked() int {
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
@@ -315,7 +344,8 @@ func (a *AbuseIPDB) RunBlacklist(ctx context.Context) {
|
|||||||
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
|
// check checks client with AbuseIPDB by addr, one of its addresses, keeps
|
||||||
// the score as client's, and notes the check as no longer under way. A
|
// the score as client's, and notes the check as no longer under way. A
|
||||||
// check that fails gives no score: it is counted, logged and raised as a
|
// check that fails gives no score: it is counted, logged and raised as a
|
||||||
// source_failure alert, and no client is checked for failureDelay.
|
// source_failure alert, and no client is checked for failureDelay. A check
|
||||||
|
// ended as ctx ends, by StopChecking, is no failure.
|
||||||
func (a *AbuseIPDB) check(
|
func (a *AbuseIPDB) check(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
client netip.Prefix,
|
client netip.Prefix,
|
||||||
@@ -323,24 +353,26 @@ func (a *AbuseIPDB) check(
|
|||||||
) {
|
) {
|
||||||
score, err := a.ask(ctx, addr)
|
score, err := a.ask(ctx, addr)
|
||||||
now := a.params.Now()
|
now := a.params.Now()
|
||||||
|
isFailure := err != nil && ctx.Err() == nil
|
||||||
|
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
|
|
||||||
delete(a.checking, client)
|
delete(a.checking, client)
|
||||||
|
|
||||||
if err == nil {
|
switch {
|
||||||
|
case err == nil:
|
||||||
a.scores.Add(
|
a.scores.Add(
|
||||||
client,
|
client,
|
||||||
Score{Client: client, Score: score, Fetched: now},
|
Score{Client: client, Score: score, Fetched: now},
|
||||||
)
|
)
|
||||||
} else {
|
case isFailure:
|
||||||
a.failures++
|
a.failures++
|
||||||
a.retryAt = now.Add(failureDelay)
|
a.retryAt = now.Add(failureDelay)
|
||||||
}
|
}
|
||||||
|
|
||||||
a.mu.Unlock()
|
a.mu.Unlock()
|
||||||
|
|
||||||
if err != nil {
|
if isFailure {
|
||||||
a.alert("checking a client with AbuseIPDB failed", err)
|
a.alert("checking a client with AbuseIPDB failed", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -394,6 +394,43 @@ func TestCheckNotAnsweredWithinTheTimeoutFailsAndHitNeverWaits(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStopCheckingEndsTheCheckUnderWayAsNoFailureAndChecksNoMore(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
queue := newQueue()
|
||||||
|
p := abuseIPDBParams()
|
||||||
|
p.Alerts = queue
|
||||||
|
abuseIPDB := &abuseIPDBStandIn{hanging: true}
|
||||||
|
checker := newAbuseIPDB(abuseIPDB, p)
|
||||||
|
|
||||||
|
wantScore(t, checker, suspect, true, 0, false)
|
||||||
|
synctest.Wait()
|
||||||
|
|
||||||
|
// The check is ended, not waited out, and is no failure.
|
||||||
|
began := time.Now()
|
||||||
|
|
||||||
|
checker.StopChecking()
|
||||||
|
|
||||||
|
if waited := time.Since(began); waited != 0 {
|
||||||
|
t.Errorf("StopChecking returned after %s, want at once", waited)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantFailures(t, checker, 0)
|
||||||
|
|
||||||
|
if got := waiting(queue); len(got) != 0 {
|
||||||
|
t.Errorf("alerts waiting %+v, want none", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// An offender without a score is then not checked.
|
||||||
|
wantScore(t, checker, other, true, 0, false)
|
||||||
|
synctest.Wait()
|
||||||
|
wantChecked(t, abuseIPDB, suspect)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
func TestKeyIsSentInTheKeyHeaderAndNeverShown(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,6 @@ import (
|
|||||||
|
|
||||||
"github.com/hashicorp/golang-lru/v2/simplelru"
|
"github.com/hashicorp/golang-lru/v2/simplelru"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -32,6 +31,8 @@ const (
|
|||||||
// it fails, and no client is checked with AbuseIPDB after a check
|
// it fails, and no client is checked with AbuseIPDB after a check
|
||||||
// fails, so that a source refusing them is not asked on every request.
|
// fails, so that a source refusing them is not asked on every request.
|
||||||
failureDelay = time.Minute
|
failureDelay = time.Minute
|
||||||
|
// maskedKey is what MaskZoneKey shows in place of a zone's key.
|
||||||
|
maskedKey = "********"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -82,8 +83,12 @@ type DNSBL struct {
|
|||||||
// verdicts are by query. Each is added as it is fetched and never moved
|
// verdicts are by query. Each is added as it is fetched and never moved
|
||||||
// up, so that the one fetched longest ago is the first dropped.
|
// up, so that the one fetched longest ago is the first dropped.
|
||||||
verdicts *simplelru.LRU[query, Verdict]
|
verdicts *simplelru.LRU[query, Verdict]
|
||||||
// asking are the queries under way.
|
// asking are the queries under way, each with the function that ends
|
||||||
asking map[query]bool
|
// it. stopped is true once StopAsking has been called, and
|
||||||
|
// queryGoroutines waits for the goroutine of each query.
|
||||||
|
asking map[query]context.CancelFunc
|
||||||
|
stopped bool
|
||||||
|
queryGoroutines sync.WaitGroup
|
||||||
// queries and failures count, by zone, the queries made and those that
|
// queries and failures count, by zone, the queries made and those that
|
||||||
// failed, and retryAt is when a zone whose last query failed may be
|
// failed, and retryAt is when a zone whose last query failed may be
|
||||||
// asked again.
|
// asked again.
|
||||||
@@ -120,13 +125,31 @@ func NewDNSBL(params DNSBLParams) *DNSBL {
|
|||||||
params: params,
|
params: params,
|
||||||
resolver: resolver,
|
resolver: resolver,
|
||||||
verdicts: verdicts,
|
verdicts: verdicts,
|
||||||
asking: map[query]bool{},
|
asking: map[query]context.CancelFunc{},
|
||||||
queries: map[string]int{},
|
queries: map[string]int{},
|
||||||
failures: map[string]int{},
|
failures: map[string]int{},
|
||||||
retryAt: map[string]time.Time{},
|
retryAt: map[string]time.Time{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MaskZoneKey returns zone with ******** in place of its key, if it is a
|
||||||
|
// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net,
|
||||||
|
// such as <key>.xbl.dq.spamhaus.net, whose first label is the key. Any
|
||||||
|
// other zone it returns as it is. A zone is shown so wherever it leaves
|
||||||
|
// the process: in the log, the alerts and the metrics.
|
||||||
|
func MaskZoneKey(zone string) string {
|
||||||
|
// DNS names ignore case, and a name may be written with a dot at its
|
||||||
|
// end.
|
||||||
|
name := strings.TrimSuffix(strings.ToLower(zone), ".")
|
||||||
|
if !strings.HasSuffix(name, ".dq.spamhaus.net") {
|
||||||
|
return zone
|
||||||
|
}
|
||||||
|
|
||||||
|
_, rest, _ := strings.Cut(zone, ".")
|
||||||
|
|
||||||
|
return maskedKey + "." + rest
|
||||||
|
}
|
||||||
|
|
||||||
// Zones returns the zones, in the order SWW_WAF_DNSBL_ZONES names them.
|
// Zones returns the zones, in the order SWW_WAF_DNSBL_ZONES names them.
|
||||||
func (d *DNSBL) Zones() []string {
|
func (d *DNSBL) Zones() []string {
|
||||||
return slices.Clone(d.params.Zones)
|
return slices.Clone(d.params.Zones)
|
||||||
@@ -134,13 +157,14 @@ func (d *DNSBL) Zones() []string {
|
|||||||
|
|
||||||
// ListedBy returns the zones whose verdict on addr, a client's address,
|
// ListedBy returns the zones whose verdict on addr, a client's address,
|
||||||
// lists it, in the order SWW_WAF_DNSBL_ZONES names them, each with its key
|
// lists it, in the order SWW_WAF_DNSBL_ZONES names them, each with its key
|
||||||
// masked, as config.MaskZoneKey masks it, since they go to the request
|
// masked, as MaskZoneKey masks it, since they go to the request
|
||||||
// log, the alerts and the metrics. A verdict is used until CacheTTL has
|
// log, the alerts and the metrics. A verdict is used until CacheTTL has
|
||||||
// passed since it was fetched. Each zone without one is asked about addr
|
// passed since it was fetched. Each zone without one is asked about addr
|
||||||
// in the background, unless a query about addr to it is under way, the
|
// in the background, unless a query about addr to it is under way, the
|
||||||
// zone is left alone after a failure, or maxQueries are under way;
|
// zone is left alone after a failure, maxQueries are under way, or
|
||||||
// ListedBy never waits for a query. ctx is the context of the client's
|
// StopAsking has been called; ListedBy never waits for a query. ctx is the
|
||||||
// request, and a query goes on after the request ends.
|
// context of the client's request, and a query goes on after the request
|
||||||
|
// ends, until StopAsking.
|
||||||
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
defer d.mu.Unlock()
|
defer d.mu.Unlock()
|
||||||
@@ -153,23 +177,47 @@ func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string {
|
|||||||
q := query{zone: zone, client: addr}
|
q := query{zone: zone, client: addr}
|
||||||
|
|
||||||
kept, found := d.verdicts.Peek(q)
|
kept, found := d.verdicts.Peek(q)
|
||||||
|
_, asking := d.asking[q]
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
|
case found && now.Sub(kept.Fetched) < d.params.CacheTTL:
|
||||||
if kept.Listed {
|
if kept.Listed {
|
||||||
listedBy = append(listedBy, config.MaskZoneKey(zone))
|
listedBy = append(listedBy, MaskZoneKey(zone))
|
||||||
}
|
}
|
||||||
case !d.asking[q] && !now.Before(d.retryAt[zone]) && len(d.asking) < maxQueries:
|
case !asking && !d.stopped && !now.Before(d.retryAt[zone]) &&
|
||||||
d.asking[q] = true
|
len(d.asking) < maxQueries:
|
||||||
|
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||||
|
d.asking[q] = end
|
||||||
d.queries[zone]++
|
d.queries[zone]++
|
||||||
|
|
||||||
go d.ask(context.WithoutCancel(ctx), q)
|
d.queryGoroutines.Go(func() {
|
||||||
|
defer end()
|
||||||
|
|
||||||
|
d.ask(queryCtx, q)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return listedBy
|
return listedBy
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StopAsking ends the queries under way, and returns once their
|
||||||
|
// goroutines have. A query it ends gives no verdict and is no failure: it
|
||||||
|
// is neither counted, logged nor raised as an alert, and the zone is not
|
||||||
|
// left alone after it. After StopAsking, ListedBy asks no zone.
|
||||||
|
func (d *DNSBL) StopAsking() {
|
||||||
|
d.mu.Lock()
|
||||||
|
|
||||||
|
d.stopped = true
|
||||||
|
for _, end := range d.asking {
|
||||||
|
end()
|
||||||
|
}
|
||||||
|
|
||||||
|
d.mu.Unlock()
|
||||||
|
|
||||||
|
d.queryGoroutines.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
// Queries returns how many queries were made to zone.
|
// Queries returns how many queries were made to zone.
|
||||||
func (d *DNSBL) Queries(zone string) int {
|
func (d *DNSBL) Queries(zone string) int {
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
@@ -238,30 +286,32 @@ func (d *DNSBL) Load(verdicts []Verdict) {
|
|||||||
// query as no longer under way. A query that fails gives no verdict: it
|
// query as no longer under way. A query that fails gives no verdict: it
|
||||||
// is counted, logged and raised as a source_failure alert, which show the
|
// is counted, logged and raised as a source_failure alert, which show the
|
||||||
// zone with its key masked, and the zone is not asked again for
|
// zone with its key masked, and the zone is not asked again for
|
||||||
// failureDelay.
|
// failureDelay. A query ended as ctx ends, by StopAsking, is no failure.
|
||||||
func (d *DNSBL) ask(ctx context.Context, q query) {
|
func (d *DNSBL) ask(ctx context.Context, q query) {
|
||||||
listed, err := d.lookUp(ctx, q)
|
listed, err := d.lookUp(ctx, q)
|
||||||
now := d.params.Now()
|
now := d.params.Now()
|
||||||
|
isFailure := err != nil && ctx.Err() == nil
|
||||||
|
|
||||||
d.mu.Lock()
|
d.mu.Lock()
|
||||||
|
|
||||||
delete(d.asking, q)
|
delete(d.asking, q)
|
||||||
|
|
||||||
if err == nil {
|
switch {
|
||||||
|
case err == nil:
|
||||||
d.verdicts.Add(q, Verdict{
|
d.verdicts.Add(q, Verdict{
|
||||||
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
|
Zone: q.zone, Client: q.client, Listed: listed, Fetched: now,
|
||||||
})
|
})
|
||||||
} else {
|
case isFailure:
|
||||||
d.failures[q.zone]++
|
d.failures[q.zone]++
|
||||||
d.retryAt[q.zone] = now.Add(failureDelay)
|
d.retryAt[q.zone] = now.Add(failureDelay)
|
||||||
}
|
}
|
||||||
|
|
||||||
d.mu.Unlock()
|
d.mu.Unlock()
|
||||||
|
|
||||||
if err != nil {
|
if isFailure {
|
||||||
const failed = "asking a DNSBL zone failed"
|
const failed = "asking a DNSBL zone failed"
|
||||||
|
|
||||||
shown := config.MaskZoneKey(q.zone)
|
shown := MaskZoneKey(q.zone)
|
||||||
|
|
||||||
// Raised before it is logged, so that the alert is there once the
|
// Raised before it is logged, so that the alert is there once the
|
||||||
// log line is.
|
// log line is.
|
||||||
|
|||||||
@@ -210,6 +210,45 @@ func TestQueryNotAnsweredWithinTheTimeoutFails(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||||
|
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
queue := newQueue()
|
||||||
|
p := dnsblParams(zone)
|
||||||
|
p.Alerts = queue
|
||||||
|
resolver := &resolverStandIn{hanging: true}
|
||||||
|
dnsbl := newDNSBL(resolver, p)
|
||||||
|
|
||||||
|
wantZones(t, dnsbl, listed)
|
||||||
|
synctest.Wait()
|
||||||
|
|
||||||
|
// The query is ended, not waited out, and is no failure.
|
||||||
|
began := time.Now()
|
||||||
|
|
||||||
|
dnsbl.StopAsking()
|
||||||
|
|
||||||
|
if waited := time.Since(began); waited != 0 {
|
||||||
|
t.Errorf("StopAsking returned after %s, want at once", waited)
|
||||||
|
}
|
||||||
|
|
||||||
|
wantQueries(t, dnsbl, 1, 0)
|
||||||
|
|
||||||
|
if got := waiting(queue); len(got) != 0 {
|
||||||
|
t.Errorf("alerts waiting %+v, want none", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A client without a verdict is then asked about in no zone.
|
||||||
|
wantZones(t, dnsbl, unlisted)
|
||||||
|
synctest.Wait()
|
||||||
|
wantQueries(t, dnsbl, 1, 0)
|
||||||
|
wantAsked(t, resolver, listedName)
|
||||||
|
waitForTheResolver()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
//nolint:paralleltest // one at a time, as the comment at the top of this file says
|
||||||
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
|
func TestZoneThatFailsOrRefusesGivesNoVerdictAndIsLeftAloneForAMinute(
|
||||||
@@ -821,3 +860,28 @@ func wantAsked(t *testing.T, resolver *resolverStandIn, want ...string) {
|
|||||||
t.Errorf("asked about %v, want %v", got, want)
|
t.Errorf("asked about %v, want %v", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const (
|
||||||
|
dronebl = "dnsbl.dronebl.org"
|
||||||
|
spamhaus = "abcdefghijklmnopqrstuvwxyz.xbl.dq.spamhaus.net"
|
||||||
|
spamhausMasked = "********.xbl.dq.spamhaus.net"
|
||||||
|
)
|
||||||
|
|
||||||
|
for zone, want := range map[string]string{
|
||||||
|
spamhaus: spamhausMasked,
|
||||||
|
spamhaus + ".": spamhausMasked + ".",
|
||||||
|
"KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET",
|
||||||
|
dronebl: dronebl,
|
||||||
|
"dq.spamhaus.net": "dq.spamhaus.net",
|
||||||
|
spamhaus + ".example": spamhaus + ".example",
|
||||||
|
} {
|
||||||
|
if got := reputation.MaskZoneKey(zone); got != want {
|
||||||
|
t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,8 +7,10 @@
|
|||||||
// SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
|
// SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks
|
||||||
// clients with AbuseIPDB, and keeps their scores and the checks spent
|
// clients with AbuseIPDB, and keeps their scores and the checks spent
|
||||||
// today. The state package writes all of these to files and reads them
|
// today. The state package writes all of these to files and reads them
|
||||||
// from there, so that a restart keeps them too. Last, it fetches
|
// from there, so that a restart keeps them too. It fetches AbuseIPDB's
|
||||||
// AbuseIPDB's blacklist, which it does not keep.
|
// blacklist, which it does not keep. Last, it reads the netblocks, AS
|
||||||
|
// numbers and percentages that both the settings and the lists hold, and
|
||||||
|
// masks the keys of DNSBL zones.
|
||||||
package reputation
|
package reputation
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -20,12 +22,12 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"slices"
|
"slices"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -40,15 +42,28 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
errStatus = errors.New("the server answered")
|
errStatus = errors.New("the server answered")
|
||||||
errTooLong = errors.New("the list is longer than 16 MiB")
|
errTooLong = errors.New("the list is longer than 16 MiB")
|
||||||
errNotNetblock = errors.New(
|
errNotNetblockLine = errors.New(
|
||||||
"is not an address or a netblock, such as 192.0.2.0/24",
|
"is not an address or a netblock, such as 192.0.2.0/24",
|
||||||
)
|
)
|
||||||
errNotASNPercent = errors.New(
|
errNotASNPercent = errors.New(
|
||||||
"is not an AS number, : and a percentage, such as AS64496:50")
|
"is not an AS number, : and a percentage, such as AS64496:50")
|
||||||
|
errNotNetblock = errors.New(
|
||||||
|
"is not a netblock such as 10.0.0.0/8, or an address")
|
||||||
|
errNotASN = errors.New("is not an AS number such as AS64496")
|
||||||
|
errNotPercent = errors.New(
|
||||||
|
"is not a percentage, a whole number from 0 to 100",
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// TagList is an address list of SWW_WAF_TAG_LISTS: the tag it adds to the
|
||||||
|
// requests of the clients it holds, and the URL it is fetched from.
|
||||||
|
type TagList struct {
|
||||||
|
Name string
|
||||||
|
URL string
|
||||||
|
}
|
||||||
|
|
||||||
// List is a list as lists.json holds it: the URL it is fetched from,
|
// List is a list as lists.json holds it: the URL it is fetched from,
|
||||||
// when it was last tried, the fetch failed or not, and its last good copy:
|
// when it was last tried, the fetch failed or not, and its last good copy:
|
||||||
// when that was fetched, and its lines, as fetched, comment lines
|
// when that was fetched, and its lines, as fetched, comment lines
|
||||||
@@ -68,7 +83,7 @@ type Params struct {
|
|||||||
BlocklistURLs []string
|
BlocklistURLs []string
|
||||||
ASNLimitPercentURL string
|
ASNLimitPercentURL string
|
||||||
// TagLists are the address lists that add a tag (SWW_WAF_TAG_LISTS).
|
// TagLists are the address lists that add a tag (SWW_WAF_TAG_LISTS).
|
||||||
TagLists []config.TagList
|
TagLists []TagList
|
||||||
// Refresh is how long after a list was last fetched or tried it is
|
// Refresh is how long after a list was last fetched or tried it is
|
||||||
// fetched again (SWW_WAF_BLOCKLIST_REFRESH).
|
// fetched again (SWW_WAF_BLOCKLIST_REFRESH).
|
||||||
Refresh time.Duration
|
Refresh time.Duration
|
||||||
@@ -465,7 +480,11 @@ func parseNetblocks(lines []string) (entries, error) {
|
|||||||
|
|
||||||
netblock, ok := parseNetblock(text)
|
netblock, ok := parseNetblock(text)
|
||||||
if !ok {
|
if !ok {
|
||||||
return entries{}, fmt.Errorf("line %d %w", i+1, errNotNetblock)
|
return entries{}, fmt.Errorf(
|
||||||
|
"line %d %w",
|
||||||
|
i+1,
|
||||||
|
errNotNetblockLine,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
found.netblocks[netblock] = true
|
found.netblocks[netblock] = true
|
||||||
@@ -485,7 +504,7 @@ func parseNetblocks(lines []string) (entries, error) {
|
|||||||
// stands for, 192.0.2.0/24, and a mapped netblock shorter than /96, which
|
// stands for, 192.0.2.0/24, and a mapped netblock shorter than /96, which
|
||||||
// stands for none, is refused.
|
// stands for none, is refused.
|
||||||
func parseNetblock(text string) (netip.Prefix, bool) {
|
func parseNetblock(text string) (netip.Prefix, bool) {
|
||||||
netblock, err := config.ParseNetblock(text)
|
netblock, err := ParseNetblock(text)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return netip.Prefix{}, false
|
return netip.Prefix{}, false
|
||||||
}
|
}
|
||||||
@@ -520,9 +539,9 @@ func parsePercents(lines []string) (entries, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
asnText, percentText, _ := strings.Cut(text, ":")
|
asnText, percentText, _ := strings.Cut(text, ":")
|
||||||
asn, asnErr := config.ParseASN(asnText)
|
asn, asnErr := ParseASN(asnText)
|
||||||
|
|
||||||
percent, percentErr := config.ParsePercent(percentText)
|
percent, percentErr := ParsePercent(percentText)
|
||||||
if asnErr != nil || percentErr != nil {
|
if asnErr != nil || percentErr != nil {
|
||||||
return entries{}, fmt.Errorf("line %d %w", i+1, errNotASNPercent)
|
return entries{}, fmt.Errorf("line %d %w", i+1, errNotASNPercent)
|
||||||
}
|
}
|
||||||
@@ -536,6 +555,53 @@ func parsePercents(lines []string) (entries, error) {
|
|||||||
return found, nil
|
return found, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare
|
||||||
|
// address is a netblock of that address alone, a /32 or a /128. A
|
||||||
|
// blocklist's lines are read with it too.
|
||||||
|
func ParseNetblock(value string) (netip.Prefix, error) {
|
||||||
|
if strings.Contains(value, "/") {
|
||||||
|
netblock, err := netip.ParsePrefix(value)
|
||||||
|
if err != nil {
|
||||||
|
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
||||||
|
}
|
||||||
|
|
||||||
|
return netblock.Masked(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
addr, err := netip.ParseAddr(value)
|
||||||
|
if err != nil || addr.Zone() != "" {
|
||||||
|
return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock)
|
||||||
|
}
|
||||||
|
|
||||||
|
return netip.PrefixFrom(addr, addr.BitLen()), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseASN reads an AS number such as AS64496, in either case, and
|
||||||
|
// returns it as the lookup gives it: AS and the number, in capitals and
|
||||||
|
// without leading zeros. The file SWW_WAF_ASN_LIMIT_PERCENT_URL names is
|
||||||
|
// read with it too.
|
||||||
|
func ParseASN(value string) (string, error) {
|
||||||
|
digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS")
|
||||||
|
|
||||||
|
number, err := strconv.ParseUint(digits, 10, 32)
|
||||||
|
if !hasAS || err != nil {
|
||||||
|
return "", fmt.Errorf("%q %w", value, errNotASN)
|
||||||
|
}
|
||||||
|
|
||||||
|
return "AS" + strconv.FormatUint(number, 10), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParsePercent reads a percentage, a whole number from 0 to 100. The file
|
||||||
|
// SWW_WAF_ASN_LIMIT_PERCENT_URL names is read with it too.
|
||||||
|
func ParsePercent(value string) (int64, error) {
|
||||||
|
percent, err := strconv.ParseInt(value, 10, 64)
|
||||||
|
if err != nil || percent < 0 || percent > 100 {
|
||||||
|
return 0, fmt.Errorf("%q %w", value, errNotPercent)
|
||||||
|
}
|
||||||
|
|
||||||
|
return percent, nil
|
||||||
|
}
|
||||||
|
|
||||||
// withoutComment returns line without anything after a ; or a #, and
|
// withoutComment returns line without anything after a ; or a #, and
|
||||||
// without the spaces around what is left.
|
// without the spaces around what is left.
|
||||||
func withoutComment(line string) string {
|
func withoutComment(line string) string {
|
||||||
|
|||||||
@@ -99,8 +99,12 @@ type Client struct {
|
|||||||
// answers are by client. Each is added as it comes and never moved up,
|
// answers are by client. Each is added as it comes and never moved up,
|
||||||
// so that the one that came longest ago is the first dropped.
|
// so that the one that came longest ago is the first dropped.
|
||||||
answers *simplelru.LRU[netip.Prefix, Answer]
|
answers *simplelru.LRU[netip.Prefix, Answer]
|
||||||
// asking are the clients whose query is under way.
|
// asking are the clients whose query is under way, each with the
|
||||||
asking map[netip.Prefix]bool
|
// function that ends it. stopped is true once StopAsking has been
|
||||||
|
// called, and queryGoroutines waits for the goroutine of each query.
|
||||||
|
asking map[netip.Prefix]context.CancelFunc
|
||||||
|
stopped bool
|
||||||
|
queryGoroutines sync.WaitGroup
|
||||||
// retryAt is when routewatch may be asked again after a query failed.
|
// retryAt is when routewatch may be asked again after a query failed.
|
||||||
retryAt time.Time
|
retryAt time.Time
|
||||||
}
|
}
|
||||||
@@ -116,7 +120,7 @@ func New(params Params) *Client {
|
|||||||
params: params,
|
params: params,
|
||||||
httpClient: &http.Client{},
|
httpClient: &http.Client{},
|
||||||
answers: answers,
|
answers: answers,
|
||||||
asking: map[netip.Prefix]bool{},
|
asking: map[netip.Prefix]context.CancelFunc{},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -124,8 +128,9 @@ func New(params Params) *Client {
|
|||||||
// IPv6 group, or why it has no route. It never waits for a query. A client
|
// IPv6 group, or why it has no route. It never waits for a query. A client
|
||||||
// without an answer is asked about in the background, by addr, the address its
|
// without an answer is asked about in the background, by addr, the address its
|
||||||
// request came from, unless its query is under way, a query failed less than
|
// request came from, unless its query is under way, a query failed less than
|
||||||
// failureKept ago, or maxQueries are under way. ctx is the context of the
|
// failureKept ago, maxQueries are under way, or StopAsking has been called. ctx
|
||||||
// client's request, and a query goes on after the request ends.
|
// is the context of the client's request, and a query goes on after the request
|
||||||
|
// ends, until StopAsking.
|
||||||
func (c *Client) Route(
|
func (c *Client) Route(
|
||||||
ctx context.Context, client netip.Prefix, addr netip.Addr,
|
ctx context.Context, client netip.Prefix, addr netip.Addr,
|
||||||
) (Route, Missing) {
|
) (Route, Missing) {
|
||||||
@@ -147,15 +152,38 @@ func (c *Client) Route(
|
|||||||
return Route{}, RequestFailed
|
return Route{}, RequestFailed
|
||||||
}
|
}
|
||||||
|
|
||||||
if !c.asking[client] && len(c.asking) < maxQueries {
|
_, asking := c.asking[client]
|
||||||
c.asking[client] = true
|
if !asking && !c.stopped && len(c.asking) < maxQueries {
|
||||||
|
queryCtx, end := context.WithCancel(context.WithoutCancel(ctx))
|
||||||
|
c.asking[client] = end
|
||||||
|
|
||||||
go c.ask(context.WithoutCancel(ctx), client, addr)
|
c.queryGoroutines.Go(func() {
|
||||||
|
defer end()
|
||||||
|
|
||||||
|
c.ask(queryCtx, client, addr)
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
return Route{}, NoAnswerYet
|
return Route{}, NoAnswerYet
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StopAsking ends the queries under way, and returns once their goroutines
|
||||||
|
// have. A query it ends gives no answer and is no failure: it is not logged,
|
||||||
|
// and routewatch is not left alone after it. After StopAsking, Route asks
|
||||||
|
// routewatch nothing.
|
||||||
|
func (c *Client) StopAsking() {
|
||||||
|
c.mu.Lock()
|
||||||
|
|
||||||
|
c.stopped = true
|
||||||
|
for _, end := range c.asking {
|
||||||
|
end()
|
||||||
|
}
|
||||||
|
|
||||||
|
c.mu.Unlock()
|
||||||
|
|
||||||
|
c.queryGoroutines.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
// Snapshot returns every answer still in use, sorted by client, as
|
// Snapshot returns every answer still in use, sorted by client, as
|
||||||
// routewatch.json lists them.
|
// routewatch.json lists them.
|
||||||
func (c *Client) Snapshot() []Answer {
|
func (c *Client) Snapshot() []Answer {
|
||||||
@@ -205,7 +233,8 @@ func (c *Client) Load(answers []Answer) {
|
|||||||
// ask asks routewatch about addr, keeps the answer as client's, and notes the
|
// ask asks routewatch about addr, keeps the answer as client's, and notes the
|
||||||
// query as no longer under way. A query that fails gives no answer, and
|
// query as no longer under way. A query that fails gives no answer, and
|
||||||
// routewatch is not asked again for failureKept. Only the first failure of
|
// routewatch is not asked again for failureKept. Only the first failure of
|
||||||
// those under way is logged.
|
// those under way is logged. A query ended as ctx ends, by StopAsking, is no
|
||||||
|
// failure.
|
||||||
func (c *Client) ask(
|
func (c *Client) ask(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
client netip.Prefix,
|
client netip.Prefix,
|
||||||
@@ -213,18 +242,20 @@ func (c *Client) ask(
|
|||||||
) {
|
) {
|
||||||
route, err := c.query(ctx, addr)
|
route, err := c.query(ctx, addr)
|
||||||
now := c.params.Now()
|
now := c.params.Now()
|
||||||
|
isFailure := err != nil && ctx.Err() == nil
|
||||||
|
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
|
|
||||||
delete(c.asking, client)
|
delete(c.asking, client)
|
||||||
|
|
||||||
firstFailure := err != nil && !now.Before(c.retryAt)
|
firstFailure := isFailure && !now.Before(c.retryAt)
|
||||||
|
|
||||||
if err == nil {
|
switch {
|
||||||
|
case err == nil:
|
||||||
c.answers.Add(client, Answer{
|
c.answers.Add(client, Answer{
|
||||||
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
|
Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
|
||||||
})
|
})
|
||||||
} else {
|
case isFailure:
|
||||||
c.retryAt = now.Add(failureKept)
|
c.retryAt = now.Add(failureKept)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -361,6 +361,54 @@ func TestHangingRoutewatchNeverDelaysAndFailsTheQuery(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStopAskingEndsTheQueryUnderWayAsNoFailureAndAsksNoMore(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// In a synctest bubble, as TestHangingRoutewatchNeverDelaysAndFailsTheQuery
|
||||||
|
// says.
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
f := &fixture{t: t, log: &syncBuffer{}}
|
||||||
|
f.client = routewatch.New(routewatch.Params{
|
||||||
|
URL: silentURL, Now: time.Now,
|
||||||
|
ProcessLog: slog.New(slog.NewTextHandler(f.log, nil)),
|
||||||
|
})
|
||||||
|
|
||||||
|
f.route()
|
||||||
|
synctest.Wait()
|
||||||
|
|
||||||
|
// The query is ended, not waited out.
|
||||||
|
began := time.Now()
|
||||||
|
|
||||||
|
f.client.StopAsking()
|
||||||
|
|
||||||
|
if waited := time.Since(began); waited != 0 {
|
||||||
|
t.Errorf("StopAsking returned after %s, want at once", waited)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A query started now would fail within waitLimit. None is, and the
|
||||||
|
// one ended is no failure: nothing is logged, and routewatch is not
|
||||||
|
// left alone.
|
||||||
|
for range 2 {
|
||||||
|
if _, missing := f.route(); missing != routewatch.NoAnswerYet {
|
||||||
|
t.Errorf(
|
||||||
|
"Route gave %q, want %q",
|
||||||
|
missing,
|
||||||
|
routewatch.NoAnswerYet,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
time.Sleep(waitLimit)
|
||||||
|
synctest.Wait()
|
||||||
|
}
|
||||||
|
|
||||||
|
if f.log.String() != "" {
|
||||||
|
t.Errorf("process log %q, want nothing", f.log.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
|
func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,12 @@ package rproxy
|
|||||||
|
|
||||||
import "time"
|
import "time"
|
||||||
|
|
||||||
|
// StopBreakerClock has the circuit breaker of s read at as the time, so that how
|
||||||
|
// long requests take cannot matter to it. It is called before s serves.
|
||||||
|
func (s *Server) StopBreakerClock(at time.Time) {
|
||||||
|
s.h.breakerClock = func() time.Time { return at }
|
||||||
|
}
|
||||||
|
|
||||||
// Breaker is the circuit breaker, for tests that give it the time.
|
// Breaker is the circuit breaker, for tests that give it the time.
|
||||||
type Breaker struct {
|
type Breaker struct {
|
||||||
b breaker
|
b breaker
|
||||||
|
|||||||
@@ -870,10 +870,14 @@ func TestAnAnswerRProxyGivesItselfHasTenSecondsToBeSentAtLeast(
|
|||||||
func TestAVisitorTooSlowToSendTheRestIsAnswered408(t *testing.T) {
|
func TestAVisitorTooSlowToSendTheRestIsAnswered408(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
// rproxy is waiting for the rest of the body well within this limit, even on
|
||||||
|
// a busy host, and answers when it runs out, well before dial's deadline.
|
||||||
|
const limit = 5 * time.Second
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, greet)
|
app := startApp(t, greet)
|
||||||
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||||
maxRequestTime: "3000",
|
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
||||||
})
|
})
|
||||||
|
|
||||||
conn := dial(t, addr)
|
conn := dial(t, addr)
|
||||||
@@ -1262,7 +1266,9 @@ func TestAVisitorThatStopsALittleShortIsReadUntilTheRequestTimeLimit(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const limit = 2 * time.Second
|
// The app's answer arrives well within this limit, even on a busy host, and
|
||||||
|
// the connection closes when it runs out, well before dial's deadline.
|
||||||
|
const limit = 5 * time.Second
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, answerBeforeTheBody)
|
app := startApp(t, answerBeforeTheBody)
|
||||||
@@ -1427,17 +1433,20 @@ func TestTheRequestTimeLimitDoesNotCutAnAnswerOnceTheAppHasTheRequest(
|
|||||||
|
|
||||||
const answer = "the end of a long answer"
|
const answer = "the end of a long answer"
|
||||||
|
|
||||||
|
// The request reaches the app well within this limit, even on a busy host.
|
||||||
|
const limit = 5 * time.Second
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
app := startApp(t, func(w http.ResponseWriter, r *http.Request) {
|
||||||
_, _ = io.Copy(io.Discard, r.Body)
|
_, _ = io.Copy(io.Discard, r.Body)
|
||||||
_ = http.NewResponseController(w).Flush()
|
_ = http.NewResponseController(w).Flush()
|
||||||
|
|
||||||
time.Sleep(2100 * time.Millisecond)
|
time.Sleep(limit + 100*time.Millisecond)
|
||||||
|
|
||||||
_, _ = io.WriteString(w, answer)
|
_, _ = io.WriteString(w, answer)
|
||||||
})
|
})
|
||||||
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
addr, _ := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||||
maxRequestTime: "2000",
|
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
||||||
})
|
})
|
||||||
|
|
||||||
got := visit(
|
got := visit(
|
||||||
|
|||||||
@@ -94,6 +94,7 @@ func New(params Params) *Server {
|
|||||||
reportURL: (&url.URL{
|
reportURL: (&url.URL{
|
||||||
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
|
Scheme: backend.Scheme, Host: backend.Host, Path: waf.ReportPath,
|
||||||
}).String(),
|
}).String(),
|
||||||
|
breakerClock: time.Now,
|
||||||
}
|
}
|
||||||
|
|
||||||
return &Server{
|
return &Server{
|
||||||
@@ -133,8 +134,10 @@ type handler struct {
|
|||||||
wafTransport http.RoundTripper
|
wafTransport http.RoundTripper
|
||||||
// reportURL is where the reports go.
|
// reportURL is where the reports go.
|
||||||
reportURL string
|
reportURL string
|
||||||
// breaker is the circuit breaker for asks to waf.
|
// breaker is the circuit breaker for asks to waf, and breakerClock gives it
|
||||||
breaker breaker
|
// the time.
|
||||||
|
breaker breaker
|
||||||
|
breakerClock func() time.Time
|
||||||
// reports waits for the reports being sent. reportsMu guards starting one
|
// reports waits for the reports being sent. reportsMu guards starting one
|
||||||
// against reportsStopped, which is set once nothing is to wait for another.
|
// against reportsStopped, which is set once nothing is to wait for another.
|
||||||
reports sync.WaitGroup
|
reports sync.WaitGroup
|
||||||
|
|||||||
@@ -356,6 +356,19 @@ func startServerLogging(
|
|||||||
) (string, *rproxy.Server) {
|
) (string, *rproxy.Server) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
server := newServer(t, backend, appURL, env, requestLog, processLog)
|
||||||
|
|
||||||
|
return serve(t, server), server
|
||||||
|
}
|
||||||
|
|
||||||
|
// newServer returns rproxy, not yet serving, with its request lines going to
|
||||||
|
// requestLog and its process lines to processLog.
|
||||||
|
func newServer(
|
||||||
|
t *testing.T, backend, appURL string, env map[string]string,
|
||||||
|
requestLog, processLog io.Writer,
|
||||||
|
) *rproxy.Server {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
settings := map[string]string{
|
settings := map[string]string{
|
||||||
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
|
"SWW_RPROXY_WAF_BACKEND": backend, "SWW_RPROXY_UPSTREAM_URL": appURL,
|
||||||
wafBackendTimeout: "60000",
|
wafBackendTimeout: "60000",
|
||||||
@@ -367,7 +380,7 @@ func startServerLogging(
|
|||||||
t.Fatalf("settings %v: %v", settings, err)
|
t.Fatalf("settings %v: %v", settings, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
server := rproxy.New(rproxy.Params{
|
return rproxy.New(rproxy.Params{
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
RequestLog: requestLog,
|
RequestLog: requestLog,
|
||||||
ProcessLog: requestlog.NewProcessLogger(
|
ProcessLog: requestlog.NewProcessLogger(
|
||||||
@@ -376,6 +389,11 @@ func startServerLogging(
|
|||||||
slog.LevelInfo,
|
slog.LevelInfo,
|
||||||
),
|
),
|
||||||
})
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// serve has server serve until the test ends, and returns its address.
|
||||||
|
func serve(t *testing.T, server *rproxy.Server) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
listener, err := (&net.ListenConfig{}).Listen(
|
listener, err := (&net.ListenConfig{}).Listen(
|
||||||
t.Context(),
|
t.Context(),
|
||||||
@@ -396,7 +414,7 @@ func startServerLogging(
|
|||||||
server.WaitForReports()
|
server.WaitForReports()
|
||||||
})
|
})
|
||||||
|
|
||||||
return listener.Addr().String(), server
|
return listener.Addr().String()
|
||||||
}
|
}
|
||||||
|
|
||||||
// answered is the visitor's answer.
|
// answered is the visitor's answer.
|
||||||
@@ -1434,14 +1452,18 @@ func TestFiveFailedAsksKeepTheNextRequestFromAskingWAF(t *testing.T) {
|
|||||||
|
|
||||||
greet(w, r)
|
greet(w, r)
|
||||||
})
|
})
|
||||||
addr, out := startRProxy(
|
out := &output{}
|
||||||
t,
|
server := newServer(
|
||||||
standIn.URL,
|
t, standIn.URL, app.URL,
|
||||||
app.URL,
|
map[string]string{wafBackendTimeout: "200"},
|
||||||
map[string]string{
|
out, out,
|
||||||
wafBackendTimeout: "200",
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
|
// The breaker's clock stands still, as in
|
||||||
|
// TestAnswersThatCannotBePassedOnOpenTheBreaker: on a slow host, the five
|
||||||
|
// failures could take more than five seconds, and the next request could
|
||||||
|
// come more than a second later.
|
||||||
|
server.StopBreakerClock(time.Now())
|
||||||
|
addr := serve(t, server)
|
||||||
|
|
||||||
// An ask can time out before it reaches the stand-in, so these
|
// An ask can time out before it reaches the stand-in, so these
|
||||||
// may leave fewer than five questions, or leave some late.
|
// may leave fewer than five questions, or leave some late.
|
||||||
@@ -1532,9 +1554,13 @@ func TestAnswersThatCannotBePassedOnOpenTheBreaker(t *testing.T) {
|
|||||||
tooLarge(w, r)
|
tooLarge(w, r)
|
||||||
})
|
})
|
||||||
app := startApp(t, greet)
|
app := startApp(t, greet)
|
||||||
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
out := &output{}
|
||||||
wafBackendTimeout: "60000",
|
server := newServer(t, standIn.URL, app.URL, nil, out, out)
|
||||||
})
|
// The breaker's clock stands still. On a slow host, the five failures could
|
||||||
|
// otherwise take more than the five seconds that open the breaker. The next
|
||||||
|
// request could come more than a second later, when it may ask waf again.
|
||||||
|
server.StopBreakerClock(time.Now())
|
||||||
|
addr := serve(t, server)
|
||||||
|
|
||||||
for range 5 {
|
for range 5 {
|
||||||
if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
|
if got := get(t, addr, "/"); got.status != http.StatusBadGateway {
|
||||||
@@ -1809,11 +1835,15 @@ func TestWaitForReportsWaitsForAReportBeingSent(t *testing.T) {
|
|||||||
func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
|
func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
// The app's answer to the upgrade arrives well within these limits, even on a
|
||||||
|
// busy host.
|
||||||
|
const limit = 5 * time.Second
|
||||||
|
|
||||||
standIn := startWAF(t, allow)
|
standIn := startWAF(t, allow)
|
||||||
app := startApp(t, echoAfterUpgrade)
|
app := startApp(t, echoAfterUpgrade)
|
||||||
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
addr, out := startRProxy(t, standIn.URL, app.URL, map[string]string{
|
||||||
maxRequestTime: "2000",
|
maxRequestTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
||||||
maxResponseTime: "2000",
|
maxResponseTime: strconv.FormatInt(limit.Milliseconds(), 10),
|
||||||
})
|
})
|
||||||
|
|
||||||
conn := dial(t, addr)
|
conn := dial(t, addr)
|
||||||
@@ -1838,7 +1868,10 @@ func TestReportOfAWebSocketCountsWhatItCarried(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Past both time limits, the connection still carries what is sent.
|
// Past both time limits, the connection still carries what is sent.
|
||||||
time.Sleep(2100 * time.Millisecond)
|
time.Sleep(limit + 100*time.Millisecond)
|
||||||
|
|
||||||
|
// dial's deadline ran from before the upgrade and this wait.
|
||||||
|
_ = conn.SetDeadline(time.Now().Add(waitLimit))
|
||||||
|
|
||||||
const message = "still here\n"
|
const message = "still here\n"
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ import (
|
|||||||
"maps"
|
"maps"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/waf"
|
"sneak.berlin/go/smallwebwaf/internal/waf"
|
||||||
)
|
)
|
||||||
@@ -97,7 +96,7 @@ func (rq *request) askUnlessBreakerOpen(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
) (*wafAnswer, error) {
|
) (*wafAnswer, error) {
|
||||||
b := &rq.h.breaker
|
b := &rq.h.breaker
|
||||||
if !b.mayAsk(time.Now()) {
|
if !b.mayAsk(rq.h.breakerClock()) {
|
||||||
return nil, errBreakerOpen
|
return nil, errBreakerOpen
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -107,7 +106,7 @@ func (rq *request) askUnlessBreakerOpen(
|
|||||||
case err == nil:
|
case err == nil:
|
||||||
b.noteAnswer()
|
b.noteAnswer()
|
||||||
case ctx.Err() == nil:
|
case ctx.Err() == nil:
|
||||||
b.noteFailure(time.Now())
|
b.noteFailure(rq.h.breakerClock())
|
||||||
}
|
}
|
||||||
|
|
||||||
return answer, err
|
return answer, err
|
||||||
|
|||||||
@@ -21,9 +21,9 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/fsnotify/fsnotify"
|
"github.com/fsnotify/fsnotify"
|
||||||
|
"golang.org/x/net/http/httpguts"
|
||||||
|
|
||||||
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
"sneak.berlin/go/smallwebwaf/internal/alerts"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/config"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// The actions a rule takes when it matches.
|
// The actions a rule takes when it matches.
|
||||||
@@ -378,7 +378,7 @@ func parse(line string) (Rule, bool, error) {
|
|||||||
rule.Target,
|
rule.Target,
|
||||||
errNotTarget,
|
errNotTarget,
|
||||||
)
|
)
|
||||||
case isHeader && !config.IsHeaderName(headerName):
|
case isHeader && !httpguts.ValidHeaderFieldName(headerName):
|
||||||
return Rule{}, false, fmt.Errorf(
|
return Rule{}, false, fmt.Errorf(
|
||||||
"the target %q %w",
|
"the target %q %w",
|
||||||
rule.Target,
|
rule.Target,
|
||||||
|
|||||||
@@ -755,8 +755,7 @@ func TestRProxyStoppingWaitsForAReportBeingSent(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection. A
|
// sendAndKeepOpen sends request to rproxy at addr, and returns the connection.
|
||||||
// read from it ends with an error once waitLimit is up.
|
|
||||||
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -769,11 +768,6 @@ func sendAndKeepOpen(t *testing.T, addr, request string) net.Conn {
|
|||||||
_ = conn.Close()
|
_ = conn.Close()
|
||||||
})
|
})
|
||||||
|
|
||||||
err = conn.SetDeadline(time.Now().Add(waitLimit))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("set the deadline: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = io.WriteString(conn, request)
|
_, err = io.WriteString(conn, request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("send: %v", err)
|
t.Fatalf("send: %v", err)
|
||||||
@@ -838,6 +832,9 @@ func TestRProxyStoppingCutsOffARequestAfterFiveSeconds(t *testing.T) {
|
|||||||
|
|
||||||
stop()
|
stop()
|
||||||
|
|
||||||
|
// Counted from the stop, so a slow wait for the app cannot shorten it.
|
||||||
|
_ = conn.SetReadDeadline(stopped.Add(cutOffAfter + waitLimit))
|
||||||
|
|
||||||
text, err := io.ReadAll(conn)
|
text, err := io.ReadAll(conn)
|
||||||
if err != nil || len(text) != 0 {
|
if err != nil || len(text) != 0 {
|
||||||
t.Errorf("read %q (%v), want the connection cut off", text, err)
|
t.Errorf("read %q (%v), want the connection cut off", text, err)
|
||||||
|
|||||||
+36
-12
@@ -118,7 +118,8 @@ func RunWAF(ctx context.Context, params Params) int {
|
|||||||
"settings", cfg)
|
"settings", cfg)
|
||||||
|
|
||||||
// The last write follows the stop of every hook, and so the end of every
|
// The last write follows the stop of every hook, and so the end of every
|
||||||
// loop that changes the state or raises an alert. A hook of the state
|
// loop that changes the state or raises an alert, and of every ask of
|
||||||
|
// GeoJS, the DNSBL zones, AbuseIPDB and routewatch. A hook of the state
|
||||||
// files would stop before the loops of the parts built ahead of them.
|
// files would stop before the loops of the parts built ahead of them.
|
||||||
return run(ctx, app, failed, processLog, func() error {
|
return run(ctx, app, failed, processLog, func() error {
|
||||||
err := files.WriteAll()
|
err := files.WriteAll()
|
||||||
@@ -297,11 +298,13 @@ func newLists(
|
|||||||
return lists
|
return lists
|
||||||
}
|
}
|
||||||
|
|
||||||
// newDNSBL returns what asks the DNSBL zones, none asked yet.
|
// newDNSBL returns what asks the DNSBL zones, none asked yet. At the stop it
|
||||||
|
// ends the queries under way, as reputation.DNSBL.StopAsking says.
|
||||||
func newDNSBL(
|
func newDNSBL(
|
||||||
cfg *config.WAF, processLog *slog.Logger, alertQueue *alerts.Queue,
|
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||||
|
alertQueue *alerts.Queue,
|
||||||
) *reputation.DNSBL {
|
) *reputation.DNSBL {
|
||||||
return reputation.NewDNSBL(reputation.DNSBLParams{
|
dnsbl := reputation.NewDNSBL(reputation.DNSBLParams{
|
||||||
Zones: cfg.DNSBLZones,
|
Zones: cfg.DNSBLZones,
|
||||||
Resolver: cfg.DNSBLResolver,
|
Resolver: cfg.DNSBLResolver,
|
||||||
CacheTTL: cfg.IPReputationCacheTTL,
|
CacheTTL: cfg.IPReputationCacheTTL,
|
||||||
@@ -310,12 +313,17 @@ func newDNSBL(
|
|||||||
ProcessLog: processLog,
|
ProcessLog: processLog,
|
||||||
Alerts: alertQueue,
|
Alerts: alertQueue,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
lc.Append(fx.StopHook(dnsbl.StopAsking))
|
||||||
|
|
||||||
|
return dnsbl
|
||||||
}
|
}
|
||||||
|
|
||||||
// newAbuseIPDB returns what checks clients with AbuseIPDB while
|
// newAbuseIPDB returns what checks clients with AbuseIPDB while
|
||||||
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it
|
// SWW_WAF_ABUSEIPDB_KEY is set, none checked yet. Until waf stops, it
|
||||||
// fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in
|
// fetches AbuseIPDB's blacklist as it is due, and bans the clients on it in
|
||||||
// ledger, as waf.BanAbuseIPDBBlacklist says.
|
// ledger, as waf.BanAbuseIPDBBlacklist says. At the stop it ends the checks
|
||||||
|
// under way, as reputation.AbuseIPDB.StopChecking says.
|
||||||
func newAbuseIPDB(
|
func newAbuseIPDB(
|
||||||
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||||
alertQueue *alerts.Queue, ledger *bans.Ledger,
|
alertQueue *alerts.Queue, ledger *bans.Ledger,
|
||||||
@@ -341,24 +349,30 @@ func newAbuseIPDB(
|
|||||||
})
|
})
|
||||||
|
|
||||||
lc.Append(background(abuseIPDB.RunBlacklist))
|
lc.Append(background(abuseIPDB.RunBlacklist))
|
||||||
|
lc.Append(fx.StopHook(abuseIPDB.StopChecking))
|
||||||
|
|
||||||
return abuseIPDB
|
return abuseIPDB
|
||||||
}
|
}
|
||||||
|
|
||||||
// newRouteWatch returns what asks routewatch about clients, or nil while
|
// newRouteWatch returns what asks routewatch about clients, or nil while
|
||||||
// SWW_WAF_ROUTEWATCH_URL is unset.
|
// SWW_WAF_ROUTEWATCH_URL is unset. At the stop it ends the queries under
|
||||||
|
// way, as routewatch.Client.StopAsking says.
|
||||||
func newRouteWatch(
|
func newRouteWatch(
|
||||||
cfg *config.WAF, processLog *slog.Logger,
|
lc fx.Lifecycle, cfg *config.WAF, processLog *slog.Logger,
|
||||||
) *routewatch.Client {
|
) *routewatch.Client {
|
||||||
if cfg.RouteWatchURL == "" {
|
if cfg.RouteWatchURL == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return routewatch.New(routewatch.Params{
|
routeWatch := routewatch.New(routewatch.Params{
|
||||||
URL: cfg.RouteWatchURL,
|
URL: cfg.RouteWatchURL,
|
||||||
Now: utcNow,
|
Now: utcNow,
|
||||||
ProcessLog: processLog,
|
ProcessLog: processLog,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
lc.Append(fx.StopHook(routeWatch.StopAsking))
|
||||||
|
|
||||||
|
return routeWatch
|
||||||
}
|
}
|
||||||
|
|
||||||
// newMetrics returns the metrics, with those read from the other parts as
|
// newMetrics returns the metrics, with those read from the other parts as
|
||||||
@@ -396,12 +410,18 @@ func newMetrics(
|
|||||||
|
|
||||||
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
// newGeoJS returns what looks clients up at GeoJS, none looked up yet. Each
|
||||||
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each
|
// answer goes to the limiter and the ledger, as waf.AddLookup says, and each
|
||||||
// failure is handled as waf.GeoJSFailed says.
|
// failure is handled as waf.GeoJSFailed says. At the stop it ends the request
|
||||||
|
// under way, as geojs.GeoJS.StopAsking says.
|
||||||
func newGeoJS(
|
func newGeoJS(
|
||||||
cfg *config.WAF, m *metrics.Metrics, limiter *ratelimit.Limiter,
|
lc fx.Lifecycle,
|
||||||
ledger *bans.Ledger, alertQueue *alerts.Queue, processLog *slog.Logger,
|
cfg *config.WAF,
|
||||||
|
m *metrics.Metrics,
|
||||||
|
limiter *ratelimit.Limiter,
|
||||||
|
ledger *bans.Ledger,
|
||||||
|
alertQueue *alerts.Queue,
|
||||||
|
processLog *slog.Logger,
|
||||||
) *geojs.GeoJS {
|
) *geojs.GeoJS {
|
||||||
return geojs.New(geojs.Params{
|
geoJS := geojs.New(geojs.Params{
|
||||||
URL: geojs.URL,
|
URL: geojs.URL,
|
||||||
Answered: func(answer geojs.Answer) {
|
Answered: func(answer geojs.Answer) {
|
||||||
waf.AddLookup(cfg, limiter, ledger, answer)
|
waf.AddLookup(cfg, limiter, ledger, answer)
|
||||||
@@ -413,6 +433,10 @@ func newGeoJS(
|
|||||||
},
|
},
|
||||||
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
WentWithoutAnswer: m.RequestsWithoutGeoJSAnswer.Inc,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
lc.Append(fx.StopHook(geoJS.StopAsking))
|
||||||
|
|
||||||
|
return geoJS
|
||||||
}
|
}
|
||||||
|
|
||||||
// newWAFServer returns the server waf runs, with its parts and the time
|
// newWAFServer returns the server waf runs, with its parts and the time
|
||||||
|
|||||||
@@ -53,13 +53,13 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
if country == "" && !rq.skippedLookupWait {
|
if country == "" && !rq.skippedLookupWait {
|
||||||
unknown = percentage{
|
unknown = percentage{
|
||||||
cfg.UnknownCountryLimitPercent,
|
cfg.UnknownCountryLimitPercent,
|
||||||
"SWW_WAF_UNKNOWN_COUNTRY_LIMIT_PERCENT",
|
config.UnknownCountryLimitPercentSetting,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fetched := percentage{percent: whole}
|
fetched := percentage{percent: whole}
|
||||||
if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed {
|
if percent, listed := rq.h.lists.ASNLimitPercent(asn); listed {
|
||||||
fetched = percentage{percent, "SWW_WAF_ASN_LIMIT_PERCENT_URL"}
|
fetched = percentage{percent, config.ASNLimitPercentURLSetting}
|
||||||
}
|
}
|
||||||
|
|
||||||
blocklisted := percentage{percent: whole}
|
blocklisted := percentage{percent: whole}
|
||||||
@@ -67,7 +67,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
cfg.BlocklistAction == config.ReputationActionLimit {
|
cfg.BlocklistAction == config.ReputationActionLimit {
|
||||||
blocklisted = percentage{
|
blocklisted = percentage{
|
||||||
cfg.BlocklistLimitPercent,
|
cfg.BlocklistLimitPercent,
|
||||||
"SWW_WAF_BLOCKLIST_ACTION",
|
config.BlocklistActionSetting,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -76,11 +76,11 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
|
cfg.DNSBLAbuseIPDBAction == config.ReputationActionLimit {
|
||||||
dnsblAbuseIPDBListed = percentage{
|
dnsblAbuseIPDBListed = percentage{
|
||||||
cfg.DNSBLAbuseIPDBLimitPercent,
|
cfg.DNSBLAbuseIPDBLimitPercent,
|
||||||
"SWW_WAF_DNSBL_ABUSEIPDB_ACTION",
|
config.DNSBLAbuseIPDBActionSetting,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const tagSetting = "SWW_WAF_TAG_LIMIT_PERCENT"
|
const tagSetting = config.TagLimitPercentSetting
|
||||||
|
|
||||||
asnRequests := lowest(
|
asnRequests := lowest(
|
||||||
given(cfg.TagLimitPercent, asnTag(asn), tagSetting),
|
given(cfg.TagLimitPercent, asnTag(asn), tagSetting),
|
||||||
@@ -108,7 +108,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
asnBytes = given(
|
asnBytes = given(
|
||||||
cfg.ASNBytesLimitPercent,
|
cfg.ASNBytesLimitPercent,
|
||||||
asn,
|
asn,
|
||||||
"SWW_WAF_ASN_BYTES_LIMIT_PERCENT",
|
config.ASNBytesLimitPercentSetting,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -116,7 +116,7 @@ func (rq *request) limitPercentages() (percentage, percentage) {
|
|||||||
countryBytes = given(
|
countryBytes = given(
|
||||||
cfg.CountryBytesLimitPercent,
|
cfg.CountryBytesLimitPercent,
|
||||||
country,
|
country,
|
||||||
"SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT",
|
config.CountryBytesLimitPercentSetting,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"sneak.berlin/go/smallwebwaf/internal/config"
|
||||||
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
"sneak.berlin/go/smallwebwaf/internal/requestlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -33,7 +34,8 @@ func (countries) check(
|
|||||||
return &verdict{
|
return &verdict{
|
||||||
action: requestlog.ActionCountryDenied,
|
action: requestlog.ActionCountryDenied,
|
||||||
figures: map[string]any{
|
figures: map[string]any{
|
||||||
"country": country, "setting": "SWW_WAF_EXCLUSIVELY_ALLOWED_COUNTRIES",
|
"country": country,
|
||||||
|
"setting": config.ExclusivelyAllowedCountriesSetting,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user