diff --git a/go.mod b/go.mod index 959cd68..b0d5138 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/prometheus/client_golang v1.24.1 github.com/spf13/viper v1.21.0 go.uber.org/fx v1.24.0 + golang.org/x/net v0.60.0 golang.org/x/sys v0.48.0 ) @@ -50,7 +51,6 @@ require ( go.uber.org/zap v1.26.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect - golang.org/x/net v0.60.0 // indirect golang.org/x/sync v0.23.0 // indirect golang.org/x/text v0.42.0 // indirect google.golang.org/protobuf v1.36.11 // indirect diff --git a/internal/config/config.go b/internal/config/config.go index f85a00e..b399c97 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -19,6 +19,8 @@ import ( "time" "github.com/spf13/viper" + + "sneak.berlin/go/smallwebwaf/internal/reputation" ) // off is the value that switches a timeout, a size limit or a rate limit @@ -40,9 +42,7 @@ var ( errNotPositive = errors.New("must be more than zero, or off") errNotMilliseconds = errors.New( "is not a whole number of milliseconds above zero, such as 2000") - errEmptyItem = errors.New("has an empty item in its list") - errNotNetblock = errors.New( - "is not a netblock such as 10.0.0.0/8, or an address") + errEmptyItem = errors.New("has an empty item in its list") errNotListenAddr = errors.New( "is not an address to listen on, such as :8080") errNotCommand = errors.New("is not rproxy or waf") @@ -253,7 +253,7 @@ func parseNetblocks(value string) ([]netip.Prefix, error) { netblocks := make([]netip.Prefix, 0, len(items)) for _, item := range items { - netblock, err := ParseNetblock(item) + netblock, err := reputation.ParseNetblock(item) if err != nil { return nil, err } @@ -264,27 +264,6 @@ func parseNetblocks(value string) ([]netip.Prefix, error) { return netblocks, nil } -// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare -// address is a netblock of that address alone, a /32 or a /128. A -// blocklist's lines are read with it too. -func ParseNetblock(value string) (netip.Prefix, error) { - if strings.Contains(value, "/") { - netblock, err := netip.ParsePrefix(value) - if err != nil { - return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock) - } - - return netblock.Masked(), nil - } - - addr, err := netip.ParseAddr(value) - if err != nil || addr.Zone() != "" { - return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock) - } - - return netip.PrefixFrom(addr, addr.BitLen()), nil -} - // parseListenAddr checks an address to listen on: an optional host and a // port number. func parseListenAddr(value string) (string, error) { diff --git a/internal/config/waf.go b/internal/config/waf.go index 50dc897..03a1602 100644 --- a/internal/config/waf.go +++ b/internal/config/waf.go @@ -20,11 +20,13 @@ import ( "unicode/utf8" "github.com/spf13/viper" + "golang.org/x/net/http/httpguts" "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/anomaly" "sneak.berlin/go/smallwebwaf/internal/coreruleset" "sneak.berlin/go/smallwebwaf/internal/remotelog" + "sneak.berlin/go/smallwebwaf/internal/reputation" ) // WAF is the settings of `sww waf`. A time or size limit of zero is off. @@ -100,7 +102,7 @@ type WAF struct { // with a tag that percentage of every rate limit and byte limit, from 1 // to 99 (SWW_WAF_TAG_LIMIT_PERCENT). All hold tags as requests carry them: in // lower case, a country code in capitals. - TagLists []TagList + TagLists []reputation.TagList TagDeny []string TagLimitPercent map[string]int64 // RouteWatchURL is where the routewatch instance is that adds the asn= @@ -497,12 +499,8 @@ var ( "is not a whole number above zero, such as 60, or off", ) errNotUTF8 = errors.New("is not valid UTF-8") - errNotASN = errors.New("is not an AS number such as AS64496") errNotPercentItem = errors.New( "is not a code, : and a percentage, such as AS64496:50 or cn:25") - errNotPercent = errors.New( - "is not a percentage, a whole number from 0 to 100", - ) errNotAbuseIPDBThreshold = errors.New( "is not a whole number from 25 to 100, the scores AbuseIPDB's " + "blacklist can be cut at", @@ -625,7 +623,7 @@ func WAFFromEnvironment(v *viper.Viper) (*WAF, error) { RouteWatchURL: env.routeWatchURL("SWW_WAF_ROUTEWATCH_URL"), ASNBytesLimitPercent: env.percents( "SWW_WAF_ASN_BYTES_LIMIT_PERCENT", - ParseASN, + reputation.ParseASN, ), CountryBytesLimitPercent: env.percents( "SWW_WAF_COUNTRY_BYTES_LIMIT_PERCENT", @@ -1011,7 +1009,7 @@ func (e *environment) percents( } // tagLists reads SWW_WAF_TAG_LISTS, which is empty by default. -func (e *environment) tagLists(name string) []TagList { +func (e *environment) tagLists(name string) []reputation.TagList { lists, err := parseTagLists(e.value(name, "")) e.check(name, err) @@ -1037,7 +1035,7 @@ func (e *environment) tagPercents(name string) map[string]int64 { // percent reads a setting that is a percentage, from 0 to 100. func (e *environment) percent(name, defaultValue string) int64 { - percent, err := ParsePercent(e.value(name, defaultValue)) + percent, err := reputation.ParsePercent(e.value(name, defaultValue)) e.check(name, err) return percent @@ -1122,7 +1120,7 @@ func (e *environment) action( string(value), string(ReputationActionLimit)+":", ) - percent, err := ParsePercent(percentText) + percent, err := reputation.ParsePercent(percentText) if !isLimit || err != nil { e.check(name, fmt.Errorf("%q %w", value, errNotAction)) } @@ -1131,8 +1129,8 @@ func (e *environment) action( } // zones reads the setting that is the list of DNSBL zones. It is empty by -// default. The log shows each zone with its key masked, as MaskZoneKey -// masks it. +// default. The log shows each zone with its key masked, as +// reputation.MaskZoneKey masks it. func (e *environment) zones(name string) []string { value, _ := e.lookup(name) zones, err := parseZones(value) @@ -1140,7 +1138,7 @@ func (e *environment) zones(name string) []string { logged := make([]string, len(zones)) for i, zone := range zones { - logged[i] = MaskZoneKey(zone) + logged[i] = reputation.MaskZoneKey(zone) } e.settings = append( @@ -1301,7 +1299,11 @@ func (e *environment) checkTags(cfg *WAF) { // checkTag returns an error when tag would never apply, as checkTags says. func checkTag(cfg *WAF, tag string) error { if !requestCanGetTag(cfg, tag) { - return fmt.Errorf("%q %w", MaskZoneKey(tag), errNoRequestGetsTag) + return fmt.Errorf( + "%q %w", + reputation.MaskZoneKey(tag), + errNoRequestGetsTag, + ) } return nil @@ -1309,7 +1311,7 @@ func checkTag(cfg *WAF, tag string) error { // requestCanGetTag reports whether anything cfg sets up can give a request tag. func requestCanGetTag(cfg *WAF, tag string) bool { - isListName := func(list TagList) bool { return list.Name == tag } + isListName := func(list reputation.TagList) bool { return list.Name == tag } if isLookupTag(tag) || slices.ContainsFunc(cfg.TagLists, isListName) { return true } @@ -1333,7 +1335,7 @@ func requestCanGetTag(cfg *WAF, tag string) bool { return isZoneTag && slices.ContainsFunc(cfg.DNSBLZones, func(each string) bool { - return strings.ToLower(MaskZoneKey(each)) == zone + return strings.ToLower(reputation.MaskZoneKey(each)) == zone }) } @@ -1831,7 +1833,9 @@ func parseNamedNetblocks(value string) ([]anomaly.NamedNetblock, error) { return nil, fmt.Errorf("%q %w", item, errNotNamedNetblock) } - netblock, err := ParseNetblock(strings.TrimSpace(netblockText)) + netblock, err := reputation.ParseNetblock( + strings.TrimSpace(netblockText), + ) if err != nil { return nil, err } @@ -2037,21 +2041,6 @@ func parseCountry(value string) (string, error) { return country, nil } -// ParseASN reads an AS number such as AS64496, in either case, and -// returns it as the lookup gives it: AS and the number, in capitals and -// without leading zeros. The file SWW_WAF_ASN_LIMIT_PERCENT_URL names is -// read with it too. -func ParseASN(value string) (string, error) { - digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS") - - number, err := strconv.ParseUint(digits, 10, 32) - if !hasAS || err != nil { - return "", fmt.Errorf("%q %w", value, errNotASN) - } - - return "AS" + strconv.FormatUint(number, 10), nil -} - // The tags a request can get that the settings can name, as waf // writes them. The prefixes are followed by a value. const ( @@ -2065,24 +2054,17 @@ const ( ExploitSignatureTag = "exploitsignature" ) -// TagList is an address list of SWW_WAF_TAG_LISTS: the tag it adds to the -// requests of the clients it holds, and the URL it is fetched from. -type TagList struct { - Name string - URL string -} - // parseTagLists reads a comma-separated list of name=URL pairs, such as // torexit=https://example.com/tor.txt, and returns each list with its name // in lower case. A name can hold a =, as vpn=mullvad does, and ends at the // = before http. A name or a URL listed twice is an error. -func parseTagLists(value string) ([]TagList, error) { +func parseTagLists(value string) ([]reputation.TagList, error) { items, err := parseList(value) if err != nil { return nil, err } - lists := make([]TagList, 0, len(items)) + lists := make([]reputation.TagList, 0, len(items)) for _, item := range items { end := strings.Index(item, "=http") @@ -2090,9 +2072,12 @@ func parseTagLists(value string) ([]TagList, error) { return nil, fmt.Errorf("%q %w", item, errNotTagList) } - list := TagList{Name: strings.ToLower(item[:end]), URL: item[end+1:]} + list := reputation.TagList{ + Name: strings.ToLower(item[:end]), + URL: item[end+1:], + } - if slices.ContainsFunc(lists, func(earlier TagList) bool { + if slices.ContainsFunc(lists, func(earlier reputation.TagList) bool { return earlier.Name == list.Name || earlier.URL == list.URL }) { return nil, fmt.Errorf("%q %w", item, errListedTwice) @@ -2105,7 +2090,8 @@ func parseTagLists(value string) ([]TagList, error) { } // parseTags reads a comma-separated list of tags, which parseTag reads. A -// tag listed twice is an error, which shows the tag as MaskZoneKey does. +// tag listed twice is an error, which shows the tag as +// reputation.MaskZoneKey does. func parseTags(value string) ([]string, error) { items, err := parseList(value) if err != nil { @@ -2125,7 +2111,7 @@ func parseTags(value string) ([]string, error) { if slices.Contains(tags, tag) { return nil, fmt.Errorf( "%q %w", - MaskZoneKey(item), + reputation.MaskZoneKey(item), errListedTwice, ) } @@ -2194,7 +2180,7 @@ func parseTagPercents(value string) (map[string]int64, error) { if colon <= 0 { return nil, fmt.Errorf( "%q %w", - MaskZoneKey(item), + reputation.MaskZoneKey(item), errNotTagPercentItem, ) } @@ -2208,7 +2194,7 @@ func parseTagPercents(value string) (map[string]int64, error) { if err != nil || percent < 1 || percent > 99 { return nil, fmt.Errorf( "%q %w", - MaskZoneKey(item[colon+1:]), + reputation.MaskZoneKey(item[colon+1:]), errNotTagPercent, ) } @@ -2220,7 +2206,11 @@ func parseTagPercents(value string) (map[string]int64, error) { if _, listed := percents[tag]; listed { // tag, since item[:colon] may end in a space, which MaskZoneKey // would not mask. - return nil, fmt.Errorf("%q %w", MaskZoneKey(tag), errListedTwice) + return nil, fmt.Errorf( + "%q %w", + reputation.MaskZoneKey(tag), + errListedTwice, + ) } percents[tag] = percent @@ -2254,7 +2244,7 @@ func parsePercents( return nil, err } - percent, err := ParsePercent(percentText) + percent, err := reputation.ParsePercent(percentText) if err != nil { return nil, err } @@ -2269,38 +2259,6 @@ func parsePercents( return percents, nil } -// ParsePercent reads a percentage, a whole number from 0 to 100. The file -// SWW_WAF_ASN_LIMIT_PERCENT_URL names is read with it too. -func ParsePercent(value string) (int64, error) { - percent, err := strconv.ParseInt(value, 10, 64) - if err != nil || percent < 0 || percent > 100 { - return 0, fmt.Errorf("%q %w", value, errNotPercent) - } - - return percent, nil -} - -// headerNameChars are the characters RFC 9110 allows in a header name: -// letters, digits and these marks. -const headerNameChars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + - "0123456789!#$%&'*+-.^_`|~" - -// IsHeaderName reports whether name can be a header name: one or more of -// the characters RFC 9110 allows in one. -func IsHeaderName(name string) bool { - if name == "" { - return false - } - - for _, char := range name { - if !strings.ContainsRune(headerNameChars, char) { - return false - } - } - - return true -} - // parseHeaderNames reads a comma-separated list of header names in either // case, and returns them in lower case. Host and Transfer-Encoding are // refused: Go's HTTP server takes them out of the request's headers. @@ -2313,7 +2271,7 @@ func parseHeaderNames(value string) ([]string, error) { headers := make([]string, 0, len(items)) for _, item := range items { - if !IsHeaderName(item) { + if !httpguts.ValidHeaderFieldName(item) { return nil, fmt.Errorf("%q %w", item, errNotHeaderName) } @@ -2469,8 +2427,8 @@ const ( // other name for one that does not exist, so that the zone would list no // client. A zone listed twice is an error, whatever the case of its // letters, which DNS names ignore, and whatever its key, since -// MaskZoneKey shows two keys of one zone alike. An error shows a zone as -// MaskZoneKey does. +// reputation.MaskZoneKey shows two keys of one zone alike. An error shows a +// zone as it does. func parseZones(value string) ([]string, error) { zones, err := parseList(value) if err != nil { @@ -2480,11 +2438,14 @@ func parseZones(value string) ([]string, error) { } for i, zone := range zones { - shown := MaskZoneKey(zone) + shown := reputation.MaskZoneKey(zone) listedBefore := slices.ContainsFunc( zones[:i], func(earlier string) bool { - return strings.EqualFold(MaskZoneKey(earlier), shown) + return strings.EqualFold( + reputation.MaskZoneKey(earlier), + shown, + ) }, ) @@ -2501,24 +2462,6 @@ func parseZones(value string) ([]string, error) { return zones, nil } -// MaskZoneKey returns zone with ******** in place of its key, if it is a -// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net, -// such as .xbl.dq.spamhaus.net, whose first label is the key. Any -// other zone it returns as it is. A zone is shown so wherever it leaves -// the process: in the log, the alerts and the metrics. -func MaskZoneKey(zone string) string { - // DNS names ignore case, and a name may be written with a dot at its - // end. - name := strings.TrimSuffix(strings.ToLower(zone), ".") - if !strings.HasSuffix(name, ".dq.spamhaus.net") { - return zone - } - - _, rest, _ := strings.Cut(zone, ".") - - return masked + "." + rest -} - // isZone reports whether each label of zone is as parseZones takes it. It // also checks the hostnames of SWW_WAF_APPS, which are DNS names too. func isZone(zone string) bool { @@ -2574,7 +2517,7 @@ func parseWebhookHeaders(value string) (http.Header, string, error) { name, headerValue, found := strings.Cut(item, ":") name = strings.TrimSpace(name) - if !found || !IsHeaderName(name) || + if !found || !httpguts.ValidHeaderFieldName(name) || strings.ContainsAny(headerValue, "\r\n\x00") { return nil, "", fmt.Errorf( "item %d %w", diff --git a/internal/config/waf_test.go b/internal/config/waf_test.go index 8f4b862..209da16 100644 --- a/internal/config/waf_test.go +++ b/internal/config/waf_test.go @@ -18,10 +18,12 @@ import ( "time" "github.com/spf13/viper" + "golang.org/x/net/http/httpguts" "sneak.berlin/go/smallwebwaf/internal/anomaly" "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/config/configtest" + "sneak.berlin/go/smallwebwaf/internal/reputation" ) // The settings, by name. @@ -864,6 +866,32 @@ func TestInstanceNameAndLoggedHeadersAsSet(t *testing.T) { } } +// config and rules check header names with httpguts.ValidHeaderFieldName. +// It takes a name of one or more of the characters RFC 9110 allows in one, +// as the check it replaced did: letters, digits and these marks. +func TestHeaderNameCheckTakesTheRFC9110Characters(t *testing.T) { + t.Parallel() + + const marks = "!#$%&'*+-.^_`|~" + + if httpguts.ValidHeaderFieldName("") { + t.Error("the empty name is taken") + } + + for b := range 256 { + char := byte(b) + want := 'a' <= char && char <= 'z' || 'A' <= char && char <= 'Z' || + '0' <= char && char <= '9' || strings.IndexByte(marks, char) >= 0 + + // string(char) would be the UTF-8 of U+0080 to U+00FF, two bytes, + // for a byte above 0x7f. + name := string([]byte{char}) + if got := httpguts.ValidHeaderFieldName(name); got != want { + t.Errorf("the name %q is taken: %t, want %t", name, got, want) + } + } +} + func TestRemoteLogSettingsDefaults(t *testing.T) { t.Parallel() @@ -1694,7 +1722,7 @@ func TestTagSettingsAsSet(t *testing.T) { tagLimitPercent: "vpn=mullvad :50, asn=014061:25, COUNTRY=cn:1, dnsbl=zen.example:99", }) - wantLists := []config.TagList{ + wantLists := []reputation.TagList{ {Name: torExitTag, URL: torURL}, {Name: "vpn=mullvad", URL: "https://example.com/mullvad.txt"}, } @@ -2288,25 +2316,6 @@ func TestInvalidDNSBLSettingStopsTheStartSayingWhatIsWrong(t *testing.T) { } } -func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet( - t *testing.T, -) { - t.Parallel() - - for zone, want := range map[string]string{ - spamhaus: spamhausMasked, - spamhaus + ".": spamhausMasked + ".", - "KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET", - dronebl: dronebl, - "dq.spamhaus.net": "dq.spamhaus.net", - spamhaus + ".example": spamhaus + ".example", - } { - if got := config.MaskZoneKey(zone); got != want { - t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want) - } - } -} - func TestDNSBLZoneKeyIsLoggedMaskedAndNeverShown(t *testing.T) { t.Parallel() diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 6e66332..a1c4aa8 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -14,7 +14,6 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "sneak.berlin/go/smallwebwaf/internal/alerts" "sneak.berlin/go/smallwebwaf/internal/bans" - "sneak.berlin/go/smallwebwaf/internal/config" "sneak.berlin/go/smallwebwaf/internal/ratelimit" "sneak.berlin/go/smallwebwaf/internal/remotelog" "sneak.berlin/go/smallwebwaf/internal/reputation" @@ -301,7 +300,7 @@ const sourceLabel = "source" // AddReputation adds the metrics of the lists fetched from URLs and of the // DNSBL zones, by source, each list's URL or each zone, its key masked as -// config.MaskZoneKey masks it: the requests whose client a blocklist, a +// reputation.MaskZoneKey masks it: the requests whose client a blocklist, a // zone's verdict or AbuseIPDB's score lists, which ReputationHit counts, // and, read from lists and dnsbl as the metrics are asked for, for a list, // the fetches that failed and when the copy in use was fetched, and for a @@ -320,7 +319,9 @@ func (m *Metrics) AddReputation( m.registry.MustRegister(m.reputationHits) for _, zone := range dnsbl.Zones() { - source := prometheus.Labels{sourceLabel: config.MaskZoneKey(zone)} + source := prometheus.Labels{ + sourceLabel: reputation.MaskZoneKey(zone), + } m.addReputationQueries( source, diff --git a/internal/reputation/dnsbl.go b/internal/reputation/dnsbl.go index 4391f77..09fac3e 100644 --- a/internal/reputation/dnsbl.go +++ b/internal/reputation/dnsbl.go @@ -16,7 +16,6 @@ import ( "github.com/hashicorp/golang-lru/v2/simplelru" "sneak.berlin/go/smallwebwaf/internal/alerts" - "sneak.berlin/go/smallwebwaf/internal/config" ) const ( @@ -32,6 +31,8 @@ const ( // it fails, and no client is checked with AbuseIPDB after a check // fails, so that a source refusing them is not asked on every request. failureDelay = time.Minute + // maskedKey is what MaskZoneKey shows in place of a zone's key. + maskedKey = "********" ) var ( @@ -127,6 +128,24 @@ func NewDNSBL(params DNSBLParams) *DNSBL { } } +// MaskZoneKey returns zone with ******** in place of its key, if it is a +// zone of Spamhaus's keyed query service, a name under dq.spamhaus.net, +// such as .xbl.dq.spamhaus.net, whose first label is the key. Any +// other zone it returns as it is. A zone is shown so wherever it leaves +// the process: in the log, the alerts and the metrics. +func MaskZoneKey(zone string) string { + // DNS names ignore case, and a name may be written with a dot at its + // end. + name := strings.TrimSuffix(strings.ToLower(zone), ".") + if !strings.HasSuffix(name, ".dq.spamhaus.net") { + return zone + } + + _, rest, _ := strings.Cut(zone, ".") + + return maskedKey + "." + rest +} + // Zones returns the zones, in the order SWW_WAF_DNSBL_ZONES names them. func (d *DNSBL) Zones() []string { return slices.Clone(d.params.Zones) @@ -134,7 +153,7 @@ func (d *DNSBL) Zones() []string { // ListedBy returns the zones whose verdict on addr, a client's address, // lists it, in the order SWW_WAF_DNSBL_ZONES names them, each with its key -// masked, as config.MaskZoneKey masks it, since they go to the request +// masked, as MaskZoneKey masks it, since they go to the request // log, the alerts and the metrics. A verdict is used until CacheTTL has // passed since it was fetched. Each zone without one is asked about addr // in the background, unless a query about addr to it is under way, the @@ -157,7 +176,7 @@ func (d *DNSBL) ListedBy(ctx context.Context, addr netip.Addr) []string { switch { case found && now.Sub(kept.Fetched) < d.params.CacheTTL: if kept.Listed { - listedBy = append(listedBy, config.MaskZoneKey(zone)) + listedBy = append(listedBy, MaskZoneKey(zone)) } case !d.asking[q] && !now.Before(d.retryAt[zone]) && len(d.asking) < maxQueries: d.asking[q] = true @@ -261,7 +280,7 @@ func (d *DNSBL) ask(ctx context.Context, q query) { if err != nil { const failed = "asking a DNSBL zone failed" - shown := config.MaskZoneKey(q.zone) + shown := MaskZoneKey(q.zone) // Raised before it is logged, so that the alert is there once the // log line is. diff --git a/internal/reputation/dnsbl_test.go b/internal/reputation/dnsbl_test.go index dfe1e74..b91a614 100644 --- a/internal/reputation/dnsbl_test.go +++ b/internal/reputation/dnsbl_test.go @@ -821,3 +821,28 @@ func wantAsked(t *testing.T, resolver *resolverStandIn, want ...string) { t.Errorf("asked about %v, want %v", got, want) } } + +func TestMaskZoneKeyMasksTheFirstLabelOfAZoneUnderDqSpamhausNet( + t *testing.T, +) { + t.Parallel() + + const ( + dronebl = "dnsbl.dronebl.org" + spamhaus = "abcdefghijklmnopqrstuvwxyz.xbl.dq.spamhaus.net" + spamhausMasked = "********.xbl.dq.spamhaus.net" + ) + + for zone, want := range map[string]string{ + spamhaus: spamhausMasked, + spamhaus + ".": spamhausMasked + ".", + "KEY.ZEN.DQ.SPAMHAUS.NET": "********.ZEN.DQ.SPAMHAUS.NET", + dronebl: dronebl, + "dq.spamhaus.net": "dq.spamhaus.net", + spamhaus + ".example": spamhaus + ".example", + } { + if got := reputation.MaskZoneKey(zone); got != want { + t.Errorf("MaskZoneKey(%q) is %q, want %q", zone, got, want) + } + } +} diff --git a/internal/reputation/reputation.go b/internal/reputation/reputation.go index a8790e9..fe9a855 100644 --- a/internal/reputation/reputation.go +++ b/internal/reputation/reputation.go @@ -7,8 +7,10 @@ // SWW_WAF_DNSBL_ZONES about clients, and keeps their verdicts, and checks // clients with AbuseIPDB, and keeps their scores and the checks spent // today. The state package writes all of these to files and reads them -// from there, so that a restart keeps them too. Last, it fetches -// AbuseIPDB's blacklist, which it does not keep. +// from there, so that a restart keeps them too. It fetches AbuseIPDB's +// blacklist, which it does not keep. Last, it reads the netblocks, AS +// numbers and percentages that both the settings and the lists hold, and +// masks the keys of DNSBL zones. package reputation import ( @@ -20,12 +22,12 @@ import ( "net/http" "net/netip" "slices" + "strconv" "strings" "sync" "time" "sneak.berlin/go/smallwebwaf/internal/alerts" - "sneak.berlin/go/smallwebwaf/internal/config" ) const ( @@ -40,15 +42,28 @@ const ( ) var ( - errStatus = errors.New("the server answered") - errTooLong = errors.New("the list is longer than 16 MiB") - errNotNetblock = errors.New( + errStatus = errors.New("the server answered") + errTooLong = errors.New("the list is longer than 16 MiB") + errNotNetblockLine = errors.New( "is not an address or a netblock, such as 192.0.2.0/24", ) errNotASNPercent = errors.New( "is not an AS number, : and a percentage, such as AS64496:50") + errNotNetblock = errors.New( + "is not a netblock such as 10.0.0.0/8, or an address") + errNotASN = errors.New("is not an AS number such as AS64496") + errNotPercent = errors.New( + "is not a percentage, a whole number from 0 to 100", + ) ) +// TagList is an address list of SWW_WAF_TAG_LISTS: the tag it adds to the +// requests of the clients it holds, and the URL it is fetched from. +type TagList struct { + Name string + URL string +} + // List is a list as lists.json holds it: the URL it is fetched from, // when it was last tried, the fetch failed or not, and its last good copy: // when that was fetched, and its lines, as fetched, comment lines @@ -68,7 +83,7 @@ type Params struct { BlocklistURLs []string ASNLimitPercentURL string // TagLists are the address lists that add a tag (SWW_WAF_TAG_LISTS). - TagLists []config.TagList + TagLists []TagList // Refresh is how long after a list was last fetched or tried it is // fetched again (SWW_WAF_BLOCKLIST_REFRESH). Refresh time.Duration @@ -465,7 +480,11 @@ func parseNetblocks(lines []string) (entries, error) { netblock, ok := parseNetblock(text) if !ok { - return entries{}, fmt.Errorf("line %d %w", i+1, errNotNetblock) + return entries{}, fmt.Errorf( + "line %d %w", + i+1, + errNotNetblockLine, + ) } found.netblocks[netblock] = true @@ -485,7 +504,7 @@ func parseNetblocks(lines []string) (entries, error) { // stands for, 192.0.2.0/24, and a mapped netblock shorter than /96, which // stands for none, is refused. func parseNetblock(text string) (netip.Prefix, bool) { - netblock, err := config.ParseNetblock(text) + netblock, err := ParseNetblock(text) if err != nil { return netip.Prefix{}, false } @@ -520,9 +539,9 @@ func parsePercents(lines []string) (entries, error) { } asnText, percentText, _ := strings.Cut(text, ":") - asn, asnErr := config.ParseASN(asnText) + asn, asnErr := ParseASN(asnText) - percent, percentErr := config.ParsePercent(percentText) + percent, percentErr := ParsePercent(percentText) if asnErr != nil || percentErr != nil { return entries{}, fmt.Errorf("line %d %w", i+1, errNotASNPercent) } @@ -536,6 +555,53 @@ func parsePercents(lines []string) (entries, error) { return found, nil } +// ParseNetblock reads a netblock in CIDR form, such as 10.0.0.0/8. A bare +// address is a netblock of that address alone, a /32 or a /128. A +// blocklist's lines are read with it too. +func ParseNetblock(value string) (netip.Prefix, error) { + if strings.Contains(value, "/") { + netblock, err := netip.ParsePrefix(value) + if err != nil { + return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock) + } + + return netblock.Masked(), nil + } + + addr, err := netip.ParseAddr(value) + if err != nil || addr.Zone() != "" { + return netip.Prefix{}, fmt.Errorf("%q %w", value, errNotNetblock) + } + + return netip.PrefixFrom(addr, addr.BitLen()), nil +} + +// ParseASN reads an AS number such as AS64496, in either case, and +// returns it as the lookup gives it: AS and the number, in capitals and +// without leading zeros. The file SWW_WAF_ASN_LIMIT_PERCENT_URL names is +// read with it too. +func ParseASN(value string) (string, error) { + digits, hasAS := strings.CutPrefix(strings.ToUpper(value), "AS") + + number, err := strconv.ParseUint(digits, 10, 32) + if !hasAS || err != nil { + return "", fmt.Errorf("%q %w", value, errNotASN) + } + + return "AS" + strconv.FormatUint(number, 10), nil +} + +// ParsePercent reads a percentage, a whole number from 0 to 100. The file +// SWW_WAF_ASN_LIMIT_PERCENT_URL names is read with it too. +func ParsePercent(value string) (int64, error) { + percent, err := strconv.ParseInt(value, 10, 64) + if err != nil || percent < 0 || percent > 100 { + return 0, fmt.Errorf("%q %w", value, errNotPercent) + } + + return percent, nil +} + // withoutComment returns line without anything after a ; or a #, and // without the spaces around what is left. func withoutComment(line string) string { diff --git a/internal/rules/rules.go b/internal/rules/rules.go index 192a957..ac9bbd1 100644 --- a/internal/rules/rules.go +++ b/internal/rules/rules.go @@ -21,9 +21,9 @@ import ( "time" "github.com/fsnotify/fsnotify" + "golang.org/x/net/http/httpguts" "sneak.berlin/go/smallwebwaf/internal/alerts" - "sneak.berlin/go/smallwebwaf/internal/config" ) // The actions a rule takes when it matches. @@ -378,7 +378,7 @@ func parse(line string) (Rule, bool, error) { rule.Target, errNotTarget, ) - case isHeader && !config.IsHeaderName(headerName): + case isHeader && !httpguts.ValidHeaderFieldName(headerName): return Rule{}, false, fmt.Errorf( "the target %q %w", rule.Target,