Rename blocklists to IP lists, add IP lookup flags, raise the download limit to 200 MiB (closes #233)
check / check (push) Failing after 10m45s
check / check (push) Failing after 10m45s
The lists are not only used for blocking, so the package, the `blocklists` field of the IP lookup answer and the state directory are now IP lists (`internal/iplist`, `ip_lists`, `iplists`). Every result gains a `flags` object with `is_tor` and `is_bogon`. The largest list file accepted is 200 MiB. Judgement call: the dated 2026-10-27 `TODO.md` entry keeps the old word. Rule suppressed: `tagliatelle` on the two result structs, for snake_case keys. Model: opus-5-5
This commit was merged in pull request #241.
This commit is contained in:
1 parent
e58c6635e7
commit
cf50d195ad
12 files changed
+375
-176
No files matched your search
@@ -0,0 +1,158 @@
|
||||
package iplist
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/netip"
|
||||
"strings"
|
||||
|
||||
"go4.org/netipx"
|
||||
)
|
||||
|
||||
const (
|
||||
// commentPrefix starts a comment line in a list file.
|
||||
commentPrefix = "#"
|
||||
|
||||
// networkSeparator separates the address and the mask length of a
|
||||
// network, as in 192.0.2.0/24.
|
||||
networkSeparator = "/"
|
||||
|
||||
// dshieldFieldSeparator separates the fields of a DShield line.
|
||||
dshieldFieldSeparator = "\t"
|
||||
|
||||
// spamhausMetadataType is the type of the line that ends a Spamhaus
|
||||
// file. That line holds the file's metadata and no network.
|
||||
spamhausMetadataType = "metadata"
|
||||
)
|
||||
|
||||
var (
|
||||
errUnknownFormat = errors.New("unknown list format")
|
||||
errNoNetwork = errors.New("no network on the line")
|
||||
errBadRange = errors.New("first and last address make no range")
|
||||
)
|
||||
|
||||
// parse reads a list file written in format and returns the addresses it
|
||||
// holds. A line it cannot read fails the whole file, so a file that is not
|
||||
// the list, such as an error page, never replaces a good copy.
|
||||
func parse(r io.Reader, format Format) (*netipx.IPSet, error) {
|
||||
var builder netipx.IPSetBuilder
|
||||
|
||||
scanner := bufio.NewScanner(r)
|
||||
|
||||
for lineNumber := 1; scanner.Scan(); lineNumber++ {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" || strings.HasPrefix(line, commentPrefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
err := addLine(&builder, line, format)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("line %d: %w", lineNumber, err)
|
||||
}
|
||||
}
|
||||
|
||||
err := scanner.Err()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return builder.IPSet()
|
||||
}
|
||||
|
||||
// addLine adds the address or network on one line of a list file to builder.
|
||||
func addLine(builder *netipx.IPSetBuilder, line string, format Format) error {
|
||||
switch format {
|
||||
case FormatLines:
|
||||
return addAddressOrNetwork(builder, line)
|
||||
case FormatSpamhausJSON:
|
||||
return addSpamhausNetwork(builder, line)
|
||||
case FormatDShield:
|
||||
return addDShieldNetwork(builder, line)
|
||||
default:
|
||||
return errUnknownFormat
|
||||
}
|
||||
}
|
||||
|
||||
// addAddressOrNetwork adds an address, such as 192.0.2.1, or a network, such
|
||||
// as 192.0.2.0/24, to builder.
|
||||
func addAddressOrNetwork(builder *netipx.IPSetBuilder, text string) error {
|
||||
if strings.Contains(text, networkSeparator) {
|
||||
network, err := netip.ParsePrefix(text)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
builder.AddPrefix(network)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
addr, err := netip.ParseAddr(text)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
builder.Add(addr)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// addSpamhausNetwork adds the network on a line of a Spamhaus file to
|
||||
// builder. The metadata line adds nothing.
|
||||
func addSpamhausNetwork(builder *netipx.IPSetBuilder, line string) error {
|
||||
var entry struct {
|
||||
CIDR string `json:"cidr"`
|
||||
Type string `json:"type"`
|
||||
}
|
||||
|
||||
err := json.Unmarshal([]byte(line), &entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if entry.Type == spamhausMetadataType {
|
||||
return nil
|
||||
}
|
||||
|
||||
network, err := netip.ParsePrefix(entry.CIDR)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
builder.AddPrefix(network)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// addDShieldNetwork adds the network on a line of a DShield file to builder.
|
||||
// The line's first two fields are the network's first and last address.
|
||||
func addDShieldNetwork(builder *netipx.IPSetBuilder, line string) error {
|
||||
firstField, rest, found := strings.Cut(line, dshieldFieldSeparator)
|
||||
if !found {
|
||||
return errNoNetwork
|
||||
}
|
||||
|
||||
lastField, _, _ := strings.Cut(rest, dshieldFieldSeparator)
|
||||
|
||||
first, err := netip.ParseAddr(firstField)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
last, err := netip.ParseAddr(lastField)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
network := netipx.IPRangeFrom(first, last)
|
||||
if !network.IsValid() {
|
||||
return errBadRange
|
||||
}
|
||||
|
||||
builder.AddRange(network)
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user