package iplist import ( "bufio" "encoding/json" "errors" "fmt" "io" "net" "net/netip" "strings" "go4.org/netipx" ) const ( // commentPrefix starts a comment line in a list file. commentPrefix = "#" // networkSeparator separates the address and the mask length of a // network, as in 192.0.2.0/24. networkSeparator = "/" // dshieldFieldSeparator separates the fields of a DShield line. dshieldFieldSeparator = "\t" // spamhausMetadataType is the type of the line that ends a Spamhaus // file. That line holds the file's metadata and no network. spamhausMetadataType = "metadata" ) var ( errUnknownFormat = errors.New("unknown list format") errNoNetwork = errors.New("no network on the line") errBadRange = errors.New("first and last address make no range") errNoAddresses = errors.New("no relay addresses in the document") ) // parse reads a list file written in format and returns the addresses it // holds. Anything it cannot read fails the whole file, so a file that is not // the list, such as an error page, never replaces a good copy. func parse(r io.Reader, format Format) (*netipx.IPSet, error) { switch format { case FormatLines: return parseLines(r, addAddressOrNetwork) case FormatSpamhausJSON: return parseLines(r, addSpamhausNetwork) case FormatDShield: return parseLines(r, addDShieldNetwork) case FormatOnionooJSON: return parseOnionoo(r) default: return nil, errUnknownFormat } } // parseLines reads a list file line by line, skipping blank lines and // comments, and calls addLine with each other line. func parseLines( r io.Reader, addLine func(*netipx.IPSetBuilder, string) error, ) (*netipx.IPSet, error) { var builder netipx.IPSetBuilder scanner := bufio.NewScanner(r) for lineNumber := 1; scanner.Scan(); lineNumber++ { line := strings.TrimSpace(scanner.Text()) if line == "" || strings.HasPrefix(line, commentPrefix) { continue } err := addLine(&builder, line) if err != nil { return nil, fmt.Errorf("line %d: %w", lineNumber, err) } } err := scanner.Err() if err != nil { return nil, err } return builder.IPSet() } // addAddressOrNetwork adds an address, such as 192.0.2.1, or a network, such // as 192.0.2.0/24, to builder. func addAddressOrNetwork(builder *netipx.IPSetBuilder, text string) error { if strings.Contains(text, networkSeparator) { network, err := netip.ParsePrefix(text) if err != nil { return err } builder.AddPrefix(network) return nil } addr, err := netip.ParseAddr(text) if err != nil { return err } builder.Add(addr) return nil } // addSpamhausNetwork adds the network on a line of a Spamhaus file to // builder. The metadata line adds nothing. func addSpamhausNetwork(builder *netipx.IPSetBuilder, line string) error { var entry struct { CIDR string `json:"cidr"` Type string `json:"type"` } err := json.Unmarshal([]byte(line), &entry) if err != nil { return err } if entry.Type == spamhausMetadataType { return nil } network, err := netip.ParsePrefix(entry.CIDR) if err != nil { return err } builder.AddPrefix(network) return nil } // addDShieldNetwork adds the network on a line of a DShield file to builder. // The line's first two fields are the network's first and last address. func addDShieldNetwork(builder *netipx.IPSetBuilder, line string) error { firstField, rest, found := strings.Cut(line, dshieldFieldSeparator) if !found { return errNoNetwork } lastField, _, _ := strings.Cut(rest, dshieldFieldSeparator) first, err := netip.ParseAddr(firstField) if err != nil { return err } last, err := netip.ParseAddr(lastField) if err != nil { return err } network := netipx.IPRangeFrom(first, last) if !network.IsValid() { return errBadRange } builder.AddRange(network) return nil } // parseOnionoo returns every address in the exit_addresses and or_addresses // fields of the relays in an Onionoo details document. A document with no // addresses fails, because the Tor network always has relays. func parseOnionoo(r io.Reader) (*netipx.IPSet, error) { data, err := io.ReadAll(r) if err != nil { return nil, err } //nolint:tagliatelle // Onionoo's own field names var details struct { Relays []struct { ExitAddresses []string `json:"exit_addresses"` ORAddresses []string `json:"or_addresses"` } `json:"relays"` } err = json.Unmarshal(data, &details) if err != nil { return nil, err } var builder netipx.IPSetBuilder for _, relay := range details.Relays { for _, text := range relay.ExitAddresses { err = addAddress(&builder, text) if err != nil { return nil, err } } for _, text := range relay.ORAddresses { err = addAddressWithPort(&builder, text) if err != nil { return nil, err } } } addresses, err := builder.IPSet() if err != nil { return nil, err } if len(addresses.Ranges()) == 0 { return nil, errNoAddresses } return addresses, nil } // addAddress adds an address, such as 192.0.2.1, to builder. func addAddress(builder *netipx.IPSetBuilder, text string) error { addr, err := netip.ParseAddr(text) if err != nil { return err } builder.Add(addr) return nil } // addAddressWithPort adds the address of text, such as 192.0.2.1:443 or // [2001:db8::1]:443, to builder. Onionoo allows a list of ports after the // address, so the part after the address is not read. func addAddressWithPort(builder *netipx.IPSetBuilder, text string) error { host, _, err := net.SplitHostPort(text) if err != nil { return err } return addAddress(builder, host) }