package iplist import ( "bufio" "encoding/json" "errors" "fmt" "io" "net" "net/netip" "slices" "strings" "go4.org/netipx" ) const ( // commentPrefix starts a comment line in a list file. commentPrefix = "#" // networkSeparator separates the address and the mask length of a // network, as in 192.0.2.0/24. networkSeparator = "/" // dshieldFieldSeparator separates the fields of a DShield line. dshieldFieldSeparator = "\t" // iCloudPrivateRelayFieldSeparator separates the fields of a line of // Apple's iCloud Private Relay file. iCloudPrivateRelayFieldSeparator = "," // spamhausMetadataType is the type of the line that ends a Spamhaus // file. That line holds the file's metadata and no network. spamhausMetadataType = "metadata" ) var ( errUnknownFormat = errors.New("unknown list format") errNoNetwork = errors.New("no network on the line") errBadRange = errors.New("first and last address make no range") errNoAddresses = errors.New("no addresses in the document") ) // parse reads a list file written in format and returns the addresses it // holds. Anything it cannot read fails the whole file, so a file that is not // the list, such as an error page, never replaces a good copy. func parse(r io.Reader, format Format) (*netipx.IPSet, error) { switch format { case FormatLines: return parseLines(r, addAddressOrNetwork) case FormatSpamhausJSON: return parseLines(r, addSpamhausNetwork) case FormatDShield: return parseLines(r, addDShieldNetwork) case FormatOnionooJSON: return parseOnionoo(r) case FormatMullvadJSON: return parseMullvad(r) case FormatNordVPNJSON: return parseNordVPN(r) case FormatPIAJSON: return parsePIA(r) case FormatIVPNJSON: return parseIVPN(r) case FormatProtonVPN: return parseLinesNotEmpty(r, addAddressOrNetwork) case FormatICloudPrivateRelayCSV: return parseLinesNotEmpty(r, addICloudPrivateRelayNetwork) default: return nil, errUnknownFormat } } // parseLines reads a list file line by line, skipping blank lines and // comments, and calls addLine with each other line. func parseLines( r io.Reader, addLine func(*netipx.IPSetBuilder, string) error, ) (*netipx.IPSet, error) { var builder netipx.IPSetBuilder scanner := bufio.NewScanner(r) for lineNumber := 1; scanner.Scan(); lineNumber++ { line := strings.TrimSpace(scanner.Text()) if line == "" || strings.HasPrefix(line, commentPrefix) { continue } err := addLine(&builder, line) if err != nil { return nil, fmt.Errorf("line %d: %w", lineNumber, err) } } err := scanner.Err() if err != nil { return nil, err } return builder.IPSet() } // addAddressOrNetwork adds an address, such as 192.0.2.1, or a network, such // as 192.0.2.0/24, to builder. func addAddressOrNetwork(builder *netipx.IPSetBuilder, text string) error { if strings.Contains(text, networkSeparator) { network, err := netip.ParsePrefix(text) if err != nil { return err } builder.AddPrefix(network) return nil } addr, err := netip.ParseAddr(text) if err != nil { return err } builder.Add(addr) return nil } // addSpamhausNetwork adds the network on a line of a Spamhaus file to // builder. The metadata line adds nothing. func addSpamhausNetwork(builder *netipx.IPSetBuilder, line string) error { var entry struct { CIDR string `json:"cidr"` Type string `json:"type"` } err := json.Unmarshal([]byte(line), &entry) if err != nil { return err } if entry.Type == spamhausMetadataType { return nil } network, err := netip.ParsePrefix(entry.CIDR) if err != nil { return err } builder.AddPrefix(network) return nil } // addDShieldNetwork adds the network on a line of a DShield file to builder. // The line's first two fields are the network's first and last address. func addDShieldNetwork(builder *netipx.IPSetBuilder, line string) error { firstField, rest, found := strings.Cut(line, dshieldFieldSeparator) if !found { return errNoNetwork } lastField, _, _ := strings.Cut(rest, dshieldFieldSeparator) first, err := netip.ParseAddr(firstField) if err != nil { return err } last, err := netip.ParseAddr(lastField) if err != nil { return err } network := netipx.IPRangeFrom(first, last) if !network.IsValid() { return errBadRange } builder.AddRange(network) return nil } // parseOnionoo returns every address in the exit_addresses and or_addresses // fields of the relays in an Onionoo details document. A document with no // addresses fails, because the Tor network always has relays. func parseOnionoo(r io.Reader) (*netipx.IPSet, error) { //nolint:tagliatelle // Onionoo's own field names var details struct { Relays []struct { ExitAddresses []string `json:"exit_addresses"` ORAddresses []string `json:"or_addresses"` } `json:"relays"` } err := readJSON(r, &details) if err != nil { return nil, err } var builder netipx.IPSetBuilder for _, relay := range details.Relays { for _, text := range relay.ExitAddresses { err = addAddress(&builder, text) if err != nil { return nil, err } } for _, text := range relay.ORAddresses { err = addAddressWithPort(&builder, text) if err != nil { return nil, err } } } addresses, err := builder.IPSet() if err != nil { return nil, err } if len(addresses.Ranges()) == 0 { return nil, errNoAddresses } return addresses, nil } // parseMullvad returns the addresses of the relays in a Mullvad document. A // relay with no IPv6 address has null for it, and only bridges have // ipv4_v2ray. func parseMullvad(r io.Reader) (*netipx.IPSet, error) { //nolint:tagliatelle // Mullvad's own field names var relays []struct { IPv4 string `json:"ipv4_addr_in"` IPv6 string `json:"ipv6_addr_in"` IPv4V2Ray string `json:"ipv4_v2ray"` } err := readJSON(r, &relays) if err != nil { return nil, err } var texts []string for _, relay := range relays { texts = append(texts, relay.IPv4, relay.IPv6, relay.IPv4V2Ray) } return addressSet(texts) } // parseNordVPN returns the addresses of the servers in a NordVPN document. A // server with no IPv6 address has an empty ipv6_station. func parseNordVPN(r io.Reader) (*netipx.IPSet, error) { //nolint:tagliatelle // NordVPN's own field names var servers []struct { Station string `json:"station"` IPv6Station string `json:"ipv6_station"` } err := readJSON(r, &servers) if err != nil { return nil, err } var texts []string for _, server := range servers { texts = append(texts, server.Station, server.IPv6Station) } return addressSet(texts) } // parsePIA returns the addresses of the servers in a Private Internet Access // document. json.Decoder stops at the end of the JSON, so the signature after // it is not read. func parsePIA(r io.Reader) (*netipx.IPSet, error) { var list struct { Regions []struct { Servers map[string][]struct { IP string `json:"ip"` } `json:"servers"` } `json:"regions"` } err := json.NewDecoder(r).Decode(&list) if err != nil { return nil, err } var texts []string for _, region := range list.Regions { for _, servers := range region.Servers { for _, server := range servers { texts = append(texts, server.IP) } } } return addressSet(texts) } // parseIVPN returns the addresses of the hosts in an IVPN document. A host's // ipv6 object holds only an address inside the VPN, so it is not read. func parseIVPN(r io.Reader) (*netipx.IPSet, error) { type gateway struct { Hosts []struct { Host string `json:"host"` V2Ray string `json:"v2ray"` } `json:"hosts"` } var list struct { WireGuard []gateway `json:"wireguard"` OpenVPN []gateway `json:"openvpn"` } err := readJSON(r, &list) if err != nil { return nil, err } var texts []string for _, gw := range slices.Concat(list.WireGuard, list.OpenVPN) { for _, host := range gw.Hosts { texts = append(texts, host.Host, host.V2Ray) } } return addressSet(texts) } // parseLinesNotEmpty is parseLines for a list that always holds addresses, so // a file with none fails. func parseLinesNotEmpty( r io.Reader, addLine func(*netipx.IPSetBuilder, string) error, ) (*netipx.IPSet, error) { addresses, err := parseLines(r, addLine) if err != nil { return nil, err } if len(addresses.Ranges()) == 0 { return nil, errNoAddresses } return addresses, nil } // addICloudPrivateRelayNetwork adds the network at the start of a line of // Apple's iCloud Private Relay file to builder. func addICloudPrivateRelayNetwork(builder *netipx.IPSetBuilder, line string) error { field, _, found := strings.Cut(line, iCloudPrivateRelayFieldSeparator) if !found { return errNoNetwork } network, err := netip.ParsePrefix(field) if err != nil { return err } builder.AddPrefix(network) return nil } // readJSON reads r to its end and decodes it, as one JSON document, into v. func readJSON(r io.Reader, v any) error { data, err := io.ReadAll(r) if err != nil { return err } return json.Unmarshal(data, v) } // addressSet returns the set of the addresses in texts, skipping empty ones. // A list with no addresses fails, because every VPN service has servers. func addressSet(texts []string) (*netipx.IPSet, error) { var builder netipx.IPSetBuilder for _, text := range texts { if text == "" { continue } err := addAddress(&builder, text) if err != nil { return nil, err } } addresses, err := builder.IPSet() if err != nil { return nil, err } if len(addresses.Ranges()) == 0 { return nil, errNoAddresses } return addresses, nil } // addAddress adds an address, such as 192.0.2.1, to builder. func addAddress(builder *netipx.IPSetBuilder, text string) error { addr, err := netip.ParseAddr(text) if err != nil { return err } builder.Add(addr) return nil } // addAddressWithPort adds the address of text, such as 192.0.2.1:443 or // [2001:db8::1]:443, to builder. Onionoo allows a list of ports after the // address, so the part after the address is not read. func addAddressWithPort(builder *netipx.IPSetBuilder, text string) error { host, _, err := net.SplitHostPort(text) if err != nil { return err } return addAddress(builder, host) }