Add the public VPN server lists, with is_vpn and vpn_providers (closes #235)

The IP lookup now checks Mullvad, NordVPN, Private Internet Access, IVPN and X4BNet's community list, each downloaded once a day. Each result has `flags.is_vpn` and `vpn_providers`.

Judgement call: X4BNet publishes IPv4 and IPv6 as two files, so they are two lists, `x4bnet` and `x4bnet_v6`.
Judgement call: Private Internet Access's signature is not read or checked.
Rule suppressed: `tagliatelle` on the structs holding the publishers' field names.

Model: opus-5-5

internal/iplist/iplist.go

@@ -34,6 +34,23 @@ const (
 	// addresses in two fields: exit_addresses, which has IPv4 only, and
 	// or_addresses, which has IPv4 and IPv6, each with a port.
 	FormatOnionooJSON
+
+	// FormatMullvadJSON is Mullvad's JSON list of relays. A relay's addresses
+	// are in ipv4_addr_in, ipv6_addr_in and ipv4_v2ray.
+	FormatMullvadJSON
+
+	// FormatNordVPNJSON is NordVPN's JSON list of servers. A server's
+	// addresses are in station and ipv6_station.
+	FormatNordVPNJSON
+
+	// FormatPIAJSON is Private Internet Access's JSON server list, then a
+	// blank line and a signature. Each region's servers has lists of servers
+	// by protocol, and each server has its address in ip.
+	FormatPIAJSON
+
+	// FormatIVPNJSON is IVPN's JSON server list. The hosts of its wireguard
+	// and openvpn gateways have their addresses in host and v2ray.
+	FormatIVPNJSON
 )
 
 // List is one public IP list.
@@ -53,6 +70,9 @@ type List struct {
 
 	// Format is how the list's file is written.
 	Format Format
+
+	// VPN says that the list holds servers of VPN services.
+	VPN bool
 }
 
 // Listing says that a list contains an address.
@@ -63,6 +83,10 @@ type Listing struct {
 	// Updated is when routewatch last downloaded the list, or last heard from
 	// its publisher that its copy was still current.
 	Updated time.Time `json:"updated"`
+
+	// VPN says that the list holds servers of VPN services. A lookup answer
+	// reports it through is_vpn and vpn_providers.
+	VPN bool `json:"-"`
 }
 
 // listCopy is a list's last good copy in memory.
@@ -129,6 +153,7 @@ func (b *IPLists) Lookup(addr netip.Addr) []Listing {
 				Name:        list.Name,
 				Description: list.Description,
 				Updated:     c.updated,
+				VPN:         list.VPN,
 			})
 		}
 	}

internal/iplist/iplist_internal_test.go

@@ -43,6 +43,36 @@ const (
 	],"bridges":[]}`
 )
 
+// Small documents in the VPN services' formats. PIA's and IVPN's have no
+// public IPv6 addresses, and IVPN's holds addresses that are not its servers,
+// which must not be read.
+const (
+	mullvadBody = `[
+		{"hostname":"a-wg-001","ipv4_addr_in":"192.0.2.40","ipv6_addr_in":"2001:db8::40"},
+		{"hostname":"a-br-001","ipv4_addr_in":"192.0.2.41","ipv6_addr_in":null,
+		 "ipv4_v2ray":"192.0.2.42"}
+	]`
+	nordVPNBody = `[
+		{"hostname":"a1.example.com","station":"192.0.2.50","ipv6_station":""},
+		{"hostname":"a2.example.com","station":"192.0.2.51","ipv6_station":"2001:db8::51"}
+	]`
+	piaBody = `{"groups":{"wg":[{"name":"wireguard","ports":[1337]}]},"regions":[
+		{"id":"a","servers":{
+			"wg":[{"ip":"192.0.2.60","cn":"a1"}],
+			"ikev2":[{"ip":"192.0.2.61","cn":"a1"}]}},
+		{"id":"b","servers":{"meta":[{"ip":"192.0.2.62","cn":"b1","van":true}]}}
+	]}` + "\n\nc2lnbmF0dXJl\nbGluZSB0d28=\n"
+	ivpnBody = `{"wireguard":[{"gateway":"a.wg.example.com","hosts":[
+		{"hostname":"a1","host":"192.0.2.70","v2ray":"192.0.2.71",
+		 "ipv6":{"local_ip":"2001:db8::70/96"}}]}],
+	"openvpn":[{"gateway":"a.gw.example.com","hosts":[
+		{"hostname":"a1","host":"192.0.2.72"}]}],
+	"config":{"api":{"ips":["192.0.2.73"]}}}`
+)
+
+// errorPage is a file that is no list, as a publisher's error page is.
+const errorPage = "<!DOCTYPE html>\n<html></html>\n"
+
 // testETag is the ETag a publisher sends with a version of the test list.
 const testETag = `"v1"`
 
@@ -158,13 +188,7 @@ func lookup(t *testing.T, b *IPLists, addr string) []Listing {
 func TestLookupFindsListedAddresses(t *testing.T) {
 	t.Parallel()
 
-	tests := []struct {
-		name     string
-		format   Format
-		body     string
-		listed   []string
-		unlisted []string
-	}{
+	runLookupCases(t, []lookupCase{
 		{
 			name:     "single addresses",
 			format:   FormatLines,
@@ -206,9 +230,60 @@ func TestLookupFindsListedAddresses(t *testing.T) {
 			},
 			unlisted: []string{"192.0.2.12", "2001:db8::2"},
 		},
-	}
+	})
+}
 
-	for _, tt := range tests {
+// TestLookupFindsVPNServers downloads a list in each VPN service's format and
+// looks up addresses on it and off it.
+func TestLookupFindsVPNServers(t *testing.T) {
+	t.Parallel()
+
+	runLookupCases(t, []lookupCase{
+		{
+			name:     "Mullvad relays",
+			format:   FormatMullvadJSON,
+			body:     mullvadBody,
+			listed:   []string{"192.0.2.40", "192.0.2.41", "192.0.2.42", "2001:db8::40"},
+			unlisted: []string{"192.0.2.43", "2001:db8::41"},
+		},
+		{
+			name:     "NordVPN servers",
+			format:   FormatNordVPNJSON,
+			body:     nordVPNBody,
+			listed:   []string{"192.0.2.50", "192.0.2.51", "2001:db8::51"},
+			unlisted: []string{"192.0.2.52", "2001:db8::50"},
+		},
+		{
+			name:     "PIA servers, with the signature after the JSON",
+			format:   FormatPIAJSON,
+			body:     piaBody,
+			listed:   []string{"192.0.2.60", "192.0.2.61", "192.0.2.62"},
+			unlisted: []string{"192.0.2.63", "2001:db8::60"},
+		},
+		{
+			name:     "IVPN hosts",
+			format:   FormatIVPNJSON,
+			body:     ivpnBody,
+			listed:   []string{"192.0.2.70", "192.0.2.71", "192.0.2.72"},
+			unlisted: []string{"192.0.2.73", "2001:db8::70"},
+		},
+	})
+}
+
+// lookupCase is a list file, the addresses it holds and some it does not.
+type lookupCase struct {
+	name     string
+	format   Format
+	body     string
+	listed   []string
+	unlisted []string
+}
+
+// runLookupCases downloads each case's list file and checks its addresses.
+func runLookupCases(t *testing.T, cases []lookupCase) {
+	t.Helper()
+
+	for _, tt := range cases {
 		t.Run(tt.name, func(t *testing.T) {
 			t.Parallel()
 
@@ -260,7 +335,7 @@ func TestFailedDownloadKeepsThePreviousCopy(t *testing.T) {
 		body   string
 	}{
 		{"server error", http.StatusInternalServerError, "unavailable"},
-		{"not a list", http.StatusOK, "<!DOCTYPE html>\n<html></html>\n"},
+		{"not a list", http.StatusOK, errorPage},
 	}
 
 	for _, tt := range tests {
@@ -306,7 +381,7 @@ func TestFileThatIsNotAnOnionooDocumentKeepsThePreviousCopy(t *testing.T) {
 		name string
 		body string
 	}{
-		{"not JSON", "<!DOCTYPE html>\n<html></html>\n"},
+		{"not JSON", errorPage},
 		{"other JSON", `{"status":"ok","data":{"relays_count":1}}`},
 		{"JSON array", `[{"or_addresses":["192.0.2.1:443"]}]`},
 		{"no relays", `{"relays":[],"bridges":[]}`},
@@ -348,6 +423,112 @@ func TestFileThatIsNotAnOnionooDocumentKeepsThePreviousCopy(t *testing.T) {
 	}
 }
 
+// TestWrongVPNDocumentFailsToParse parses documents that are not a VPN
+// service's list. Each fails, so none can replace a good copy.
+func TestWrongVPNDocumentFailsToParse(t *testing.T) {
+	t.Parallel()
+
+	formats := []struct {
+		name       string
+		format     Format
+		good       string
+		empty      string
+		badAddress string
+	}{
+		{
+			"Mullvad", FormatMullvadJSON, mullvadBody,
+			`[{"hostname":"a"}]`, `[{"ipv4_addr_in":"example"}]`,
+		},
+		{
+			"NordVPN", FormatNordVPNJSON, nordVPNBody,
+			`[{"hostname":"a","station":"","ipv6_station":""}]`,
+			`[{"station":"example"}]`,
+		},
+		{
+			"PIA", FormatPIAJSON, piaBody,
+			`{"regions":[{"id":"a","servers":{}}]}`,
+			`{"regions":[{"servers":{"wg":[{"ip":"example"}]}}]}`,
+		},
+		{
+			"IVPN", FormatIVPNJSON, ivpnBody,
+			`{"wireguard":[{"hosts":[]}],"openvpn":[]}`,
+			`{"wireguard":[{"hosts":[{"host":"example"}]}]}`,
+		},
+	}
+
+	for _, f := range formats {
+		bodies := map[string]string{
+			"not JSON":      errorPage,
+			"other JSON":    `{"status":"ok","data":{"count":1}}`,
+			"no addresses":  f.empty,
+			"bad address":   f.badAddress,
+			"cut off":       f.good[:len(f.good)/2],
+			"an empty file": "",
+		}
+
+		for name, body := range bodies {
+			t.Run(f.name+" "+name, func(t *testing.T) {
+				t.Parallel()
+
+				addresses, err := parse(strings.NewReader(body), f.format)
+				if err == nil {
+					t.Errorf("parse returned %v and no error", addresses.Ranges())
+				}
+			})
+		}
+	}
+}
+
+// TestLookupMarksVPNLists looks up an address on a VPN list and on another
+// list. Only the VPN list's listing says it is one.
+func TestLookupMarksVPNLists(t *testing.T) {
+	t.Parallel()
+
+	_, url := newPublisher(t, goodListBody)
+	vpnList := testList(url, FormatLines)
+	vpnList.Name = "test_vpn_list"
+	vpnList.VPN = true
+	otherList := testList(url, FormatLines)
+
+	b := New(Params{
+		Dir:    t.TempDir(),
+		Lists:  []List{vpnList, otherList},
+		Logger: slog.New(slog.DiscardHandler),
+	})
+
+	for _, list := range b.lists {
+		err := b.download(t.Context(), list)
+		if err != nil {
+			t.Fatalf("download of %s: %v", list.Name, err)
+		}
+	}
+
+	listings := lookup(t, b, listedAddr)
+	if len(listings) != 2 || !listings[0].VPN || listings[1].VPN {
+		t.Errorf("listings %+v, want the first of two marked as a VPN list",
+			listings)
+	}
+}
+
+// TestPublicListsMarkTheVPNLists checks which public lists are VPN lists,
+// since their names are what vpn_providers reports.
+func TestPublicListsMarkTheVPNLists(t *testing.T) {
+	t.Parallel()
+
+	var got []string
+
+	for _, list := range Lists() {
+		if list.VPN {
+			got = append(got, list.Name)
+		}
+	}
+
+	want := []string{"ivpn", "mullvad", "nordvpn", "pia", "x4bnet", "x4bnet_v6"}
+	if !slices.Equal(got, want) {
+		t.Errorf("VPN lists %v, want %v", got, want)
+	}
+}
+
 // checkGoodCopyOnDisk checks that dir holds the good version of the test list
 // and no part of a failed download.
 func checkGoodCopyOnDisk(t *testing.T, dir string) {

internal/iplist/lists.go

@@ -1,6 +1,9 @@
 package iplist
 
-import "time"
+import (
+	"slices"
+	"time"
+)
 
 // TorExitsList is the name of the list of Tor exit nodes.
 const TorExitsList = "tor_exits"
@@ -32,12 +35,20 @@ const (
 	// Onionoo sets no download limit:
 	// https://metrics.torproject.org/onionoo.html
 	torExitsInterval = time.Hour
+
+	// A VPN service's servers change slowly, so once a day is enough.
+	vpnListInterval = 24 * time.Hour
 )
 
 // Lists returns the public IP lists routewatch downloads. Each can be
 // downloaded whole without an account or a key. Spamhaus merged its EDROP
 // list into DROP, so DROP covers both.
 func Lists() []List {
+	return slices.Concat(nonVPNLists(), vpnLists())
+}
+
+// nonVPNLists returns the public lists that are not lists of VPN servers.
+func nonVPNLists() []List {
 	return []List{
 		{
 			Name: "dshield",
@@ -96,3 +107,64 @@ func Lists() []List {
 		},
 	}
 }
+
+// vpnLists returns the public lists of VPN servers. Their names are what
+// vpn_providers reports.
+func vpnLists() []List {
+	return []List{
+		{
+			Name:        "ivpn",
+			Description: "A server of the IVPN service",
+			URL:         "https://api.ivpn.net/v5/servers.json",
+			Interval:    vpnListInterval,
+			Format:      FormatIVPNJSON,
+			VPN:         true,
+		},
+		{
+			Name:        "mullvad",
+			Description: "A server of the Mullvad VPN service",
+			URL:         "https://api.mullvad.net/www/relays/all/",
+			Interval:    vpnListInterval,
+			Format:      FormatMullvadJSON,
+			VPN:         true,
+		},
+		{
+			Name:        "nordvpn",
+			Description: "A server of the NordVPN service",
+			URL: "https://api.nordvpn.com/v1/servers?limit=0" +
+				"&fields[servers.station]&fields[servers.ipv6_station]" +
+				"&fields[servers.hostname]",
+			Interval: vpnListInterval,
+			Format:   FormatNordVPNJSON,
+			VPN:      true,
+		},
+		{
+			Name:        "pia",
+			Description: "A server of the Private Internet Access VPN service",
+			URL:         "https://serverlist.piaservers.net/vpninfo/servers/v6",
+			Interval:    vpnListInterval,
+			Format:      FormatPIAJSON,
+			VPN:         true,
+		},
+		{
+			Name: "x4bnet",
+			Description: "In an IPv4 network of a VPN service, according to " +
+				"X4BNet's community list",
+			URL: "https://raw.githubusercontent.com/X4BNet/lists_vpn/main/" +
+				"output/vpn/ipv4.txt",
+			Interval: vpnListInterval,
+			Format:   FormatLines,
+			VPN:      true,
+		},
+		{
+			Name: "x4bnet_v6",
+			Description: "In an IPv6 network of a VPN service, according to " +
+				"X4BNet's community list",
+			URL: "https://raw.githubusercontent.com/X4BNet/lists_vpn/main/" +
+				"output/vpn/ipv6.txt",
+			Interval: vpnListInterval,
+			Format:   FormatLines,
+			VPN:      true,
+		},
+	}
+}

internal/iplist/parse.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"net"
 	"net/netip"
+	"slices"
 	"strings"
 
 	"go4.org/netipx"
@@ -33,7 +34,7 @@ var (
 	errUnknownFormat = errors.New("unknown list format")
 	errNoNetwork     = errors.New("no network on the line")
 	errBadRange      = errors.New("first and last address make no range")
-	errNoAddresses   = errors.New("no relay addresses in the document")
+	errNoAddresses   = errors.New("no addresses in the document")
 )
 
 // parse reads a list file written in format and returns the addresses it
@@ -49,6 +50,14 @@ func parse(r io.Reader, format Format) (*netipx.IPSet, error) {
 		return parseLines(r, addDShieldNetwork)
 	case FormatOnionooJSON:
 		return parseOnionoo(r)
+	case FormatMullvadJSON:
+		return parseMullvad(r)
+	case FormatNordVPNJSON:
+		return parseNordVPN(r)
+	case FormatPIAJSON:
+		return parsePIA(r)
+	case FormatIVPNJSON:
+		return parseIVPN(r)
 	default:
 		return nil, errUnknownFormat
 	}
@@ -168,11 +177,6 @@ func addDShieldNetwork(builder *netipx.IPSetBuilder, line string) error {
 // fields of the relays in an Onionoo details document. A document with no
 // addresses fails, because the Tor network always has relays.
 func parseOnionoo(r io.Reader) (*netipx.IPSet, error) {
-	data, err := io.ReadAll(r)
-	if err != nil {
-		return nil, err
-	}
-
 	//nolint:tagliatelle // Onionoo's own field names
 	var details struct {
 		Relays []struct {
@@ -181,7 +185,7 @@ func parseOnionoo(r io.Reader) (*netipx.IPSet, error) {
 		} `json:"relays"`
 	}
 
-	err = json.Unmarshal(data, &details)
+	err := readJSON(r, &details)
 	if err != nil {
 		return nil, err
 	}
@@ -216,6 +220,153 @@ func parseOnionoo(r io.Reader) (*netipx.IPSet, error) {
 	return addresses, nil
 }
 
+// parseMullvad returns the addresses of the relays in a Mullvad document. A
+// relay with no IPv6 address has null for it, and only bridges have
+// ipv4_v2ray.
+func parseMullvad(r io.Reader) (*netipx.IPSet, error) {
+	//nolint:tagliatelle // Mullvad's own field names
+	var relays []struct {
+		IPv4      string `json:"ipv4_addr_in"`
+		IPv6      string `json:"ipv6_addr_in"`
+		IPv4V2Ray string `json:"ipv4_v2ray"`
+	}
+
+	err := readJSON(r, &relays)
+	if err != nil {
+		return nil, err
+	}
+
+	var texts []string
+
+	for _, relay := range relays {
+		texts = append(texts, relay.IPv4, relay.IPv6, relay.IPv4V2Ray)
+	}
+
+	return addressSet(texts)
+}
+
+// parseNordVPN returns the addresses of the servers in a NordVPN document. A
+// server with no IPv6 address has an empty ipv6_station.
+func parseNordVPN(r io.Reader) (*netipx.IPSet, error) {
+	//nolint:tagliatelle // NordVPN's own field names
+	var servers []struct {
+		Station     string `json:"station"`
+		IPv6Station string `json:"ipv6_station"`
+	}
+
+	err := readJSON(r, &servers)
+	if err != nil {
+		return nil, err
+	}
+
+	var texts []string
+
+	for _, server := range servers {
+		texts = append(texts, server.Station, server.IPv6Station)
+	}
+
+	return addressSet(texts)
+}
+
+// parsePIA returns the addresses of the servers in a Private Internet Access
+// document. json.Decoder stops at the end of the JSON, so the signature after
+// it is not read.
+func parsePIA(r io.Reader) (*netipx.IPSet, error) {
+	var list struct {
+		Regions []struct {
+			Servers map[string][]struct {
+				IP string `json:"ip"`
+			} `json:"servers"`
+		} `json:"regions"`
+	}
+
+	err := json.NewDecoder(r).Decode(&list)
+	if err != nil {
+		return nil, err
+	}
+
+	var texts []string
+
+	for _, region := range list.Regions {
+		for _, servers := range region.Servers {
+			for _, server := range servers {
+				texts = append(texts, server.IP)
+			}
+		}
+	}
+
+	return addressSet(texts)
+}
+
+// parseIVPN returns the addresses of the hosts in an IVPN document. A host's
+// ipv6 object holds only an address inside the VPN, so it is not read.
+func parseIVPN(r io.Reader) (*netipx.IPSet, error) {
+	type gateway struct {
+		Hosts []struct {
+			Host  string `json:"host"`
+			V2Ray string `json:"v2ray"`
+		} `json:"hosts"`
+	}
+
+	var list struct {
+		WireGuard []gateway `json:"wireguard"`
+		OpenVPN   []gateway `json:"openvpn"`
+	}
+
+	err := readJSON(r, &list)
+	if err != nil {
+		return nil, err
+	}
+
+	var texts []string
+
+	for _, gw := range slices.Concat(list.WireGuard, list.OpenVPN) {
+		for _, host := range gw.Hosts {
+			texts = append(texts, host.Host, host.V2Ray)
+		}
+	}
+
+	return addressSet(texts)
+}
+
+// readJSON reads r to its end and decodes it, as one JSON document, into v.
+func readJSON(r io.Reader, v any) error {
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return err
+	}
+
+	return json.Unmarshal(data, v)
+}
+
+// addressSet returns the set of the addresses in texts, skipping empty ones.
+// A list with no addresses fails, because every VPN service has servers.
+func addressSet(texts []string) (*netipx.IPSet, error) {
+	var builder netipx.IPSetBuilder
+
+	for _, text := range texts {
+		if text == "" {
+			continue
+		}
+
+		err := addAddress(&builder, text)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	addresses, err := builder.IPSet()
+	if err != nil {
+		return nil, err
+	}
+
+	if len(addresses.Ranges()) == 0 {
+		return nil, errNoAddresses
+	}
+
+	return addresses, nil
+}
+
 // addAddress adds an address, such as 192.0.2.1, to builder.
 func addAddress(builder *netipx.IPSetBuilder, text string) error {
 	addr, err := netip.ParseAddr(text)