Resolve PTR names over DNS-over-HTTPS and cache them for the record TTL (closes #238 )

`internal/ptr` asks Google, Cloudflare, AdGuard and Quad9 in turn, 2 seconds in all per address. It keeps answers in memory and writes them through to `cache.db`, opened by the new `internal/cachedb`. A reserved address is not sent to the resolvers. The cache holds at most 250,000 addresses. Judgement calls: - Each resolver gets an equal share of the time left, so a hung resolver does not use up the 2 seconds. - The negative TTL is the smaller of the SOA minimum and the SOA's own TTL (RFC 2308). - Names that are not host names are dropped, as `net.Resolver.LookupAddr` does. Model: opus-5-5
2026-11-10 12:13:46 +00:00
9 changed files with 30 additions and 287 deletions
@@ -287,7 +287,6 @@ address. Every entry has the same fields:
    - `is_tor`: true when the address is on the `tor_exits` list
    - `is_vpn`: true when the address is on any list of VPN servers (see IP
      lists)
-    - `is_relay`: true when the address is on the `icloud_private_relay` list
    - `is_bogon`: true when the address is reserved, so that no network should
      announce it on the internet
 - `vpn_providers`: the names of the lists of VPN servers the address is on, or
@@ -422,9 +421,6 @@ The lists:
  last 30 days. From
  [FireHOL](https://iplists.firehol.org/?ipset=firehol_level3), downloaded every
  3 hours.
- `icloud_private_relay`: the networks where iCloud Private Relay traffic
-  leaves. From [Apple](https://mask-api.icloud.com/egress-ip-ranges.csv),
-  downloaded once a day.
 - `ivpn`: VPN servers run by IVPN. From
  [IVPN](https://api.ivpn.net/v5/servers.json), downloaded once a day.
 - `mullvad`: VPN servers run by Mullvad. From
@@ -435,14 +431,6 @@ The lists:
 - `pia`: VPN servers run by Private Internet Access. From
  [Private Internet Access](https://serverlist.piaservers.net/vpninfo/servers/v6),
  downloaded once a day.
- `protonvpn`: the /24 networks that hold ProtonVPN servers. The whole /24 is
-  listed because the ProtonVPN-IPs README says traffic leaves from unpublished
-  addresses in it. From the public
-  [ProtonVPN-IPs](https://github.com/tn3w/ProtonVPN-IPs) mirror of Proton's
-  server list, downloaded once a day.
- `protonvpn_entry`: the IPv4 and IPv6 addresses of ProtonVPN servers. From the
-  same [ProtonVPN-IPs](https://github.com/tn3w/ProtonVPN-IPs) mirror, downloaded
-  once a day.
 - `spamhaus_drop` and `spamhaus_dropv6`: IPv4 and IPv6 networks hijacked or
  leased by spammers or cyber criminals, according to Spamhaus. From
  [Spamhaus DROP](https://www.spamhaus.org/blocklists/do-not-route-or-peer/),
@@ -610,12 +598,6 @@ The ceilings add up to 3.7 GiB:
  limit caps the SQLite library.
 - About 0.2 GiB for everything else in the runtime.

-The `icloud_private_relay` list has about 285,000 networks. Neighbouring
-networks merge, which leaves about 6,900 address ranges. Loaded, the list adds
-about 0.3 MiB to the Go heap and under 6 MiB to the process's resident memory.
-Parsing the file raises the process's peak by up to about 55 MiB for about 0.2
-seconds. That happens at startup and whenever a download brings a new version.
-
 The SQLite library gets its memory from glibc, the C library. glibc keeps freed
 memory in pools called arenas, and hands it back to the kernel slowly. It
 creates up to eight arenas per processor core. So on a host with many cores the
@@ -53,13 +53,6 @@ problem the run finds.

 # Completed Steps

- 2026-11-10: the IP lookup checks the `icloud_private_relay` list of Apple's
-  iCloud Private Relay networks, downloaded once a day. Each result has
-  `flags.is_relay` (closes https://git.eeqj.de/sneak/routewatch/issues/237).
- 2026-11-10: the IP lookup checks the `protonvpn` list of ProtonVPN's /24
-  networks and the `protonvpn_entry` list of its servers' IPv4 and IPv6
-  addresses, from a public mirror of Proton's server list (closes
-  https://git.eeqj.de/sneak/routewatch/issues/236).
 - 2026-11-10: the IP lookup finds PTR names over DNS-over-HTTPS and caches them
  for the record's TTL, in memory and in `cache.db` (closes
  https://git.eeqj.de/sneak/routewatch/issues/238).
@@ -21,7 +21,7 @@ const (
 	downloadTimeout = time.Minute

 	// maxDownloadSize is the size of the largest list file accepted, in
-	// bytes. The largest list, iCloud Private Relay, is about 12 MiB.
+	// bytes. The largest list, FireHOL abusers, is about 2.5 MiB.
 	maxDownloadSize = 200 << 20
 )

@@ -51,16 +51,6 @@ const (
 	// FormatIVPNJSON is IVPN's JSON server list. The hosts of its wireguard
 	// and openvpn gateways have their addresses in host and v2ray.
 	FormatIVPNJSON
-
-	// FormatProtonVPN has one address or network per line, with comment lines
-	// that start with #. Unlike FormatLines, a file with no addresses or
-	// networks fails.
-	FormatProtonVPN
-
-	// FormatICloudPrivateRelayCSV is Apple's CSV file of iCloud Private Relay
-	// networks. Each line starts with a network, followed by its country,
-	// region and city.
-	FormatICloudPrivateRelayCSV
 )

 // List is one public IP list.
@@ -70,31 +70,6 @@ const (
 	"config":{"api":{"ips":["192.0.2.73"]}}}`
 )

-// protonVPNBody is a small file in the format of the ProtonVPN-IPs list.
-const protonVPNBody = "#\n# protonvpn_ip_ranges.txt\n" +
-	"# https://github.com/tn3w/ProtonVPN-IPs\n#\n" +
-	"# The /24 block of every advertised IPv4.\n#\n" +
-	"192.0.2.0/24\n198.51.100.0/24\n"
-
-// protonVPNEntryBody is a small file in the format of the ProtonVPN-IPs list
-// of server addresses.
-const protonVPNEntryBody = "#\n# protonvpn_entry_ips.txt\n" +
-	"# https://github.com/tn3w/ProtonVPN-IPs\n#\n" +
-	"# Entry IPs clients connect to, resolved from ProtonVPN hostnames.\n#\n" +
-	"192.0.2.80\n2001:db8::80\n"
-
-// iCloudPrivateRelayAddr is an address on iCloudPrivateRelayBody.
-const iCloudPrivateRelayAddr = "2001:db8:1::5"
-
-// iCloudPrivateRelayBody is a small file in the format of Apple's iCloud
-// Private Relay file.
-const iCloudPrivateRelayBody = "192.0.2.0/31,GB,GB-EN,London,\n" +
-	"198.51.100.32/27,US,US-CA,Los Angeles,\n" +
-	"2001:db8:1::/64,US,US-AK,Anchorage,\n"
-
-// emptyFileCase names the case of a wrong document that is an empty file.
-const emptyFileCase = "an empty file"
-
 // errorPage is a file that is no list, as a publisher's error page is.
 const errorPage = "<!DOCTYPE html>\n<html></html>\n"

@@ -255,15 +230,6 @@ func TestLookupFindsListedAddresses(t *testing.T) {
 			},
 			unlisted: []string{"192.0.2.12", "2001:db8::2"},
 		},
-		{
-			name:   "iCloud Private Relay networks",
-			format: FormatICloudPrivateRelayCSV,
-			body:   iCloudPrivateRelayBody,
-			listed: []string{
-				"192.0.2.1", "198.51.100.63", iCloudPrivateRelayAddr,
-			},
-			unlisted: []string{"192.0.2.2", "198.51.100.64", "2001:db8:2::1"},
-		},
 	})
 }

@@ -301,20 +267,6 @@ func TestLookupFindsVPNServers(t *testing.T) {
 			listed:   []string{"192.0.2.70", "192.0.2.71", "192.0.2.72"},
 			unlisted: []string{"192.0.2.73", "2001:db8::70"},
 		},
-		{
-			name:     "ProtonVPN networks",
-			format:   FormatProtonVPN,
-			body:     protonVPNBody,
-			listed:   []string{"192.0.2.0", "192.0.2.255", "198.51.100.7"},
-			unlisted: []string{"192.0.3.1", "2001:db8::90"},
-		},
-		{
-			name:     "ProtonVPN server addresses",
-			format:   FormatProtonVPN,
-			body:     protonVPNEntryBody,
-			listed:   []string{"192.0.2.80", "2001:db8::80"},
-			unlisted: []string{"192.0.2.81", "2001:db8::81"},
-		},
 	})
 }

@@ -506,12 +458,12 @@ func TestWrongVPNDocumentFailsToParse(t *testing.T) {

 	for _, f := range formats {
 		bodies := map[string]string{
-			"not JSON":     errorPage,
-			"other JSON":   `{"status":"ok","data":{"count":1}}`,
-			"no addresses": f.empty,
-			"bad address":  f.badAddress,
-			"cut off":      f.good[:len(f.good)/2],
-			emptyFileCase:  "",
+			"not JSON":      errorPage,
+			"other JSON":    `{"status":"ok","data":{"count":1}}`,
+			"no addresses":  f.empty,
+			"bad address":   f.badAddress,
+			"cut off":       f.good[:len(f.good)/2],
+			"an empty file": "",
 		}

 		for name, body := range bodies {
@@ -527,59 +479,6 @@ func TestWrongVPNDocumentFailsToParse(t *testing.T) {
 	}
 }

-// TestWrongProtonVPNDocumentFailsToParse parses documents that are not a
-// ProtonVPN-IPs list. Each fails, so none can replace a good copy.
-func TestWrongProtonVPNDocumentFailsToParse(t *testing.T) {
-	t.Parallel()
-
-	bodies := map[string]string{
-		"error page":    errorPage,
-		"JSON":          `{"status":"ok"}`,
-		"comments only": "#\n# protonvpn_ip_ranges.txt\n#\n",
-		"bad network":   "192.0.2.0/24\nexample\n",
-		emptyFileCase:   "",
-	}
-
-	for name, body := range bodies {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			addresses, err := parse(strings.NewReader(body), FormatProtonVPN)
-			if err == nil {
-				t.Errorf("parse returned %v and no error", addresses.Ranges())
-			}
-		})
-	}
-}
-
-// TestWrongICloudPrivateRelayDocumentFailsToParse parses documents that are
-// not Apple's iCloud Private Relay file. Each fails, so none can replace a
-// good copy.
-func TestWrongICloudPrivateRelayDocumentFailsToParse(t *testing.T) {
-	t.Parallel()
-
-	bodies := map[string]string{
-		"error page":     errorPage,
-		"JSON":           `{"status":"ok","count":1}`,
-		"no other field": "192.0.2.0/24\n",
-		"bad network":    "192.0.2.0/24,GB,GB-EN,London,\nexample,GB,GB-EN,London,\n",
-		"an address":     "192.0.2.1,GB,GB-EN,London,\n",
-		emptyFileCase:    "",
-	}
-
-	for name, body := range bodies {
-		t.Run(name, func(t *testing.T) {
-			t.Parallel()
-
-			addresses, err := parse(strings.NewReader(body),
-				FormatICloudPrivateRelayCSV)
-			if err == nil {
-				t.Errorf("parse returned %v and no error", addresses.Ranges())
-			}
-		})
-	}
-}
-
 // TestLookupMarksVPNLists looks up an address on a VPN list and on another
 // list. Only the VPN list's listing says it is one.
 func TestLookupMarksVPNLists(t *testing.T) {
@@ -624,10 +523,7 @@ func TestPublicListsMarkTheVPNLists(t *testing.T) {
 		}
 	}

-	want := []string{
-		"ivpn", "mullvad", "nordvpn", "pia", "protonvpn", "protonvpn_entry",
-		"x4bnet", "x4bnet_v6",
-	}
+	want := []string{"ivpn", "mullvad", "nordvpn", "pia", "x4bnet", "x4bnet_v6"}
 	if !slices.Equal(got, want) {
 		t.Errorf("VPN lists %v, want %v", got, want)
 	}
@@ -934,33 +830,15 @@ func TestStopCancelsADownloadInProgress(t *testing.T) {
 func TestTorExitsListHoldsIPv6Exits(t *testing.T) {
 	t.Parallel()

-	checkPublicListHolds(t, TorExitsList, onionooBody, onionooIPv6ExitAddr)
-}
-
-// TestICloudPrivateRelayListHoldsItsNetworks downloads a file in Apple's
-// format as the public iCloud Private Relay list, which is what an address's
-// is_relay answer comes from.
-func TestICloudPrivateRelayListHoldsItsNetworks(t *testing.T) {
-	t.Parallel()
-
-	checkPublicListHolds(t, ICloudPrivateRelayList, iCloudPrivateRelayBody,
-		iCloudPrivateRelayAddr)
-}
-
-// checkPublicListHolds downloads body as the public list called name and
-// checks that addr has one listing, which names that list.
-func checkPublicListHolds(t *testing.T, name, body, addr string) {
-	t.Helper()
-
 	index := slices.IndexFunc(Lists(), func(list List) bool {
-		return list.Name == name
+		return list.Name == TorExitsList
 	})
 	if index < 0 {
-		t.Fatalf("no list named %s", name)
+		t.Fatalf("no list named %s", TorExitsList)
 	}

 	list := Lists()[index]
-	_, list.URL = newPublisher(t, body)
+	_, list.URL = newPublisher(t, onionooBody)
 	b := newTestIPLists(t.TempDir(), list)

 	err := b.download(t.Context(), list)
@@ -968,9 +846,9 @@ func checkPublicListHolds(t *testing.T, name, body, addr string) {
 		t.Fatalf("download: %v", err)
 	}

-	listings := lookup(t, b, addr)
-	if len(listings) != 1 || listings[0].Name != name {
-		t.Errorf("listings %+v, want one for %s", listings, name)
+	listings := lookup(t, b, onionooIPv6ExitAddr)
+	if len(listings) != 1 || listings[0].Name != TorExitsList {
+		t.Errorf("listings %+v, want one for %s", listings, TorExitsList)
 	}
 }

@@ -8,13 +8,6 @@ import (
 // TorExitsList is the name of the list of Tor exit nodes.
 const TorExitsList = "tor_exits"

-// ICloudPrivateRelayList is the name of the list of iCloud Private Relay's
-// networks.
-const ICloudPrivateRelayList = "icloud_private_relay"
-
-// protonVPNIPsURL is where the ProtonVPN-IPs mirror publishes its files.
-const protonVPNIPsURL = "https://raw.githubusercontent.com/tn3w/ProtonVPN-IPs/master/"
-
 // How often each list is downloaded. No list is downloaded more often than
 // its publisher updates it or allows.
 const (
@@ -34,9 +27,6 @@ const (
 	fireHOLAbusersInterval = 5 * time.Hour
 	fireHOLLevel3Interval  = 3 * time.Hour

-	// Apple's iCloud Private Relay file is 12 MB, so once a day.
-	iCloudPrivateRelayInterval = 24 * time.Hour
-
 	// Spamhaus asks that DROP be downloaded at most once a day:
 	// https://www.spamhaus.org/blocklists/do-not-route-or-peer/
 	spamhausDROPInterval = 24 * time.Hour
@@ -91,13 +81,6 @@ func nonVPNLists() []List {
 			Interval: fireHOLLevel3Interval,
 			Format:   FormatLines,
 		},
-		{
-			Name:        ICloudPrivateRelayList,
-			Description: "In a network where iCloud Private Relay traffic leaves",
-			URL:         "https://mask-api.icloud.com/egress-ip-ranges.csv",
-			Interval:    iCloudPrivateRelayInterval,
-			Format:      FormatICloudPrivateRelayCSV,
-		},
 		{
 			Name: "spamhaus_drop",
 			Description: "In an IPv4 network that Spamhaus says spammers " +
@@ -163,28 +146,6 @@ func vpnLists() []List {
 			Format:      FormatPIAJSON,
 			VPN:         true,
 		},
-		{
-			// Proton's own server list needs a logged-in session, so this is
-			// the public ProtonVPN-IPs mirror of it. Proton publishes only
-			// the addresses clients connect to. The mirror's README says
-			// traffic leaves from others in the same /24, so this file of
-			// /24 networks is the one it says to use for detecting exits.
-			Name: "protonvpn",
-			Description: "In a /24 network that holds a server of the " +
-				"ProtonVPN service",
-			URL:      protonVPNIPsURL + "protonvpn_ip_ranges.txt",
-			Interval: vpnListInterval,
-			Format:   FormatProtonVPN,
-			VPN:      true,
-		},
-		{
-			Name:        "protonvpn_entry",
-			Description: "A server of the ProtonVPN service",
-			URL:         protonVPNIPsURL + "protonvpn_entry_ips.txt",
-			Interval:    vpnListInterval,
-			Format:      FormatProtonVPN,
-			VPN:         true,
-		},
 		{
 			Name: "x4bnet",
 			Description: "In an IPv4 network of a VPN service, according to " +
@@ -25,10 +25,6 @@ const (
 	// dshieldFieldSeparator separates the fields of a DShield line.
 	dshieldFieldSeparator = "\t"

-	// iCloudPrivateRelayFieldSeparator separates the fields of a line of
-	// Apple's iCloud Private Relay file.
-	iCloudPrivateRelayFieldSeparator = ","
-
 	// spamhausMetadataType is the type of the line that ends a Spamhaus
 	// file. That line holds the file's metadata and no network.
 	spamhausMetadataType = "metadata"
@@ -62,10 +58,6 @@ func parse(r io.Reader, format Format) (*netipx.IPSet, error) {
 		return parsePIA(r)
 	case FormatIVPNJSON:
 		return parseIVPN(r)
-	case FormatProtonVPN:
-		return parseLinesNotEmpty(r, addAddressOrNetwork)
-	case FormatICloudPrivateRelayCSV:
-		return parseLinesNotEmpty(r, addICloudPrivateRelayNetwork)
 	default:
 		return nil, errUnknownFormat
 	}
@@ -337,41 +329,6 @@ func parseIVPN(r io.Reader) (*netipx.IPSet, error) {
 	return addressSet(texts)
 }

-// parseLinesNotEmpty is parseLines for a list that always holds addresses, so
-// a file with none fails.
-func parseLinesNotEmpty(
-	r io.Reader, addLine func(*netipx.IPSetBuilder, string) error,
-) (*netipx.IPSet, error) {
-	addresses, err := parseLines(r, addLine)
-	if err != nil {
-		return nil, err
-	}
-
-	if len(addresses.Ranges()) == 0 {
-		return nil, errNoAddresses
-	}
-
-	return addresses, nil
-}
-
-// addICloudPrivateRelayNetwork adds the network at the start of a line of
-// Apple's iCloud Private Relay file to builder.
-func addICloudPrivateRelayNetwork(builder *netipx.IPSetBuilder, line string) error {
-	field, _, found := strings.Cut(line, iCloudPrivateRelayFieldSeparator)
-	if !found {
-		return errNoNetwork
-	}
-
-	network, err := netip.ParsePrefix(field)
-	if err != nil {
-		return err
-	}
-
-	builder.AddPrefix(network)
-
-	return nil
-}
-
 // readJSON reads r to its end and decodes it, as one JSON document, into v.
 func readJSON(r io.Reader, v any) error {
 	data, err := io.ReadAll(r)
@@ -797,9 +797,6 @@ type IPLookupFlags struct {
 	// IsVPN says that the address is on a list of VPN servers.
 	IsVPN bool `json:"is_vpn"`

-	// IsRelay says that the address is on the iCloud Private Relay list.
-	IsRelay bool `json:"is_relay"`
-
 	// IsBogon says that the address is reserved, as package bogon tells.
 	IsBogon bool `json:"is_bogon"`
 }
@@ -1028,9 +1025,6 @@ func ipLookupFlags(ip string, listings []iplist.Listing) IPLookupFlags {
 		IsVPN: slices.ContainsFunc(listings, func(l iplist.Listing) bool {
 			return l.VPN
 		}),
-		IsRelay: slices.ContainsFunc(listings, func(l iplist.Listing) bool {
-			return l.Name == iplist.ICloudPrivateRelayList
-		}),
 		IsBogon: isBogon(ip),
 	}
 }
@@ -1504,10 +1504,6 @@ const listedUnroutedAddr = "203.0.113.1"
 // torExitAddr is the address that fakeIPLists lists on the Tor exit list.
 const torExitAddr = "203.0.113.2"

-// relayAddr is the address that fakeIPLists lists on the iCloud Private Relay
-// list.
-const relayAddr = "203.0.113.5"
-
 // The addresses that fakeIPLists lists on one and on two VPN lists, and the
 // names of those lists.
 const (
@@ -1517,11 +1513,10 @@ const (
 	otherTestVPN    = "other_test_vpn"
 )

-// fakeIPLists has five lists. The test list holds addrInTestPrefixV4,
+// fakeIPLists has four lists. The test list holds addrInTestPrefixV4,
 // listedUnroutedAddr and twoVPNListsAddr. The Tor exit list holds torExitAddr.
-// The iCloud Private Relay list holds relayAddr. The test VPN list holds
-// oneVPNListAddr and twoVPNListsAddr, and the other test VPN list holds
-// twoVPNListsAddr.
+// The test VPN list holds oneVPNListAddr and twoVPNListsAddr, and the other
+// test VPN list holds twoVPNListsAddr.
 type fakeIPLists struct{}

 func testListing() iplist.Listing {
@@ -1542,10 +1537,6 @@ func (fakeIPLists) Lookup(addr netip.Addr) []iplist.Listing {
 		return []iplist.Listing{{Name: iplist.TorExitsList}}
 	}

-	if addr == netip.MustParseAddr(relayAddr) {
-		return []iplist.Listing{{Name: iplist.ICloudPrivateRelayList}}
-	}
-
 	if addr == netip.MustParseAddr(oneVPNListAddr) {
 		return []iplist.Listing{{Name: testVPNList, VPN: true}}
 	}
@@ -1578,22 +1569,20 @@ func TestIPLookupFlags(t *testing.T) {
 		name  string
 		addr  string
 		tor   bool
-		relay bool
 		bogon bool
 	}{
-		{"on the Tor exit list", torExitAddr, true, false, true},
-		{"on the iCloud Private Relay list", relayAddr, false, true, true},
-		{"on another list", listedUnroutedAddr, false, false, true},
-		{"routed IPv4", addrInTestPrefixV4, false, false, true},
-		{"routed IPv6", addrInTestPrefixV6, false, false, true},
-		{"private IPv4", "10.0.0.1", false, false, true},
-		{"loopback IPv4", "127.0.0.1", false, false, true},
-		{"public IPv4", "1.1.1.1", false, false, false},
-		{"private IPv6", "fd00::1", false, false, true},
-		{"loopback IPv6", "::1", false, false, true},
-		{"public IPv6", "2606:4700:4700::1111", false, false, false},
-		{"public IPv4 in IPv6 form", "::ffff:1.1.1.1", false, false, false},
-		{"private IPv4 in IPv6 form", "::ffff:10.0.0.1", false, false, true},
+		{"on the Tor exit list", torExitAddr, true, true},
+		{"on another list", listedUnroutedAddr, false, true},
+		{"routed IPv4", addrInTestPrefixV4, false, true},
+		{"routed IPv6", addrInTestPrefixV6, false, true},
+		{"private IPv4", "10.0.0.1", false, true},
+		{"loopback IPv4", "127.0.0.1", false, true},
+		{"public IPv4", "1.1.1.1", false, false},
+		{"private IPv6", "fd00::1", false, true},
+		{"loopback IPv6", "::1", false, true},
+		{"public IPv6", "2606:4700:4700::1111", false, false},
+		{"public IPv4 in IPv6 form", "::ffff:1.1.1.1", false, false},
+		{"private IPv4 in IPv6 form", "::ffff:10.0.0.1", false, true},
 	}

 	for _, tt := range tests {
@@ -1615,8 +1604,7 @@ func TestIPLookupFlags(t *testing.T) {
 			}

 			want := map[string]bool{
-				"is_tor": tt.tor, "is_vpn": false, "is_relay": tt.relay,
-				"is_bogon": tt.bogon,
+				"is_tor": tt.tor, "is_vpn": false, "is_bogon": tt.bogon,
 			}
 			if !maps.Equal(got, want) {
 				t.Errorf("%s: flags %v, want %v", path, got, want)