Add the iCloud Private Relay list to the IP lookup, with is_relay (closes #237)

Adds Apple's iCloud Private Relay egress list as `icloud_private_relay`, downloaded once a day, and `flags.is_relay`, true when an address is on it. Loaded, the list adds about 0.3 MiB to the Go heap; parsing it allocates about 94 MiB, freed by the next garbage collection.

Judgement call: a line with no comma fails the parse, so a plain network list served at Apple's URL cannot replace a good copy.

Model: opus-5-5

internal/iplist/download.go

@@ -21,7 +21,7 @@ const (
 	downloadTimeout = time.Minute
 
 	// maxDownloadSize is the size of the largest list file accepted, in
-	// bytes. The largest list, FireHOL abusers, is about 2.5 MiB.
+	// bytes. The largest list, iCloud Private Relay, is about 12 MiB.
 	maxDownloadSize = 200 << 20
 )
 

internal/iplist/iplist.go

@@ -56,6 +56,11 @@ const (
 	// that start with #. Unlike FormatLines, a file with no addresses or
 	// networks fails.
 	FormatProtonVPN
+
+	// FormatICloudPrivateRelayCSV is Apple's CSV file of iCloud Private Relay
+	// networks. Each line starts with a network, followed by its country,
+	// region and city.
+	FormatICloudPrivateRelayCSV
 )
 
 // List is one public IP list.

internal/iplist/iplist_internal_test.go

@@ -83,6 +83,18 @@ const protonVPNEntryBody = "#\n# protonvpn_entry_ips.txt\n" +
 	"# Entry IPs clients connect to, resolved from ProtonVPN hostnames.\n#\n" +
 	"192.0.2.80\n2001:db8::80\n"
 
+// iCloudPrivateRelayAddr is an address on iCloudPrivateRelayBody.
+const iCloudPrivateRelayAddr = "2001:db8:1::5"
+
+// iCloudPrivateRelayBody is a small file in the format of Apple's iCloud
+// Private Relay file.
+const iCloudPrivateRelayBody = "192.0.2.0/31,GB,GB-EN,London,\n" +
+	"198.51.100.32/27,US,US-CA,Los Angeles,\n" +
+	"2001:db8:1::/64,US,US-AK,Anchorage,\n"
+
+// emptyFileCase names the case of a wrong document that is an empty file.
+const emptyFileCase = "an empty file"
+
 // errorPage is a file that is no list, as a publisher's error page is.
 const errorPage = "<!DOCTYPE html>\n<html></html>\n"
 
@@ -243,6 +255,15 @@ func TestLookupFindsListedAddresses(t *testing.T) {
 			},
 			unlisted: []string{"192.0.2.12", "2001:db8::2"},
 		},
+		{
+			name:   "iCloud Private Relay networks",
+			format: FormatICloudPrivateRelayCSV,
+			body:   iCloudPrivateRelayBody,
+			listed: []string{
+				"192.0.2.1", "198.51.100.63", iCloudPrivateRelayAddr,
+			},
+			unlisted: []string{"192.0.2.2", "198.51.100.64", "2001:db8:2::1"},
+		},
 	})
 }
 
@@ -485,12 +506,12 @@ func TestWrongVPNDocumentFailsToParse(t *testing.T) {
 
 	for _, f := range formats {
 		bodies := map[string]string{
-			"not JSON":      errorPage,
-			"other JSON":    `{"status":"ok","data":{"count":1}}`,
-			"no addresses":  f.empty,
-			"bad address":   f.badAddress,
-			"cut off":       f.good[:len(f.good)/2],
-			"an empty file": "",
+			"not JSON":     errorPage,
+			"other JSON":   `{"status":"ok","data":{"count":1}}`,
+			"no addresses": f.empty,
+			"bad address":  f.badAddress,
+			"cut off":      f.good[:len(f.good)/2],
+			emptyFileCase:  "",
 		}
 
 		for name, body := range bodies {
@@ -516,7 +537,7 @@ func TestWrongProtonVPNDocumentFailsToParse(t *testing.T) {
 		"JSON":          `{"status":"ok"}`,
 		"comments only": "#\n# protonvpn_ip_ranges.txt\n#\n",
 		"bad network":   "192.0.2.0/24\nexample\n",
-		"an empty file": "",
+		emptyFileCase:   "",
 	}
 
 	for name, body := range bodies {
@@ -531,6 +552,34 @@ func TestWrongProtonVPNDocumentFailsToParse(t *testing.T) {
 	}
 }
 
+// TestWrongICloudPrivateRelayDocumentFailsToParse parses documents that are
+// not Apple's iCloud Private Relay file. Each fails, so none can replace a
+// good copy.
+func TestWrongICloudPrivateRelayDocumentFailsToParse(t *testing.T) {
+	t.Parallel()
+
+	bodies := map[string]string{
+		"error page":     errorPage,
+		"JSON":           `{"status":"ok","count":1}`,
+		"no other field": "192.0.2.0/24\n",
+		"bad network":    "192.0.2.0/24,GB,GB-EN,London,\nexample,GB,GB-EN,London,\n",
+		"an address":     "192.0.2.1,GB,GB-EN,London,\n",
+		emptyFileCase:    "",
+	}
+
+	for name, body := range bodies {
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			addresses, err := parse(strings.NewReader(body),
+				FormatICloudPrivateRelayCSV)
+			if err == nil {
+				t.Errorf("parse returned %v and no error", addresses.Ranges())
+			}
+		})
+	}
+}
+
 // TestLookupMarksVPNLists looks up an address on a VPN list and on another
 // list. Only the VPN list's listing says it is one.
 func TestLookupMarksVPNLists(t *testing.T) {
@@ -885,15 +934,33 @@ func TestStopCancelsADownloadInProgress(t *testing.T) {
 func TestTorExitsListHoldsIPv6Exits(t *testing.T) {
 	t.Parallel()
 
+	checkPublicListHolds(t, TorExitsList, onionooBody, onionooIPv6ExitAddr)
+}
+
+// TestICloudPrivateRelayListHoldsItsNetworks downloads a file in Apple's
+// format as the public iCloud Private Relay list, which is what an address's
+// is_relay answer comes from.
+func TestICloudPrivateRelayListHoldsItsNetworks(t *testing.T) {
+	t.Parallel()
+
+	checkPublicListHolds(t, ICloudPrivateRelayList, iCloudPrivateRelayBody,
+		iCloudPrivateRelayAddr)
+}
+
+// checkPublicListHolds downloads body as the public list called name and
+// checks that addr has one listing, which names that list.
+func checkPublicListHolds(t *testing.T, name, body, addr string) {
+	t.Helper()
+
 	index := slices.IndexFunc(Lists(), func(list List) bool {
-		return list.Name == TorExitsList
+		return list.Name == name
 	})
 	if index < 0 {
-		t.Fatalf("no list named %s", TorExitsList)
+		t.Fatalf("no list named %s", name)
 	}
 
 	list := Lists()[index]
-	_, list.URL = newPublisher(t, onionooBody)
+	_, list.URL = newPublisher(t, body)
 	b := newTestIPLists(t.TempDir(), list)
 
 	err := b.download(t.Context(), list)
@@ -901,9 +968,9 @@ func TestTorExitsListHoldsIPv6Exits(t *testing.T) {
 		t.Fatalf("download: %v", err)
 	}
 
-	listings := lookup(t, b, onionooIPv6ExitAddr)
-	if len(listings) != 1 || listings[0].Name != TorExitsList {
-		t.Errorf("listings %+v, want one for %s", listings, TorExitsList)
+	listings := lookup(t, b, addr)
+	if len(listings) != 1 || listings[0].Name != name {
+		t.Errorf("listings %+v, want one for %s", listings, name)
 	}
 }
 

internal/iplist/lists.go

@@ -8,6 +8,10 @@ import (
 // TorExitsList is the name of the list of Tor exit nodes.
 const TorExitsList = "tor_exits"
 
+// ICloudPrivateRelayList is the name of the list of iCloud Private Relay's
+// networks.
+const ICloudPrivateRelayList = "icloud_private_relay"
+
 // protonVPNIPsURL is where the ProtonVPN-IPs mirror publishes its files.
 const protonVPNIPsURL = "https://raw.githubusercontent.com/tn3w/ProtonVPN-IPs/master/"
 
@@ -30,6 +34,9 @@ const (
 	fireHOLAbusersInterval = 5 * time.Hour
 	fireHOLLevel3Interval  = 3 * time.Hour
 
+	// Apple's iCloud Private Relay file is 12 MB, so once a day.
+	iCloudPrivateRelayInterval = 24 * time.Hour
+
 	// Spamhaus asks that DROP be downloaded at most once a day:
 	// https://www.spamhaus.org/blocklists/do-not-route-or-peer/
 	spamhausDROPInterval = 24 * time.Hour
@@ -84,6 +91,13 @@ func nonVPNLists() []List {
 			Interval: fireHOLLevel3Interval,
 			Format:   FormatLines,
 		},
+		{
+			Name:        ICloudPrivateRelayList,
+			Description: "In a network where iCloud Private Relay traffic leaves",
+			URL:         "https://mask-api.icloud.com/egress-ip-ranges.csv",
+			Interval:    iCloudPrivateRelayInterval,
+			Format:      FormatICloudPrivateRelayCSV,
+		},
 		{
 			Name: "spamhaus_drop",
 			Description: "In an IPv4 network that Spamhaus says spammers " +

internal/iplist/parse.go

@@ -25,6 +25,10 @@ const (
 	// dshieldFieldSeparator separates the fields of a DShield line.
 	dshieldFieldSeparator = "\t"
 
+	// iCloudPrivateRelayFieldSeparator separates the fields of a line of
+	// Apple's iCloud Private Relay file.
+	iCloudPrivateRelayFieldSeparator = ","
+
 	// spamhausMetadataType is the type of the line that ends a Spamhaus
 	// file. That line holds the file's metadata and no network.
 	spamhausMetadataType = "metadata"
@@ -59,7 +63,9 @@ func parse(r io.Reader, format Format) (*netipx.IPSet, error) {
 	case FormatIVPNJSON:
 		return parseIVPN(r)
 	case FormatProtonVPN:
-		return parseProtonVPN(r)
+		return parseLinesNotEmpty(r, addAddressOrNetwork)
+	case FormatICloudPrivateRelayCSV:
+		return parseLinesNotEmpty(r, addICloudPrivateRelayNetwork)
 	default:
 		return nil, errUnknownFormat
 	}
@@ -331,11 +337,12 @@ func parseIVPN(r io.Reader) (*netipx.IPSet, error) {
 	return addressSet(texts)
 }
 
-// parseProtonVPN returns the addresses or networks in a ProtonVPN-IPs list. A
-// file with no addresses or networks fails, because the service always has
-// servers.
-func parseProtonVPN(r io.Reader) (*netipx.IPSet, error) {
-	addresses, err := parseLines(r, addAddressOrNetwork)
+// parseLinesNotEmpty is parseLines for a list that always holds addresses, so
+// a file with none fails.
+func parseLinesNotEmpty(
+	r io.Reader, addLine func(*netipx.IPSetBuilder, string) error,
+) (*netipx.IPSet, error) {
+	addresses, err := parseLines(r, addLine)
 	if err != nil {
 		return nil, err
 	}
@@ -347,6 +354,24 @@ func parseProtonVPN(r io.Reader) (*netipx.IPSet, error) {
 	return addresses, nil
 }
 
+// addICloudPrivateRelayNetwork adds the network at the start of a line of
+// Apple's iCloud Private Relay file to builder.
+func addICloudPrivateRelayNetwork(builder *netipx.IPSetBuilder, line string) error {
+	field, _, found := strings.Cut(line, iCloudPrivateRelayFieldSeparator)
+	if !found {
+		return errNoNetwork
+	}
+
+	network, err := netip.ParsePrefix(field)
+	if err != nil {
+		return err
+	}
+
+	builder.AddPrefix(network)
+
+	return nil
+}
+
 // readJSON reads r to its end and decodes it, as one JSON document, into v.
 func readJSON(r io.Reader, v any) error {
 	data, err := io.ReadAll(r)