diff --git a/README.md b/README.md index 47eae62..9202f5b 100644 --- a/README.md +++ b/README.md @@ -287,6 +287,7 @@ address. Every entry has the same fields: - `is_tor`: true when the address is on the `tor_exits` list - `is_vpn`: true when the address is on any list of VPN servers (see IP lists) + - `is_relay`: true when the address is on the `icloud_private_relay` list - `is_bogon`: true when the address is reserved, so that no network should announce it on the internet - `vpn_providers`: the names of the lists of VPN servers the address is on, or @@ -421,6 +422,9 @@ The lists: last 30 days. From [FireHOL](https://iplists.firehol.org/?ipset=firehol_level3), downloaded every 3 hours. +- `icloud_private_relay`: the networks where iCloud Private Relay traffic + leaves. From [Apple](https://mask-api.icloud.com/egress-ip-ranges.csv), + downloaded once a day. - `ivpn`: VPN servers run by IVPN. From [IVPN](https://api.ivpn.net/v5/servers.json), downloaded once a day. - `mullvad`: VPN servers run by Mullvad. From @@ -606,6 +610,13 @@ The ceilings add up to 3.7 GiB: limit caps the SQLite library. - About 0.2 GiB for everything else in the runtime. +The `icloud_private_relay` list has about 285,000 networks. Neighbouring +networks merge, which leaves about 6,900 address ranges. Loaded, the list adds +about 0.3 MiB to the Go heap and under 6 MiB to the process's resident memory. +Parsing the file allocates about 94 MiB in about 0.2 seconds. The next garbage +collection frees all of it but the 0.3 MiB above. Parsing happens at startup and +whenever a download brings a new version. + The SQLite library gets its memory from glibc, the C library. glibc keeps freed memory in pools called arenas, and hands it back to the kernel slowly. It creates up to eight arenas per processor core. So on a host with many cores the diff --git a/TODO.md b/TODO.md index 7cf65ff..6f84762 100644 --- a/TODO.md +++ b/TODO.md @@ -53,6 +53,9 @@ problem the run finds. # Completed Steps +- 2026-11-10: the IP lookup checks the `icloud_private_relay` list of Apple's + iCloud Private Relay networks, downloaded once a day. Each result has + `flags.is_relay` (closes https://git.eeqj.de/sneak/routewatch/issues/237). - 2026-11-10: the IP lookup checks the `protonvpn` list of ProtonVPN's /24 networks and the `protonvpn_entry` list of its servers' IPv4 and IPv6 addresses, from a public mirror of Proton's server list (closes diff --git a/internal/iplist/download.go b/internal/iplist/download.go index 0d79d07..0133417 100644 --- a/internal/iplist/download.go +++ b/internal/iplist/download.go @@ -21,7 +21,7 @@ const ( downloadTimeout = time.Minute // maxDownloadSize is the size of the largest list file accepted, in - // bytes. The largest list, FireHOL abusers, is about 2.5 MiB. + // bytes. The largest list, iCloud Private Relay, is about 12 MiB. maxDownloadSize = 200 << 20 ) diff --git a/internal/iplist/iplist.go b/internal/iplist/iplist.go index 9795665..73b826e 100644 --- a/internal/iplist/iplist.go +++ b/internal/iplist/iplist.go @@ -56,6 +56,11 @@ const ( // that start with #. Unlike FormatLines, a file with no addresses or // networks fails. FormatProtonVPN + + // FormatICloudPrivateRelayCSV is Apple's CSV file of iCloud Private Relay + // networks. Each line starts with a network, followed by its country, + // region and city. + FormatICloudPrivateRelayCSV ) // List is one public IP list. diff --git a/internal/iplist/iplist_internal_test.go b/internal/iplist/iplist_internal_test.go index 3b7576b..9fa7891 100644 --- a/internal/iplist/iplist_internal_test.go +++ b/internal/iplist/iplist_internal_test.go @@ -83,6 +83,18 @@ const protonVPNEntryBody = "#\n# protonvpn_entry_ips.txt\n" + "# Entry IPs clients connect to, resolved from ProtonVPN hostnames.\n#\n" + "192.0.2.80\n2001:db8::80\n" +// iCloudPrivateRelayAddr is an address on iCloudPrivateRelayBody. +const iCloudPrivateRelayAddr = "2001:db8:1::5" + +// iCloudPrivateRelayBody is a small file in the format of Apple's iCloud +// Private Relay file. +const iCloudPrivateRelayBody = "192.0.2.0/31,GB,GB-EN,London,\n" + + "198.51.100.32/27,US,US-CA,Los Angeles,\n" + + "2001:db8:1::/64,US,US-AK,Anchorage,\n" + +// emptyFileCase names the case of a wrong document that is an empty file. +const emptyFileCase = "an empty file" + // errorPage is a file that is no list, as a publisher's error page is. const errorPage = "\n\n" @@ -243,6 +255,15 @@ func TestLookupFindsListedAddresses(t *testing.T) { }, unlisted: []string{"192.0.2.12", "2001:db8::2"}, }, + { + name: "iCloud Private Relay networks", + format: FormatICloudPrivateRelayCSV, + body: iCloudPrivateRelayBody, + listed: []string{ + "192.0.2.1", "198.51.100.63", iCloudPrivateRelayAddr, + }, + unlisted: []string{"192.0.2.2", "198.51.100.64", "2001:db8:2::1"}, + }, }) } @@ -485,12 +506,12 @@ func TestWrongVPNDocumentFailsToParse(t *testing.T) { for _, f := range formats { bodies := map[string]string{ - "not JSON": errorPage, - "other JSON": `{"status":"ok","data":{"count":1}}`, - "no addresses": f.empty, - "bad address": f.badAddress, - "cut off": f.good[:len(f.good)/2], - "an empty file": "", + "not JSON": errorPage, + "other JSON": `{"status":"ok","data":{"count":1}}`, + "no addresses": f.empty, + "bad address": f.badAddress, + "cut off": f.good[:len(f.good)/2], + emptyFileCase: "", } for name, body := range bodies { @@ -516,7 +537,7 @@ func TestWrongProtonVPNDocumentFailsToParse(t *testing.T) { "JSON": `{"status":"ok"}`, "comments only": "#\n# protonvpn_ip_ranges.txt\n#\n", "bad network": "192.0.2.0/24\nexample\n", - "an empty file": "", + emptyFileCase: "", } for name, body := range bodies { @@ -531,6 +552,34 @@ func TestWrongProtonVPNDocumentFailsToParse(t *testing.T) { } } +// TestWrongICloudPrivateRelayDocumentFailsToParse parses documents that are +// not Apple's iCloud Private Relay file. Each fails, so none can replace a +// good copy. +func TestWrongICloudPrivateRelayDocumentFailsToParse(t *testing.T) { + t.Parallel() + + bodies := map[string]string{ + "error page": errorPage, + "JSON": `{"status":"ok","count":1}`, + "no other field": "192.0.2.0/24\n", + "bad network": "192.0.2.0/24,GB,GB-EN,London,\nexample,GB,GB-EN,London,\n", + "an address": "192.0.2.1,GB,GB-EN,London,\n", + emptyFileCase: "", + } + + for name, body := range bodies { + t.Run(name, func(t *testing.T) { + t.Parallel() + + addresses, err := parse(strings.NewReader(body), + FormatICloudPrivateRelayCSV) + if err == nil { + t.Errorf("parse returned %v and no error", addresses.Ranges()) + } + }) + } +} + // TestLookupMarksVPNLists looks up an address on a VPN list and on another // list. Only the VPN list's listing says it is one. func TestLookupMarksVPNLists(t *testing.T) { @@ -885,15 +934,33 @@ func TestStopCancelsADownloadInProgress(t *testing.T) { func TestTorExitsListHoldsIPv6Exits(t *testing.T) { t.Parallel() + checkPublicListHolds(t, TorExitsList, onionooBody, onionooIPv6ExitAddr) +} + +// TestICloudPrivateRelayListHoldsItsNetworks downloads a file in Apple's +// format as the public iCloud Private Relay list, which is what an address's +// is_relay answer comes from. +func TestICloudPrivateRelayListHoldsItsNetworks(t *testing.T) { + t.Parallel() + + checkPublicListHolds(t, ICloudPrivateRelayList, iCloudPrivateRelayBody, + iCloudPrivateRelayAddr) +} + +// checkPublicListHolds downloads body as the public list called name and +// checks that addr has one listing, which names that list. +func checkPublicListHolds(t *testing.T, name, body, addr string) { + t.Helper() + index := slices.IndexFunc(Lists(), func(list List) bool { - return list.Name == TorExitsList + return list.Name == name }) if index < 0 { - t.Fatalf("no list named %s", TorExitsList) + t.Fatalf("no list named %s", name) } list := Lists()[index] - _, list.URL = newPublisher(t, onionooBody) + _, list.URL = newPublisher(t, body) b := newTestIPLists(t.TempDir(), list) err := b.download(t.Context(), list) @@ -901,9 +968,9 @@ func TestTorExitsListHoldsIPv6Exits(t *testing.T) { t.Fatalf("download: %v", err) } - listings := lookup(t, b, onionooIPv6ExitAddr) - if len(listings) != 1 || listings[0].Name != TorExitsList { - t.Errorf("listings %+v, want one for %s", listings, TorExitsList) + listings := lookup(t, b, addr) + if len(listings) != 1 || listings[0].Name != name { + t.Errorf("listings %+v, want one for %s", listings, name) } } diff --git a/internal/iplist/lists.go b/internal/iplist/lists.go index 7990378..acdaa6b 100644 --- a/internal/iplist/lists.go +++ b/internal/iplist/lists.go @@ -8,6 +8,10 @@ import ( // TorExitsList is the name of the list of Tor exit nodes. const TorExitsList = "tor_exits" +// ICloudPrivateRelayList is the name of the list of iCloud Private Relay's +// networks. +const ICloudPrivateRelayList = "icloud_private_relay" + // protonVPNIPsURL is where the ProtonVPN-IPs mirror publishes its files. const protonVPNIPsURL = "https://raw.githubusercontent.com/tn3w/ProtonVPN-IPs/master/" @@ -30,6 +34,9 @@ const ( fireHOLAbusersInterval = 5 * time.Hour fireHOLLevel3Interval = 3 * time.Hour + // Apple's iCloud Private Relay file is 12 MB, so once a day. + iCloudPrivateRelayInterval = 24 * time.Hour + // Spamhaus asks that DROP be downloaded at most once a day: // https://www.spamhaus.org/blocklists/do-not-route-or-peer/ spamhausDROPInterval = 24 * time.Hour @@ -84,6 +91,13 @@ func nonVPNLists() []List { Interval: fireHOLLevel3Interval, Format: FormatLines, }, + { + Name: ICloudPrivateRelayList, + Description: "In a network where iCloud Private Relay traffic leaves", + URL: "https://mask-api.icloud.com/egress-ip-ranges.csv", + Interval: iCloudPrivateRelayInterval, + Format: FormatICloudPrivateRelayCSV, + }, { Name: "spamhaus_drop", Description: "In an IPv4 network that Spamhaus says spammers " + diff --git a/internal/iplist/parse.go b/internal/iplist/parse.go index 7209427..fba2ca4 100644 --- a/internal/iplist/parse.go +++ b/internal/iplist/parse.go @@ -25,6 +25,10 @@ const ( // dshieldFieldSeparator separates the fields of a DShield line. dshieldFieldSeparator = "\t" + // iCloudPrivateRelayFieldSeparator separates the fields of a line of + // Apple's iCloud Private Relay file. + iCloudPrivateRelayFieldSeparator = "," + // spamhausMetadataType is the type of the line that ends a Spamhaus // file. That line holds the file's metadata and no network. spamhausMetadataType = "metadata" @@ -59,7 +63,9 @@ func parse(r io.Reader, format Format) (*netipx.IPSet, error) { case FormatIVPNJSON: return parseIVPN(r) case FormatProtonVPN: - return parseProtonVPN(r) + return parseLinesNotEmpty(r, addAddressOrNetwork) + case FormatICloudPrivateRelayCSV: + return parseLinesNotEmpty(r, addICloudPrivateRelayNetwork) default: return nil, errUnknownFormat } @@ -331,11 +337,12 @@ func parseIVPN(r io.Reader) (*netipx.IPSet, error) { return addressSet(texts) } -// parseProtonVPN returns the addresses or networks in a ProtonVPN-IPs list. A -// file with no addresses or networks fails, because the service always has -// servers. -func parseProtonVPN(r io.Reader) (*netipx.IPSet, error) { - addresses, err := parseLines(r, addAddressOrNetwork) +// parseLinesNotEmpty is parseLines for a list that always holds addresses, so +// a file with none fails. +func parseLinesNotEmpty( + r io.Reader, addLine func(*netipx.IPSetBuilder, string) error, +) (*netipx.IPSet, error) { + addresses, err := parseLines(r, addLine) if err != nil { return nil, err } @@ -347,6 +354,24 @@ func parseProtonVPN(r io.Reader) (*netipx.IPSet, error) { return addresses, nil } +// addICloudPrivateRelayNetwork adds the network at the start of a line of +// Apple's iCloud Private Relay file to builder. +func addICloudPrivateRelayNetwork(builder *netipx.IPSetBuilder, line string) error { + field, _, found := strings.Cut(line, iCloudPrivateRelayFieldSeparator) + if !found { + return errNoNetwork + } + + network, err := netip.ParsePrefix(field) + if err != nil { + return err + } + + builder.AddPrefix(network) + + return nil +} + // readJSON reads r to its end and decodes it, as one JSON document, into v. func readJSON(r io.Reader, v any) error { data, err := io.ReadAll(r) diff --git a/internal/server/handlers.go b/internal/server/handlers.go index 01d4509..3b1291c 100644 --- a/internal/server/handlers.go +++ b/internal/server/handlers.go @@ -797,6 +797,9 @@ type IPLookupFlags struct { // IsVPN says that the address is on a list of VPN servers. IsVPN bool `json:"is_vpn"` + // IsRelay says that the address is on the iCloud Private Relay list. + IsRelay bool `json:"is_relay"` + // IsBogon says that the address is reserved, as package bogon tells. IsBogon bool `json:"is_bogon"` } @@ -1025,6 +1028,9 @@ func ipLookupFlags(ip string, listings []iplist.Listing) IPLookupFlags { IsVPN: slices.ContainsFunc(listings, func(l iplist.Listing) bool { return l.VPN }), + IsRelay: slices.ContainsFunc(listings, func(l iplist.Listing) bool { + return l.Name == iplist.ICloudPrivateRelayList + }), IsBogon: isBogon(ip), } } diff --git a/internal/server/handlers_internal_test.go b/internal/server/handlers_internal_test.go index a1898b8..cdb9258 100644 --- a/internal/server/handlers_internal_test.go +++ b/internal/server/handlers_internal_test.go @@ -1504,6 +1504,10 @@ const listedUnroutedAddr = "203.0.113.1" // torExitAddr is the address that fakeIPLists lists on the Tor exit list. const torExitAddr = "203.0.113.2" +// relayAddr is the address that fakeIPLists lists on the iCloud Private Relay +// list. +const relayAddr = "203.0.113.5" + // The addresses that fakeIPLists lists on one and on two VPN lists, and the // names of those lists. const ( @@ -1513,10 +1517,11 @@ const ( otherTestVPN = "other_test_vpn" ) -// fakeIPLists has four lists. The test list holds addrInTestPrefixV4, +// fakeIPLists has five lists. The test list holds addrInTestPrefixV4, // listedUnroutedAddr and twoVPNListsAddr. The Tor exit list holds torExitAddr. -// The test VPN list holds oneVPNListAddr and twoVPNListsAddr, and the other -// test VPN list holds twoVPNListsAddr. +// The iCloud Private Relay list holds relayAddr. The test VPN list holds +// oneVPNListAddr and twoVPNListsAddr, and the other test VPN list holds +// twoVPNListsAddr. type fakeIPLists struct{} func testListing() iplist.Listing { @@ -1537,6 +1542,10 @@ func (fakeIPLists) Lookup(addr netip.Addr) []iplist.Listing { return []iplist.Listing{{Name: iplist.TorExitsList}} } + if addr == netip.MustParseAddr(relayAddr) { + return []iplist.Listing{{Name: iplist.ICloudPrivateRelayList}} + } + if addr == netip.MustParseAddr(oneVPNListAddr) { return []iplist.Listing{{Name: testVPNList, VPN: true}} } @@ -1569,20 +1578,22 @@ func TestIPLookupFlags(t *testing.T) { name string addr string tor bool + relay bool bogon bool }{ - {"on the Tor exit list", torExitAddr, true, true}, - {"on another list", listedUnroutedAddr, false, true}, - {"routed IPv4", addrInTestPrefixV4, false, true}, - {"routed IPv6", addrInTestPrefixV6, false, true}, - {"private IPv4", "10.0.0.1", false, true}, - {"loopback IPv4", "127.0.0.1", false, true}, - {"public IPv4", "1.1.1.1", false, false}, - {"private IPv6", "fd00::1", false, true}, - {"loopback IPv6", "::1", false, true}, - {"public IPv6", "2606:4700:4700::1111", false, false}, - {"public IPv4 in IPv6 form", "::ffff:1.1.1.1", false, false}, - {"private IPv4 in IPv6 form", "::ffff:10.0.0.1", false, true}, + {"on the Tor exit list", torExitAddr, true, false, true}, + {"on the iCloud Private Relay list", relayAddr, false, true, true}, + {"on another list", listedUnroutedAddr, false, false, true}, + {"routed IPv4", addrInTestPrefixV4, false, false, true}, + {"routed IPv6", addrInTestPrefixV6, false, false, true}, + {"private IPv4", "10.0.0.1", false, false, true}, + {"loopback IPv4", "127.0.0.1", false, false, true}, + {"public IPv4", "1.1.1.1", false, false, false}, + {"private IPv6", "fd00::1", false, false, true}, + {"loopback IPv6", "::1", false, false, true}, + {"public IPv6", "2606:4700:4700::1111", false, false, false}, + {"public IPv4 in IPv6 form", "::ffff:1.1.1.1", false, false, false}, + {"private IPv4 in IPv6 form", "::ffff:10.0.0.1", false, false, true}, } for _, tt := range tests { @@ -1604,7 +1615,8 @@ func TestIPLookupFlags(t *testing.T) { } want := map[string]bool{ - "is_tor": tt.tor, "is_vpn": false, "is_bogon": tt.bogon, + "is_tor": tt.tor, "is_vpn": false, "is_relay": tt.relay, + "is_bogon": tt.bogon, } if !maps.Equal(got, want) { t.Errorf("%s: flags %v, want %v", path, got, want)