From 2ee58665ffda2cc5d90fe6c67cf3284268765f61 Mon Sep 17 00:00:00 2001 From: sneak Date: Tue, 10 Nov 2026 09:55:47 +0000 Subject: [PATCH] Take the Tor exit list from Onionoo, which has the IPv6 exits (closes #234) check.torproject.org lists IPv4 exits only, so is_tor was false for an IPv6 exit. tor_exits now comes from Onionoo's details document, read by a new format that takes every address in exit_addresses and or_addresses. Model: opus-5-5 --- README.md | 5 +- TODO.md | 3 + internal/iplist/iplist.go | 5 + internal/iplist/iplist_internal_test.go | 144 +++++++++++++++++++++--- internal/iplist/lists.go | 12 +- internal/iplist/parse.go | 115 ++++++++++++++++--- 6 files changed, 245 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index db4e730..6336f23 100644 --- a/README.md +++ b/README.md @@ -424,8 +424,9 @@ The lists: leased by spammers or cyber criminals, according to Spamhaus. From [Spamhaus DROP](https://www.spamhaus.org/blocklists/do-not-route-or-peer/), downloaded once a day. -- `tor_exits`: Tor exit nodes, where traffic from the Tor network leaves. From - the [Tor Project](https://check.torproject.org/torbulkexitlist), downloaded +- `tor_exits`: the IPv4 and IPv6 addresses of Tor exit nodes, where traffic from + the Tor network leaves. From the Tor Project's + [Onionoo](https://metrics.torproject.org/onionoo.html) service, downloaded every hour. No list is downloaded more often than its publisher updates it or allows. Each diff --git a/TODO.md b/TODO.md index 39f3c2e..38ff071 100644 --- a/TODO.md +++ b/TODO.md @@ -53,6 +53,9 @@ problem the run finds. # Completed Steps +- 2026-11-10: `tor_exits` comes from Onionoo, so it holds IPv6 exits and + `is_tor` is true for them (closes + https://git.eeqj.de/sneak/routewatch/issues/234). - 2026-11-10: each result of the IP lookup has a `flags` object with `is_tor` and `is_bogon`. The public IP lists are named `ip_lists` in the answer, `internal/iplist` in the code and `iplists` in the state directory. A list diff --git a/internal/iplist/iplist.go b/internal/iplist/iplist.go index 1d47bc2..4b74dae 100644 --- a/internal/iplist/iplist.go +++ b/internal/iplist/iplist.go @@ -29,6 +29,11 @@ const ( // start with the network's first and last address. Lines that start // with # are comments. FormatDShield + + // FormatOnionooJSON is one Onionoo details document. Its relays have + // addresses in two fields: exit_addresses, which has IPv4 only, and + // or_addresses, which has IPv4 and IPv6, each with a port. + FormatOnionooJSON ) // List is one public IP list. diff --git a/internal/iplist/iplist_internal_test.go b/internal/iplist/iplist_internal_test.go index 6e626f7..f7565f2 100644 --- a/internal/iplist/iplist_internal_test.go +++ b/internal/iplist/iplist_internal_test.go @@ -11,6 +11,7 @@ import ( "net/netip" "os" "path/filepath" + "slices" "strings" "sync" "testing" @@ -28,6 +29,20 @@ const ( newListBody = "# A test list\n" + newListedAddr + "\n" ) +// A small Onionoo details document: a relay with an exit address and IPv4 and +// IPv6 relay addresses, one without exit addresses, and one with a list of +// ports. +const ( + onionooExitAddr = "192.0.2.10" + onionooIPv6ExitAddr = "2001:db8::1" + onionooBody = `{"version":"8.0","relays":[ + {"exit_addresses":["` + onionooExitAddr + `"], + "or_addresses":["192.0.2.11:443","[` + onionooIPv6ExitAddr + `]:9001"]}, + {"or_addresses":["192.0.2.20:443"]}, + {"or_addresses":["192.0.2.30:80,443"]} + ],"bridges":[]}` +) + // testETag is the ETag a publisher sends with a version of the test list. const testETag = `"v1"` @@ -181,6 +196,16 @@ func TestLookupFindsListedAddresses(t *testing.T) { listed: []string{"192.0.2.0", "192.0.2.77"}, unlisted: []string{"192.0.3.1", "2001:db8::1"}, }, + { + name: "Onionoo relays", + format: FormatOnionooJSON, + body: onionooBody, + listed: []string{ + onionooExitAddr, "192.0.2.11", "192.0.2.20", "192.0.2.30", + onionooIPv6ExitAddr, + }, + unlisted: []string{"192.0.2.12", "2001:db8::2"}, + }, } for _, tt := range tests { @@ -195,26 +220,34 @@ func TestLookupFindsListedAddresses(t *testing.T) { t.Fatalf("download: %v", err) } - for _, addr := range tt.listed { - listings := lookup(t, b, addr) - if len(listings) != 1 || listings[0].Name != testListName || - listings[0].Description != testListDescription || - listings[0].Updated.IsZero() { - t.Errorf("%s: listings %+v, want one for %s", addr, - listings, testListName) - } - } - - for _, addr := range tt.unlisted { - listings := lookup(t, b, addr) - if listings == nil || len(listings) != 0 { - t.Errorf("%s: listings %#v, want an empty list", addr, listings) - } - } + checkLookups(t, b, tt.listed, tt.unlisted) }) } } +// checkLookups checks that each of listed has one listing that names the test +// list, and each of unlisted has an empty list of listings. +func checkLookups(t *testing.T, b *IPLists, listed, unlisted []string) { + t.Helper() + + for _, addr := range listed { + listings := lookup(t, b, addr) + if len(listings) != 1 || listings[0].Name != testListName || + listings[0].Description != testListDescription || + listings[0].Updated.IsZero() { + t.Errorf("%s: listings %+v, want one for %s", addr, + listings, testListName) + } + } + + for _, addr := range unlisted { + listings := lookup(t, b, addr) + if listings == nil || len(listings) != 0 { + t.Errorf("%s: listings %#v, want an empty list", addr, listings) + } + } +} + // TestFailedDownloadKeepsThePreviousCopy downloads a good version of a list, // then one that fails. The good copy stays in memory and on disk, and no // part of the failed download is left beside it. @@ -263,6 +296,58 @@ func TestFailedDownloadKeepsThePreviousCopy(t *testing.T) { } } +// TestFileThatIsNotAnOnionooDocumentKeepsThePreviousCopy downloads a good +// Onionoo document, then files that are not one. Each fails the parse, and +// the good copy stays. +func TestFileThatIsNotAnOnionooDocumentKeepsThePreviousCopy(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + body string + }{ + {"not JSON", "\n\n"}, + {"other JSON", `{"status":"ok","data":{"relays_count":1}}`}, + {"JSON array", `[{"or_addresses":["192.0.2.1:443"]}]`}, + {"no relays", `{"relays":[],"bridges":[]}`}, + {"relays without addresses", `{"relays":[{"nickname":"example"}]}`}, + {"relay address without a port", `{"relays":[{"or_addresses":["192.0.2.1"]}]}`}, + {"exit address that is not one", `{"relays":[{"exit_addresses":["example"]}]}`}, + {"text after the document", `{"relays":[{"or_addresses":["192.0.2.1:1"]}]} x`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + p, url := newPublisher(t, onionooBody) + b := newTestIPLists(dir, testList(url, FormatOnionooJSON)) + + err := b.download(t.Context(), b.lists[0]) + if err != nil { + t.Fatalf("first download: %v", err) + } + + p.answer(http.StatusOK, tt.body) + + err = b.download(t.Context(), b.lists[0]) + if err == nil { + t.Fatal("second download returned no error") + } + + if len(lookup(t, b, onionooExitAddr)) != 1 { + t.Errorf("%s is no longer listed", onionooExitAddr) + } + + onDisk, err := fs.ReadFile(os.DirFS(dir), testListName) + if err != nil || string(onDisk) != onionooBody { + t.Errorf("copy on disk %q (%v), want the good version", onDisk, err) + } + }) + } +} + // checkGoodCopyOnDisk checks that dir holds the good version of the test list // and no part of a failed download. func checkGoodCopyOnDisk(t *testing.T, dir string) { @@ -555,6 +640,33 @@ func TestStopCancelsADownloadInProgress(t *testing.T) { } } +// TestTorExitsListHoldsIPv6Exits downloads an Onionoo document as the public +// Tor exit list, which is what an IPv6 address's is_tor answer comes from. +func TestTorExitsListHoldsIPv6Exits(t *testing.T) { + t.Parallel() + + index := slices.IndexFunc(Lists(), func(list List) bool { + return list.Name == TorExitsList + }) + if index < 0 { + t.Fatalf("no list named %s", TorExitsList) + } + + list := Lists()[index] + _, list.URL = newPublisher(t, onionooBody) + b := newTestIPLists(t.TempDir(), list) + + err := b.download(t.Context(), list) + if err != nil { + t.Fatalf("download: %v", err) + } + + listings := lookup(t, b, onionooIPv6ExitAddr) + if len(listings) != 1 || listings[0].Name != TorExitsList { + t.Errorf("listings %+v, want one for %s", listings, TorExitsList) + } +} + // TestPublicListsAreComplete checks that every public list has what a // download and a lookup answer need, and a unique name. func TestPublicListsAreComplete(t *testing.T) { diff --git a/internal/iplist/lists.go b/internal/iplist/lists.go index 37e31c9..2441a49 100644 --- a/internal/iplist/lists.go +++ b/internal/iplist/lists.go @@ -28,8 +28,9 @@ const ( // https://www.spamhaus.org/blocklists/do-not-route-or-peer/ spamhausDROPInterval = 24 * time.Hour - // Tor publishes a new exit list every hour and sets no download limit: - // https://collector.torproject.org/recent/exit-lists/ + // Onionoo's relay data follows Tor's hourly network consensus, and + // Onionoo sets no download limit: + // https://metrics.torproject.org/onionoo.html torExitsInterval = time.Hour ) @@ -88,9 +89,10 @@ func Lists() []List { { Name: TorExitsList, Description: "A Tor exit node, where traffic from the Tor network leaves", - URL: "https://check.torproject.org/torbulkexitlist", - Interval: torExitsInterval, - Format: FormatLines, + URL: "https://onionoo.torproject.org/details?" + + "flag=Exit&running=true&fields=exit_addresses,or_addresses", + Interval: torExitsInterval, + Format: FormatOnionooJSON, }, } } diff --git a/internal/iplist/parse.go b/internal/iplist/parse.go index dc872f4..4683212 100644 --- a/internal/iplist/parse.go +++ b/internal/iplist/parse.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "net" "net/netip" "strings" @@ -32,12 +33,32 @@ var ( errUnknownFormat = errors.New("unknown list format") errNoNetwork = errors.New("no network on the line") errBadRange = errors.New("first and last address make no range") + errNoAddresses = errors.New("no relay addresses in the document") ) // parse reads a list file written in format and returns the addresses it -// holds. A line it cannot read fails the whole file, so a file that is not +// holds. Anything it cannot read fails the whole file, so a file that is not // the list, such as an error page, never replaces a good copy. func parse(r io.Reader, format Format) (*netipx.IPSet, error) { + switch format { + case FormatLines: + return parseLines(r, addAddressOrNetwork) + case FormatSpamhausJSON: + return parseLines(r, addSpamhausNetwork) + case FormatDShield: + return parseLines(r, addDShieldNetwork) + case FormatOnionooJSON: + return parseOnionoo(r) + default: + return nil, errUnknownFormat + } +} + +// parseLines reads a list file line by line, skipping blank lines and +// comments, and calls addLine with each other line. +func parseLines( + r io.Reader, addLine func(*netipx.IPSetBuilder, string) error, +) (*netipx.IPSet, error) { var builder netipx.IPSetBuilder scanner := bufio.NewScanner(r) @@ -48,7 +69,7 @@ func parse(r io.Reader, format Format) (*netipx.IPSet, error) { continue } - err := addLine(&builder, line, format) + err := addLine(&builder, line) if err != nil { return nil, fmt.Errorf("line %d: %w", lineNumber, err) } @@ -62,20 +83,6 @@ func parse(r io.Reader, format Format) (*netipx.IPSet, error) { return builder.IPSet() } -// addLine adds the address or network on one line of a list file to builder. -func addLine(builder *netipx.IPSetBuilder, line string, format Format) error { - switch format { - case FormatLines: - return addAddressOrNetwork(builder, line) - case FormatSpamhausJSON: - return addSpamhausNetwork(builder, line) - case FormatDShield: - return addDShieldNetwork(builder, line) - default: - return errUnknownFormat - } -} - // addAddressOrNetwork adds an address, such as 192.0.2.1, or a network, such // as 192.0.2.0/24, to builder. func addAddressOrNetwork(builder *netipx.IPSetBuilder, text string) error { @@ -156,3 +163,79 @@ func addDShieldNetwork(builder *netipx.IPSetBuilder, line string) error { return nil } + +// parseOnionoo returns every address in the exit_addresses and or_addresses +// fields of the relays in an Onionoo details document. A document with no +// addresses fails, because the Tor network always has relays. +func parseOnionoo(r io.Reader) (*netipx.IPSet, error) { + data, err := io.ReadAll(r) + if err != nil { + return nil, err + } + + //nolint:tagliatelle // Onionoo's own field names + var details struct { + Relays []struct { + ExitAddresses []string `json:"exit_addresses"` + ORAddresses []string `json:"or_addresses"` + } `json:"relays"` + } + + err = json.Unmarshal(data, &details) + if err != nil { + return nil, err + } + + var builder netipx.IPSetBuilder + + for _, relay := range details.Relays { + for _, text := range relay.ExitAddresses { + err = addAddress(&builder, text) + if err != nil { + return nil, err + } + } + + for _, text := range relay.ORAddresses { + err = addAddressWithPort(&builder, text) + if err != nil { + return nil, err + } + } + } + + addresses, err := builder.IPSet() + if err != nil { + return nil, err + } + + if len(addresses.Ranges()) == 0 { + return nil, errNoAddresses + } + + return addresses, nil +} + +// addAddress adds an address, such as 192.0.2.1, to builder. +func addAddress(builder *netipx.IPSetBuilder, text string) error { + addr, err := netip.ParseAddr(text) + if err != nil { + return err + } + + builder.Add(addr) + + return nil +} + +// addAddressWithPort adds the address of text, such as 192.0.2.1:443 or +// [2001:db8::1]:443, to builder. Onionoo allows a list of ports after the +// address, so the part after the address is not read. +func addAddressWithPort(builder *netipx.IPSetBuilder, text string) error { + host, _, err := net.SplitHostPort(text) + if err != nil { + return err + } + + return addAddress(builder, host) +} -- 2.54.0