Take the Tor exit list from Onionoo, which has the IPv6 exits (closes #234)

check.torproject.org lists IPv4 exits only, so is_tor was false for an IPv6
exit. tor_exits now comes from Onionoo's details document, read by a new
format that takes every address in exit_addresses and or_addresses.

Model: opus-5-5

internal/iplist/iplist_internal_test.go

@@ -11,6 +11,7 @@ import (
 	"net/netip"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"sync"
 	"testing"
@@ -28,6 +29,20 @@ const (
 	newListBody         = "# A test list\n" + newListedAddr + "\n"
 )
 
+// A small Onionoo details document: a relay with an exit address and IPv4 and
+// IPv6 relay addresses, one without exit addresses, and one with a list of
+// ports.
+const (
+	onionooExitAddr     = "192.0.2.10"
+	onionooIPv6ExitAddr = "2001:db8::1"
+	onionooBody         = `{"version":"8.0","relays":[
+		{"exit_addresses":["` + onionooExitAddr + `"],
+		 "or_addresses":["192.0.2.11:443","[` + onionooIPv6ExitAddr + `]:9001"]},
+		{"or_addresses":["192.0.2.20:443"]},
+		{"or_addresses":["192.0.2.30:80,443"]}
+	],"bridges":[]}`
+)
+
 // testETag is the ETag a publisher sends with a version of the test list.
 const testETag = `"v1"`
 
@@ -181,6 +196,16 @@ func TestLookupFindsListedAddresses(t *testing.T) {
 			listed:   []string{"192.0.2.0", "192.0.2.77"},
 			unlisted: []string{"192.0.3.1", "2001:db8::1"},
 		},
+		{
+			name:   "Onionoo relays",
+			format: FormatOnionooJSON,
+			body:   onionooBody,
+			listed: []string{
+				onionooExitAddr, "192.0.2.11", "192.0.2.20", "192.0.2.30",
+				onionooIPv6ExitAddr,
+			},
+			unlisted: []string{"192.0.2.12", "2001:db8::2"},
+		},
 	}
 
 	for _, tt := range tests {
@@ -195,26 +220,34 @@ func TestLookupFindsListedAddresses(t *testing.T) {
 				t.Fatalf("download: %v", err)
 			}
 
-			for _, addr := range tt.listed {
-				listings := lookup(t, b, addr)
-				if len(listings) != 1 || listings[0].Name != testListName ||
-					listings[0].Description != testListDescription ||
-					listings[0].Updated.IsZero() {
-					t.Errorf("%s: listings %+v, want one for %s", addr,
-						listings, testListName)
-				}
-			}
-
-			for _, addr := range tt.unlisted {
-				listings := lookup(t, b, addr)
-				if listings == nil || len(listings) != 0 {
-					t.Errorf("%s: listings %#v, want an empty list", addr, listings)
-				}
-			}
+			checkLookups(t, b, tt.listed, tt.unlisted)
 		})
 	}
 }
 
+// checkLookups checks that each of listed has one listing that names the test
+// list, and each of unlisted has an empty list of listings.
+func checkLookups(t *testing.T, b *IPLists, listed, unlisted []string) {
+	t.Helper()
+
+	for _, addr := range listed {
+		listings := lookup(t, b, addr)
+		if len(listings) != 1 || listings[0].Name != testListName ||
+			listings[0].Description != testListDescription ||
+			listings[0].Updated.IsZero() {
+			t.Errorf("%s: listings %+v, want one for %s", addr,
+				listings, testListName)
+		}
+	}
+
+	for _, addr := range unlisted {
+		listings := lookup(t, b, addr)
+		if listings == nil || len(listings) != 0 {
+			t.Errorf("%s: listings %#v, want an empty list", addr, listings)
+		}
+	}
+}
+
 // TestFailedDownloadKeepsThePreviousCopy downloads a good version of a list,
 // then one that fails. The good copy stays in memory and on disk, and no
 // part of the failed download is left beside it.
@@ -263,6 +296,58 @@ func TestFailedDownloadKeepsThePreviousCopy(t *testing.T) {
 	}
 }
 
+// TestFileThatIsNotAnOnionooDocumentKeepsThePreviousCopy downloads a good
+// Onionoo document, then files that are not one. Each fails the parse, and
+// the good copy stays.
+func TestFileThatIsNotAnOnionooDocumentKeepsThePreviousCopy(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name string
+		body string
+	}{
+		{"not JSON", "<!DOCTYPE html>\n<html></html>\n"},
+		{"other JSON", `{"status":"ok","data":{"relays_count":1}}`},
+		{"JSON array", `[{"or_addresses":["192.0.2.1:443"]}]`},
+		{"no relays", `{"relays":[],"bridges":[]}`},
+		{"relays without addresses", `{"relays":[{"nickname":"example"}]}`},
+		{"relay address without a port", `{"relays":[{"or_addresses":["192.0.2.1"]}]}`},
+		{"exit address that is not one", `{"relays":[{"exit_addresses":["example"]}]}`},
+		{"text after the document", `{"relays":[{"or_addresses":["192.0.2.1:1"]}]} x`},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			dir := t.TempDir()
+			p, url := newPublisher(t, onionooBody)
+			b := newTestIPLists(dir, testList(url, FormatOnionooJSON))
+
+			err := b.download(t.Context(), b.lists[0])
+			if err != nil {
+				t.Fatalf("first download: %v", err)
+			}
+
+			p.answer(http.StatusOK, tt.body)
+
+			err = b.download(t.Context(), b.lists[0])
+			if err == nil {
+				t.Fatal("second download returned no error")
+			}
+
+			if len(lookup(t, b, onionooExitAddr)) != 1 {
+				t.Errorf("%s is no longer listed", onionooExitAddr)
+			}
+
+			onDisk, err := fs.ReadFile(os.DirFS(dir), testListName)
+			if err != nil || string(onDisk) != onionooBody {
+				t.Errorf("copy on disk %q (%v), want the good version", onDisk, err)
+			}
+		})
+	}
+}
+
 // checkGoodCopyOnDisk checks that dir holds the good version of the test list
 // and no part of a failed download.
 func checkGoodCopyOnDisk(t *testing.T, dir string) {
@@ -555,6 +640,33 @@ func TestStopCancelsADownloadInProgress(t *testing.T) {
 	}
 }
 
+// TestTorExitsListHoldsIPv6Exits downloads an Onionoo document as the public
+// Tor exit list, which is what an IPv6 address's is_tor answer comes from.
+func TestTorExitsListHoldsIPv6Exits(t *testing.T) {
+	t.Parallel()
+
+	index := slices.IndexFunc(Lists(), func(list List) bool {
+		return list.Name == TorExitsList
+	})
+	if index < 0 {
+		t.Fatalf("no list named %s", TorExitsList)
+	}
+
+	list := Lists()[index]
+	_, list.URL = newPublisher(t, onionooBody)
+	b := newTestIPLists(t.TempDir(), list)
+
+	err := b.download(t.Context(), list)
+	if err != nil {
+		t.Fatalf("download: %v", err)
+	}
+
+	listings := lookup(t, b, onionooIPv6ExitAddr)
+	if len(listings) != 1 || listings[0].Name != TorExitsList {
+		t.Errorf("listings %+v, want one for %s", listings, TorExitsList)
+	}
+}
+
 // TestPublicListsAreComplete checks that every public list has what a
 // download and a lookup answer need, and a unique name.
 func TestPublicListsAreComplete(t *testing.T) {