routewatch/internal/iplist/iplist_internal_test.go

package iplist

import (
	"context"
	"errors"
	"io"
	"io/fs"
	"log/slog"
	"net/http"
	"net/http/httptest"
	"net/netip"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"sync"
	"testing"
	"time"
)

// The test list's name and description, the address its good version lists,
// and the address a newer version lists instead.
const (
	testListName        = "test_list"
	testListDescription = "On the test list"
	listedAddr          = "192.0.2.1"
	goodListBody        = "# A test list\n" + listedAddr + "\n"
	newListedAddr       = "192.0.2.2"
	newListBody         = "# A test list\n" + newListedAddr + "\n"
)

// A small Onionoo details document: a relay with an exit address and IPv4 and
// IPv6 relay addresses, one without exit addresses, and one with a list of
// ports.
const (
	onionooExitAddr     = "192.0.2.10"
	onionooIPv6ExitAddr = "2001:db8::1"
	onionooBody         = `{"version":"8.0","relays":[
		{"exit_addresses":["` + onionooExitAddr + `"],
		 "or_addresses":["192.0.2.11:443","[` + onionooIPv6ExitAddr + `]:9001"]},
		{"or_addresses":["192.0.2.20:443"]},
		{"or_addresses":["192.0.2.30:80,443"]}
	],"bridges":[]}`
)

// Small documents in the VPN services' formats. PIA's and IVPN's have no
// public IPv6 addresses, and IVPN's holds addresses that are not its servers,
// which must not be read.
const (
	mullvadBody = `[
		{"hostname":"a-wg-001","ipv4_addr_in":"192.0.2.40","ipv6_addr_in":"2001:db8::40"},
		{"hostname":"a-br-001","ipv4_addr_in":"192.0.2.41","ipv6_addr_in":null,
		 "ipv4_v2ray":"192.0.2.42"}
	]`
	nordVPNBody = `[
		{"hostname":"a1.example.com","station":"192.0.2.50","ipv6_station":""},
		{"hostname":"a2.example.com","station":"192.0.2.51","ipv6_station":"2001:db8::51"}
	]`
	piaBody = `{"groups":{"wg":[{"name":"wireguard","ports":[1337]}]},"regions":[
		{"id":"a","servers":{
			"wg":[{"ip":"192.0.2.60","cn":"a1"}],
			"ikev2":[{"ip":"192.0.2.61","cn":"a1"}]}},
		{"id":"b","servers":{"meta":[{"ip":"192.0.2.62","cn":"b1","van":true}]}}
	]}` + "\n\nc2lnbmF0dXJl\nbGluZSB0d28=\n"
	ivpnBody = `{"wireguard":[{"gateway":"a.wg.example.com","hosts":[
		{"hostname":"a1","host":"192.0.2.70","v2ray":"192.0.2.71",
		 "ipv6":{"local_ip":"2001:db8::70/96"}}]}],
	"openvpn":[{"gateway":"a.gw.example.com","hosts":[
		{"hostname":"a1","host":"192.0.2.72"}]}],
	"config":{"api":{"ips":["192.0.2.73"]}}}`
)

// protonVPNBody is a small file in the format of the ProtonVPN-IPs list.
const protonVPNBody = "#\n# protonvpn_ip_ranges.txt\n" +
	"# https://github.com/tn3w/ProtonVPN-IPs\n#\n" +
	"# The /24 block of every advertised IPv4.\n#\n" +
	"192.0.2.0/24\n198.51.100.0/24\n"

// protonVPNEntryBody is a small file in the format of the ProtonVPN-IPs list
// of server addresses.
const protonVPNEntryBody = "#\n# protonvpn_entry_ips.txt\n" +
	"# https://github.com/tn3w/ProtonVPN-IPs\n#\n" +
	"# Entry IPs clients connect to, resolved from ProtonVPN hostnames.\n#\n" +
	"192.0.2.80\n2001:db8::80\n"

// iCloudPrivateRelayAddr is an address on iCloudPrivateRelayBody.
const iCloudPrivateRelayAddr = "2001:db8:1::5"

// iCloudPrivateRelayBody is a small file in the format of Apple's iCloud
// Private Relay file.
const iCloudPrivateRelayBody = "192.0.2.0/31,GB,GB-EN,London,\n" +
	"198.51.100.32/27,US,US-CA,Los Angeles,\n" +
	"2001:db8:1::/64,US,US-AK,Anchorage,\n"

// emptyFileCase names the case of a wrong document that is an empty file.
const emptyFileCase = "an empty file"

// errorPage is a file that is no list, as a publisher's error page is.
const errorPage = "<!DOCTYPE html>\n<html></html>\n"

// testETag is the ETag a publisher sends with a version of the test list.
const testETag = `"v1"`

// maxWait is how long a test waits for the download goroutine.
const maxWait = 10 * time.Second

// publisher stands in for a list's publisher. It answers every request with
// its status and body, and keeps the headers of each request. A 200 answer
// carries lastModified and etag, when set, as Last-Modified and ETag. As
// from a real publisher, a request whose If-None-Match or If-Modified-Since
// matches them is answered "not modified".
type publisher struct {
	mu           sync.Mutex
	status       int
	body         string
	lastModified time.Time
	etag         string
	headers      []http.Header
}

// newPublisher starts a publisher that answers 200 with body, and returns it
// and its URL.
func newPublisher(t *testing.T, body string) (*publisher, string) {
	t.Helper()

	p := &publisher{status: http.StatusOK, body: body}
	srv := httptest.NewServer(p)
	t.Cleanup(srv.Close)

	return p, srv.URL
}

func (p *publisher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	p.mu.Lock()
	defer p.mu.Unlock()

	p.headers = append(p.headers, r.Header.Clone())

	if p.status != http.StatusOK {
		w.WriteHeader(p.status)
		_, _ = io.WriteString(w, p.body)

		return
	}

	if p.etag != "" {
		w.Header().Set(etagHeader, p.etag)
	}

	// ServeContent sends Last-Modified unless lastModified is zero, and
	// answers "not modified" when the request's conditions match.
	http.ServeContent(w, r, testListName, p.lastModified,
		strings.NewReader(p.body))
}

// answer makes the publisher answer later requests with status and body.
func (p *publisher) answer(status int, body string) {
	p.mu.Lock()
	defer p.mu.Unlock()

	p.status, p.body = status, body
}

// publish makes the publisher answer later requests with a version of the
// list: body, with lastModified and etag. A zero lastModified or an empty
// etag is not sent.
func (p *publisher) publish(body string, lastModified time.Time, etag string) {
	p.mu.Lock()
	defer p.mu.Unlock()

	p.status, p.body = http.StatusOK, body
	p.lastModified, p.etag = lastModified, etag
}

// requests returns the headers of the requests the publisher has had.
func (p *publisher) requests() []http.Header {
	p.mu.Lock()
	defer p.mu.Unlock()

	return append([]http.Header(nil), p.headers...)
}

// testList returns a list served at url, written in format.
func testList(url string, format Format) List {
	return List{
		Name:        testListName,
		Description: testListDescription,
		URL:         url,
		Interval:    time.Hour,
		Format:      format,
	}
}

func newTestIPLists(dir string, list List) *IPLists {
	return New(Params{
		Dir:    dir,
		Lists:  []List{list},
		Logger: slog.New(slog.DiscardHandler),
	})
}

// lookup looks addr up in b.
func lookup(t *testing.T, b *IPLists, addr string) []Listing {
	t.Helper()

	return b.Lookup(netip.MustParseAddr(addr))
}

// TestLookupFindsListedAddresses downloads a list in each format and looks
// up addresses on it, inside its networks, and off it, in IPv4 and IPv6.
// A listed address has one listing that names the list; any other has an
// empty list of listings.
func TestLookupFindsListedAddresses(t *testing.T) {
	t.Parallel()

	runLookupCases(t, []lookupCase{
		{
			name:     "single addresses",
			format:   FormatLines,
			body:     "# comment\n192.0.2.1\n\n2001:db8::1\n",
			listed:   []string{"192.0.2.1", "::ffff:192.0.2.1", "2001:db8::1"},
			unlisted: []string{"192.0.2.2", "2001:db8::2"},
		},
		{
			name:     "networks",
			format:   FormatLines,
			body:     "# comment\n198.51.100.0/24\n2001:db8:1::/48\n",
			listed:   []string{"198.51.100.0", "198.51.100.255", "2001:db8:1::5"},
			unlisted: []string{"198.51.101.0", "2001:db8:2::1"},
		},
		{
			name:   "Spamhaus networks",
			format: FormatSpamhausJSON,
			body: `{"cidr":"203.0.113.0/25","sblid":"SBL1","rir":"arin"}` + "\n" +
				`{"cidr":"2001:db8:3::/48","sblid":"SBL2","rir":"ripencc"}` + "\n" +
				`{"type":"metadata","timestamp":1793074442,"records":2}` + "\n",
			listed:   []string{"203.0.113.1", "2001:db8:3::1"},
			unlisted: []string{"203.0.113.200", "2001:db8:4::1"},
		},
		{
			name:   "DShield networks",
			format: FormatDShield,
			body: "#   DShield.org Recommended Block List\n" +
				"192.0.2.0\t192.0.2.255\t24\t327\tEXAMPLE\tUS\tNone\n",
			listed:   []string{"192.0.2.0", "192.0.2.77"},
			unlisted: []string{"192.0.3.1", "2001:db8::1"},
		},
		{
			name:   "Onionoo relays",
			format: FormatOnionooJSON,
			body:   onionooBody,
			listed: []string{
				onionooExitAddr, "192.0.2.11", "192.0.2.20", "192.0.2.30",
				onionooIPv6ExitAddr,
			},
			unlisted: []string{"192.0.2.12", "2001:db8::2"},
		},
		{
			name:   "iCloud Private Relay networks",
			format: FormatICloudPrivateRelayCSV,
			body:   iCloudPrivateRelayBody,
			listed: []string{
				"192.0.2.1", "198.51.100.63", iCloudPrivateRelayAddr,
			},
			unlisted: []string{"192.0.2.2", "198.51.100.64", "2001:db8:2::1"},
		},
	})
}

// TestLookupFindsVPNServers downloads a list in each VPN service's format and
// looks up addresses on it and off it.
func TestLookupFindsVPNServers(t *testing.T) {
	t.Parallel()

	runLookupCases(t, []lookupCase{
		{
			name:     "Mullvad relays",
			format:   FormatMullvadJSON,
			body:     mullvadBody,
			listed:   []string{"192.0.2.40", "192.0.2.41", "192.0.2.42", "2001:db8::40"},
			unlisted: []string{"192.0.2.43", "2001:db8::41"},
		},
		{
			name:     "NordVPN servers",
			format:   FormatNordVPNJSON,
			body:     nordVPNBody,
			listed:   []string{"192.0.2.50", "192.0.2.51", "2001:db8::51"},
			unlisted: []string{"192.0.2.52", "2001:db8::50"},
		},
		{
			name:     "PIA servers, with the signature after the JSON",
			format:   FormatPIAJSON,
			body:     piaBody,
			listed:   []string{"192.0.2.60", "192.0.2.61", "192.0.2.62"},
			unlisted: []string{"192.0.2.63", "2001:db8::60"},
		},
		{
			name:     "IVPN hosts",
			format:   FormatIVPNJSON,
			body:     ivpnBody,
			listed:   []string{"192.0.2.70", "192.0.2.71", "192.0.2.72"},
			unlisted: []string{"192.0.2.73", "2001:db8::70"},
		},
		{
			name:     "ProtonVPN networks",
			format:   FormatProtonVPN,
			body:     protonVPNBody,
			listed:   []string{"192.0.2.0", "192.0.2.255", "198.51.100.7"},
			unlisted: []string{"192.0.3.1", "2001:db8::90"},
		},
		{
			name:     "ProtonVPN server addresses",
			format:   FormatProtonVPN,
			body:     protonVPNEntryBody,
			listed:   []string{"192.0.2.80", "2001:db8::80"},
			unlisted: []string{"192.0.2.81", "2001:db8::81"},
		},
	})
}

// lookupCase is a list file, the addresses it holds and some it does not.
type lookupCase struct {
	name     string
	format   Format
	body     string
	listed   []string
	unlisted []string
}

// runLookupCases downloads each case's list file and checks its addresses.
func runLookupCases(t *testing.T, cases []lookupCase) {
	t.Helper()

	for _, tt := range cases {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			_, url := newPublisher(t, tt.body)
			b := newTestIPLists(t.TempDir(), testList(url, tt.format))

			err := b.download(t.Context(), b.lists[0])
			if err != nil {
				t.Fatalf("download: %v", err)
			}

			checkLookups(t, b, tt.listed, tt.unlisted)
		})
	}
}

// checkLookups checks that each of listed has one listing that names the test
// list, and each of unlisted has an empty list of listings.
func checkLookups(t *testing.T, b *IPLists, listed, unlisted []string) {
	t.Helper()

	for _, addr := range listed {
		listings := lookup(t, b, addr)
		if len(listings) != 1 || listings[0].Name != testListName ||
			listings[0].Description != testListDescription ||
			listings[0].Updated.IsZero() {
			t.Errorf("%s: listings %+v, want one for %s", addr,
				listings, testListName)
		}
	}

	for _, addr := range unlisted {
		listings := lookup(t, b, addr)
		if listings == nil || len(listings) != 0 {
			t.Errorf("%s: listings %#v, want an empty list", addr, listings)
		}
	}
}

// TestFailedDownloadKeepsThePreviousCopy downloads a good version of a list,
// then one that fails. The good copy stays in memory and on disk, and no
// part of the failed download is left beside it.
func TestFailedDownloadKeepsThePreviousCopy(t *testing.T) {
	t.Parallel()

	tests := []struct {
		name   string
		status int
		body   string
	}{
		{"server error", http.StatusInternalServerError, "unavailable"},
		{"not a list", http.StatusOK, errorPage},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			dir := t.TempDir()
			p, url := newPublisher(t, goodListBody)
			b := newTestIPLists(dir, testList(url, FormatLines))

			err := b.download(t.Context(), b.lists[0])
			if err != nil {
				t.Fatalf("first download: %v", err)
			}

			before := lookup(t, b, listedAddr)

			p.answer(tt.status, tt.body)

			err = b.download(t.Context(), b.lists[0])
			if err == nil {
				t.Fatal("second download returned no error")
			}

			after := lookup(t, b, listedAddr)
			if len(after) != 1 || len(before) != 1 ||
				!after[0].Updated.Equal(before[0].Updated) {
				t.Errorf("listings %+v after the failure, want %+v", after, before)
			}

			checkGoodCopyOnDisk(t, dir)
		})
	}
}

// TestFileThatIsNotAnOnionooDocumentKeepsThePreviousCopy downloads a good
// Onionoo document, then files that are not one. Each fails the parse, and
// the good copy stays.
func TestFileThatIsNotAnOnionooDocumentKeepsThePreviousCopy(t *testing.T) {
	t.Parallel()

	tests := []struct {
		name string
		body string
	}{
		{"not JSON", errorPage},
		{"other JSON", `{"status":"ok","data":{"relays_count":1}}`},
		{"JSON array", `[{"or_addresses":["192.0.2.1:443"]}]`},
		{"no relays", `{"relays":[],"bridges":[]}`},
		{"relays without addresses", `{"relays":[{"nickname":"example"}]}`},
		{"relay address without a port", `{"relays":[{"or_addresses":["192.0.2.1"]}]}`},
		{"exit address that is not one", `{"relays":[{"exit_addresses":["example"]}]}`},
		{"text after the document", `{"relays":[{"or_addresses":["192.0.2.1:1"]}]} x`},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			dir := t.TempDir()
			p, url := newPublisher(t, onionooBody)
			b := newTestIPLists(dir, testList(url, FormatOnionooJSON))

			err := b.download(t.Context(), b.lists[0])
			if err != nil {
				t.Fatalf("first download: %v", err)
			}

			p.answer(http.StatusOK, tt.body)

			err = b.download(t.Context(), b.lists[0])
			if err == nil {
				t.Fatal("second download returned no error")
			}

			if len(lookup(t, b, onionooExitAddr)) != 1 {
				t.Errorf("%s is no longer listed", onionooExitAddr)
			}

			onDisk, err := fs.ReadFile(os.DirFS(dir), testListName)
			if err != nil || string(onDisk) != onionooBody {
				t.Errorf("copy on disk %q (%v), want the good version", onDisk, err)
			}
		})
	}
}

// TestWrongVPNDocumentFailsToParse parses documents that are not a VPN
// service's list. Each fails, so none can replace a good copy.
func TestWrongVPNDocumentFailsToParse(t *testing.T) {
	t.Parallel()

	formats := []struct {
		name       string
		format     Format
		good       string
		empty      string
		badAddress string
	}{
		{
			"Mullvad", FormatMullvadJSON, mullvadBody,
			`[{"hostname":"a"}]`, `[{"ipv4_addr_in":"example"}]`,
		},
		{
			"NordVPN", FormatNordVPNJSON, nordVPNBody,
			`[{"hostname":"a","station":"","ipv6_station":""}]`,
			`[{"station":"example"}]`,
		},
		{
			"PIA", FormatPIAJSON, piaBody,
			`{"regions":[{"id":"a","servers":{}}]}`,
			`{"regions":[{"servers":{"wg":[{"ip":"example"}]}}]}`,
		},
		{
			"IVPN", FormatIVPNJSON, ivpnBody,
			`{"wireguard":[{"hosts":[]}],"openvpn":[]}`,
			`{"wireguard":[{"hosts":[{"host":"example"}]}]}`,
		},
	}

	for _, f := range formats {
		bodies := map[string]string{
			"not JSON":     errorPage,
			"other JSON":   `{"status":"ok","data":{"count":1}}`,
			"no addresses": f.empty,
			"bad address":  f.badAddress,
			"cut off":      f.good[:len(f.good)/2],
			emptyFileCase:  "",
		}

		for name, body := range bodies {
			t.Run(f.name+" "+name, func(t *testing.T) {
				t.Parallel()

				addresses, err := parse(strings.NewReader(body), f.format)
				if err == nil {
					t.Errorf("parse returned %v and no error", addresses.Ranges())
				}
			})
		}
	}
}

// TestWrongProtonVPNDocumentFailsToParse parses documents that are not a
// ProtonVPN-IPs list. Each fails, so none can replace a good copy.
func TestWrongProtonVPNDocumentFailsToParse(t *testing.T) {
	t.Parallel()

	bodies := map[string]string{
		"error page":    errorPage,
		"JSON":          `{"status":"ok"}`,
		"comments only": "#\n# protonvpn_ip_ranges.txt\n#\n",
		"bad network":   "192.0.2.0/24\nexample\n",
		emptyFileCase:   "",
	}

	for name, body := range bodies {
		t.Run(name, func(t *testing.T) {
			t.Parallel()

			addresses, err := parse(strings.NewReader(body), FormatProtonVPN)
			if err == nil {
				t.Errorf("parse returned %v and no error", addresses.Ranges())
			}
		})
	}
}

// TestWrongICloudPrivateRelayDocumentFailsToParse parses documents that are
// not Apple's iCloud Private Relay file. Each fails, so none can replace a
// good copy.
func TestWrongICloudPrivateRelayDocumentFailsToParse(t *testing.T) {
	t.Parallel()

	bodies := map[string]string{
		"error page":     errorPage,
		"JSON":           `{"status":"ok","count":1}`,
		"no other field": "192.0.2.0/24\n",
		"bad network":    "192.0.2.0/24,GB,GB-EN,London,\nexample,GB,GB-EN,London,\n",
		"an address":     "192.0.2.1,GB,GB-EN,London,\n",
		emptyFileCase:    "",
	}

	for name, body := range bodies {
		t.Run(name, func(t *testing.T) {
			t.Parallel()

			addresses, err := parse(strings.NewReader(body),
				FormatICloudPrivateRelayCSV)
			if err == nil {
				t.Errorf("parse returned %v and no error", addresses.Ranges())
			}
		})
	}
}

// TestLookupMarksVPNLists looks up an address on a VPN list and on another
// list. Only the VPN list's listing says it is one.
func TestLookupMarksVPNLists(t *testing.T) {
	t.Parallel()

	_, url := newPublisher(t, goodListBody)
	vpnList := testList(url, FormatLines)
	vpnList.Name = "test_vpn_list"
	vpnList.VPN = true
	otherList := testList(url, FormatLines)

	b := New(Params{
		Dir:    t.TempDir(),
		Lists:  []List{vpnList, otherList},
		Logger: slog.New(slog.DiscardHandler),
	})

	for _, list := range b.lists {
		err := b.download(t.Context(), list)
		if err != nil {
			t.Fatalf("download of %s: %v", list.Name, err)
		}
	}

	listings := lookup(t, b, listedAddr)
	if len(listings) != 2 || !listings[0].VPN || listings[1].VPN {
		t.Errorf("listings %+v, want the first of two marked as a VPN list",
			listings)
	}
}

// TestPublicListsMarkTheVPNLists checks which public lists are VPN lists,
// since their names are what vpn_providers reports.
func TestPublicListsMarkTheVPNLists(t *testing.T) {
	t.Parallel()

	var got []string

	for _, list := range Lists() {
		if list.VPN {
			got = append(got, list.Name)
		}
	}

	want := []string{
		"ivpn", "mullvad", "nordvpn", "pia", "protonvpn", "protonvpn_entry",
		"x4bnet", "x4bnet_v6",
	}
	if !slices.Equal(got, want) {
		t.Errorf("VPN lists %v, want %v", got, want)
	}
}

// checkGoodCopyOnDisk checks that dir holds the good version of the test list
// and no part of a failed download.
func checkGoodCopyOnDisk(t *testing.T, dir string) {
	t.Helper()

	copies := os.DirFS(dir)

	onDisk, err := fs.ReadFile(copies, testListName)
	if err != nil || string(onDisk) != goodListBody {
		t.Errorf("copy on disk %q (%v), want %q", onDisk, err, goodListBody)
	}

	_, err = fs.Stat(copies, testListName+downloadSuffix)
	if !errors.Is(err, fs.ErrNotExist) {
		t.Errorf("the failed download's file is still there (%v)", err)
	}
}

// commentLines is an endless io.Reader of comment lines. It lets a test send
// a large list file without holding the file in memory.
type commentLines struct {
	offset int
}

func (c *commentLines) Read(p []byte) (int, error) {
	const line = "# a comment line that pads a large list file\n"

	for filled := 0; filled < len(p); {
		copied := copy(p[filled:], line[c.offset:])
		filled += copied
		c.offset = (c.offset + copied) % len(line)
	}

	return len(p), nil
}

// largeListFile returns a reader of a list file of size bytes: comment lines,
// then newListedAddr on the last line.
func largeListFile(size int64) io.Reader {
	lastLine := "\n" + newListedAddr + "\n"

	return io.MultiReader(
		io.LimitReader(&commentLines{}, size-int64(len(lastLine))),
		strings.NewReader(lastLine),
	)
}

// TestLargeFileUnderTheSizeLimitIsAccepted replaces a list with a 199 MiB
// file. The address on its last line is listed, so the whole file was read.
func TestLargeFileUnderTheSizeLimitIsAccepted(t *testing.T) {
	t.Parallel()

	const size = 199 << 20

	b := newTestIPLists(t.TempDir(), testList("", FormatLines))

	err := b.replaceCopy(b.lists[0], largeListFile(size), validators{})
	if err != nil {
		t.Fatalf("replaceCopy: %v", err)
	}

	if listings := lookup(t, b, newListedAddr); len(listings) != 1 {
		t.Errorf("listings %+v, want one", listings)
	}
}

// TestFileOverTheSizeLimitKeepsThePreviousCopy downloads a good version of a
// list, then replaces it with a file one byte over the limit. The attempt
// fails, and the good copy stays in memory and on disk.
func TestFileOverTheSizeLimitKeepsThePreviousCopy(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()
	_, url := newPublisher(t, goodListBody)
	b := newTestIPLists(dir, testList(url, FormatLines))

	err := b.download(t.Context(), b.lists[0])
	if err != nil {
		t.Fatalf("download: %v", err)
	}

	err = b.replaceCopy(b.lists[0], largeListFile(maxDownloadSize+1), validators{})
	if !errors.Is(err, errTooLarge) {
		t.Fatalf("replaceCopy: %v, want %v", err, errTooLarge)
	}

	if listings := lookup(t, b, listedAddr); len(listings) != 1 {
		t.Errorf("listings of %s %+v, want one", listedAddr, listings)
	}

	checkGoodCopyOnDisk(t, dir)
}

// TestRestartLoadsCopiesFromDisk downloads a list, then starts a new
// IPLists on the same directory, as a restart does. The new one answers
// from the copy on disk at once, and does not download the list again while
// the copy is younger than the list's interval.
func TestRestartLoadsCopiesFromDisk(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()
	p, url := newPublisher(t, goodListBody)
	list := testList(url, FormatLines)

	first := newTestIPLists(dir, list)

	err := first.download(t.Context(), list)
	if err != nil {
		t.Fatalf("download: %v", err)
	}

	downloaded := lookup(t, first, listedAddr)[0].Updated

	second := newTestIPLists(dir, list)
	second.Start(t.Context())
	second.Stop(t.Context())

	// The download takes its time after writing the file, and the restart
	// reads the file's time. A busy host can put seconds between the two.
	const maxUpdatedDifference = 15 * time.Second

	listings := lookup(t, second, listedAddr)
	if len(listings) != 1 ||
		listings[0].Updated.Sub(downloaded).Abs() > maxUpdatedDifference {
		t.Errorf("after the restart, listings %+v, want one updated at %s",
			listings, downloaded)
	}

	if got := len(p.requests()); got != 1 {
		t.Errorf("the publisher had %d requests, want 1", got)
	}
}

// TestStartRefreshesAnOldCopy starts on a copy older than its list's
// interval. The list is downloaded at once, with routewatch's User-Agent and
// the Last-Modified and ETag the publisher sent with the copy, kept on disk.
// The publisher answers that the copy is still current, so the copy is kept
// and marked as updated now.
func TestStartRefreshesAnOldCopy(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()
	p, url := newPublisher(t, goodListBody)
	list := testList(url, FormatLines)

	lastModified := time.Now().Add(-3 * list.Interval).UTC().Truncate(time.Second)
	p.publish(goodListBody, lastModified, testETag)

	err := newTestIPLists(dir, list).download(t.Context(), list)
	if err != nil {
		t.Fatalf("download: %v", err)
	}

	old := time.Now().Add(-2 * list.Interval).UTC().Truncate(time.Second)

	err = os.Chtimes(filepath.Join(dir, testListName), old, old)
	if err != nil {
		t.Fatalf("Chtimes: %v", err)
	}

	b := newTestIPLists(dir, list)
	b.Start(t.Context())

	deadline := time.Now().Add(maxWait)
	for !lookup(t, b, listedAddr)[0].Updated.After(old) {
		if time.Now().After(deadline) {
			t.Fatalf("the copy was not marked updated within %s", maxWait)
		}

		time.Sleep(time.Millisecond)
	}

	b.Stop(t.Context())

	requests := p.requests()
	last := requests[len(requests)-1]

	want := lastModified.Format(http.TimeFormat)
	if got := last.Get(ifModifiedSinceHeader); got != want {
		t.Errorf("If-Modified-Since %q, want %q", got, want)
	}

	if got := last.Get(ifNoneMatchHeader); got != testETag {
		t.Errorf("If-None-Match %q, want %q", got, testETag)
	}

	if got := last.Get(userAgentHeader); got != userAgent {
		t.Errorf("User-Agent %q, want %q", got, userAgent)
	}
}

// TestNewVersionOlderThanTheLastDownloadReplacesTheCopy downloads a list.
// The publisher then puts out a new version whose Last-Modified is earlier
// than that download, as when a cache in front of the publisher passes the
// version on late. The next download sends the Last-Modified of the copy,
// not the time of the download, so it gets the new version.
func TestNewVersionOlderThanTheLastDownloadReplacesTheCopy(t *testing.T) {
	t.Parallel()

	p, url := newPublisher(t, goodListBody)
	list := testList(url, FormatLines)
	b := newTestIPLists(t.TempDir(), list)

	firstVersion := time.Now().Add(-2 * list.Interval).UTC().Truncate(time.Second)
	p.publish(goodListBody, firstVersion, "")

	err := b.download(t.Context(), list)
	if err != nil {
		t.Fatalf("first download: %v", err)
	}

	p.publish(newListBody, firstVersion.Add(list.Interval), "")

	err = b.download(t.Context(), list)
	if err != nil {
		t.Fatalf("second download: %v", err)
	}

	if listings := lookup(t, b, newListedAddr); len(listings) != 1 {
		t.Errorf("%s: listings %+v, want one from the new version",
			newListedAddr, listings)
	}

	want := firstVersion.Format(http.TimeFormat)
	if got := p.requests()[1].Get(ifModifiedSinceHeader); got != want {
		t.Errorf("If-Modified-Since %q, want %q", got, want)
	}
}

// TestDownloadWithoutLastModifiedOrETagSendsNoCondition downloads a list
// twice from a publisher that sends neither Last-Modified nor ETag. Neither
// request carries If-Modified-Since or If-None-Match, so the second download
// gets the whole list again.
func TestDownloadWithoutLastModifiedOrETagSendsNoCondition(t *testing.T) {
	t.Parallel()

	p, url := newPublisher(t, goodListBody)
	list := testList(url, FormatLines)
	b := newTestIPLists(t.TempDir(), list)

	err := b.download(t.Context(), list)
	if err != nil {
		t.Fatalf("first download: %v", err)
	}

	err = b.download(t.Context(), list)
	if err != nil {
		t.Fatalf("second download: %v", err)
	}

	for i, request := range p.requests() {
		if request.Get(ifModifiedSinceHeader) != "" ||
			request.Get(ifNoneMatchHeader) != "" {
			t.Errorf("request %d has a condition: %v", i, request)
		}
	}
}

// TestStopCancelsADownloadInProgress stops while the publisher has not
// answered. Stop cancels the download and returns once the download
// goroutine has stopped.
func TestStopCancelsADownloadInProgress(t *testing.T) {
	t.Parallel()

	arrived := make(chan struct{})

	var once sync.Once

	srv := httptest.NewServer(http.HandlerFunc(
		func(_ http.ResponseWriter, r *http.Request) {
			once.Do(func() { close(arrived) })
			<-r.Context().Done()
		}))
	t.Cleanup(srv.Close)

	b := newTestIPLists(t.TempDir(), testList(srv.URL, FormatLines))
	b.Start(t.Context())

	select {
	case <-arrived:
	case <-time.After(maxWait):
		t.Fatalf("no download began within %s", maxWait)
	}

	ctx, cancel := context.WithTimeout(t.Context(), maxWait)
	defer cancel()

	b.Stop(ctx)

	select {
	case <-b.done:
	default:
		t.Error("Stop returned before the download goroutine stopped")
	}
}

// TestTorExitsListHoldsIPv6Exits downloads an Onionoo document as the public
// Tor exit list, which is what an IPv6 address's is_tor answer comes from.
func TestTorExitsListHoldsIPv6Exits(t *testing.T) {
	t.Parallel()

	checkPublicListHolds(t, TorExitsList, onionooBody, onionooIPv6ExitAddr)
}

// TestICloudPrivateRelayListHoldsItsNetworks downloads a file in Apple's
// format as the public iCloud Private Relay list, which is what an address's
// is_relay answer comes from.
func TestICloudPrivateRelayListHoldsItsNetworks(t *testing.T) {
	t.Parallel()

	checkPublicListHolds(t, ICloudPrivateRelayList, iCloudPrivateRelayBody,
		iCloudPrivateRelayAddr)
}

// checkPublicListHolds downloads body as the public list called name and
// checks that addr has one listing, which names that list.
func checkPublicListHolds(t *testing.T, name, body, addr string) {
	t.Helper()

	index := slices.IndexFunc(Lists(), func(list List) bool {
		return list.Name == name
	})
	if index < 0 {
		t.Fatalf("no list named %s", name)
	}

	list := Lists()[index]
	_, list.URL = newPublisher(t, body)
	b := newTestIPLists(t.TempDir(), list)

	err := b.download(t.Context(), list)
	if err != nil {
		t.Fatalf("download: %v", err)
	}

	listings := lookup(t, b, addr)
	if len(listings) != 1 || listings[0].Name != name {
		t.Errorf("listings %+v, want one for %s", listings, name)
	}
}

// TestPublicListsAreComplete checks that every public list has what a
// download and a lookup answer need, and a unique name.
func TestPublicListsAreComplete(t *testing.T) {
	t.Parallel()

	names := make(map[string]bool)

	for _, list := range Lists() {
		if list.Name == "" || list.Description == "" || list.URL == "" ||
			list.Interval <= 0 || names[list.Name] {
			t.Errorf("list %+v is incomplete or has a name used before", list)
		}

		names[list.Name] = true
	}
}