Rename blocklists to IP lists and add IP lookup flags (closes #233)

The answer field is now ip_lists, the package internal/iplist and the
state directory iplists. No compatibility is kept for the old names.

Every IP lookup result gains a flags object with is_tor and is_bogon.
The largest list file accepted is now 200 MiB.

Model: opus-5-5

internal/iplist/iplist_internal_test.go

@@ -0,0 +1,573 @@
+package iplist
+
+import (
+	"context"
+	"errors"
+	"io"
+	"io/fs"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// The test list's name and description, the address its good version lists,
+// and the address a newer version lists instead.
+const (
+	testListName        = "test_list"
+	testListDescription = "On the test list"
+	listedAddr          = "192.0.2.1"
+	goodListBody        = "# A test list\n" + listedAddr + "\n"
+	newListedAddr       = "192.0.2.2"
+	newListBody         = "# A test list\n" + newListedAddr + "\n"
+)
+
+// testETag is the ETag a publisher sends with a version of the test list.
+const testETag = `"v1"`
+
+// maxWait is how long a test waits for the download goroutine.
+const maxWait = 10 * time.Second
+
+// publisher stands in for a list's publisher. It answers every request with
+// its status and body, and keeps the headers of each request. A 200 answer
+// carries lastModified and etag, when set, as Last-Modified and ETag. As
+// from a real publisher, a request whose If-None-Match or If-Modified-Since
+// matches them is answered "not modified".
+type publisher struct {
+	mu           sync.Mutex
+	status       int
+	body         string
+	lastModified time.Time
+	etag         string
+	headers      []http.Header
+}
+
+// newPublisher starts a publisher that answers 200 with body, and returns it
+// and its URL.
+func newPublisher(t *testing.T, body string) (*publisher, string) {
+	t.Helper()
+
+	p := &publisher{status: http.StatusOK, body: body}
+	srv := httptest.NewServer(p)
+	t.Cleanup(srv.Close)
+
+	return p, srv.URL
+}
+
+func (p *publisher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	p.headers = append(p.headers, r.Header.Clone())
+
+	if p.status != http.StatusOK {
+		w.WriteHeader(p.status)
+		_, _ = io.WriteString(w, p.body)
+
+		return
+	}
+
+	if p.etag != "" {
+		w.Header().Set(etagHeader, p.etag)
+	}
+
+	// ServeContent sends Last-Modified unless lastModified is zero, and
+	// answers "not modified" when the request's conditions match.
+	http.ServeContent(w, r, testListName, p.lastModified,
+		strings.NewReader(p.body))
+}
+
+// answer makes the publisher answer later requests with status and body.
+func (p *publisher) answer(status int, body string) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	p.status, p.body = status, body
+}
+
+// publish makes the publisher answer later requests with a version of the
+// list: body, with lastModified and etag. A zero lastModified or an empty
+// etag is not sent.
+func (p *publisher) publish(body string, lastModified time.Time, etag string) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	p.status, p.body = http.StatusOK, body
+	p.lastModified, p.etag = lastModified, etag
+}
+
+// requests returns the headers of the requests the publisher has had.
+func (p *publisher) requests() []http.Header {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	return append([]http.Header(nil), p.headers...)
+}
+
+// testList returns a list served at url, written in format.
+func testList(url string, format Format) List {
+	return List{
+		Name:        testListName,
+		Description: testListDescription,
+		URL:         url,
+		Interval:    time.Hour,
+		Format:      format,
+	}
+}
+
+func newTestIPLists(dir string, list List) *IPLists {
+	return New(Params{
+		Dir:    dir,
+		Lists:  []List{list},
+		Logger: slog.New(slog.DiscardHandler),
+	})
+}
+
+// lookup looks addr up in b.
+func lookup(t *testing.T, b *IPLists, addr string) []Listing {
+	t.Helper()
+
+	return b.Lookup(netip.MustParseAddr(addr))
+}
+
+// TestLookupFindsListedAddresses downloads a list in each format and looks
+// up addresses on it, inside its networks, and off it, in IPv4 and IPv6.
+// A listed address has one listing that names the list; any other has an
+// empty list of listings.
+func TestLookupFindsListedAddresses(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name     string
+		format   Format
+		body     string
+		listed   []string
+		unlisted []string
+	}{
+		{
+			name:     "single addresses",
+			format:   FormatLines,
+			body:     "# comment\n192.0.2.1\n\n2001:db8::1\n",
+			listed:   []string{"192.0.2.1", "::ffff:192.0.2.1", "2001:db8::1"},
+			unlisted: []string{"192.0.2.2", "2001:db8::2"},
+		},
+		{
+			name:     "networks",
+			format:   FormatLines,
+			body:     "# comment\n198.51.100.0/24\n2001:db8:1::/48\n",
+			listed:   []string{"198.51.100.0", "198.51.100.255", "2001:db8:1::5"},
+			unlisted: []string{"198.51.101.0", "2001:db8:2::1"},
+		},
+		{
+			name:   "Spamhaus networks",
+			format: FormatSpamhausJSON,
+			body: `{"cidr":"203.0.113.0/25","sblid":"SBL1","rir":"arin"}` + "\n" +
+				`{"cidr":"2001:db8:3::/48","sblid":"SBL2","rir":"ripencc"}` + "\n" +
+				`{"type":"metadata","timestamp":1793074442,"records":2}` + "\n",
+			listed:   []string{"203.0.113.1", "2001:db8:3::1"},
+			unlisted: []string{"203.0.113.200", "2001:db8:4::1"},
+		},
+		{
+			name:   "DShield networks",
+			format: FormatDShield,
+			body: "#   DShield.org Recommended Block List\n" +
+				"192.0.2.0\t192.0.2.255\t24\t327\tEXAMPLE\tUS\tNone\n",
+			listed:   []string{"192.0.2.0", "192.0.2.77"},
+			unlisted: []string{"192.0.3.1", "2001:db8::1"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			_, url := newPublisher(t, tt.body)
+			b := newTestIPLists(t.TempDir(), testList(url, tt.format))
+
+			err := b.download(t.Context(), b.lists[0])
+			if err != nil {
+				t.Fatalf("download: %v", err)
+			}
+
+			for _, addr := range tt.listed {
+				listings := lookup(t, b, addr)
+				if len(listings) != 1 || listings[0].Name != testListName ||
+					listings[0].Description != testListDescription ||
+					listings[0].Updated.IsZero() {
+					t.Errorf("%s: listings %+v, want one for %s", addr,
+						listings, testListName)
+				}
+			}
+
+			for _, addr := range tt.unlisted {
+				listings := lookup(t, b, addr)
+				if listings == nil || len(listings) != 0 {
+					t.Errorf("%s: listings %#v, want an empty list", addr, listings)
+				}
+			}
+		})
+	}
+}
+
+// TestFailedDownloadKeepsThePreviousCopy downloads a good version of a list,
+// then one that fails. The good copy stays in memory and on disk, and no
+// part of the failed download is left beside it.
+func TestFailedDownloadKeepsThePreviousCopy(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name   string
+		status int
+		body   string
+	}{
+		{"server error", http.StatusInternalServerError, "unavailable"},
+		{"not a list", http.StatusOK, "<!DOCTYPE html>\n<html></html>\n"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			dir := t.TempDir()
+			p, url := newPublisher(t, goodListBody)
+			b := newTestIPLists(dir, testList(url, FormatLines))
+
+			err := b.download(t.Context(), b.lists[0])
+			if err != nil {
+				t.Fatalf("first download: %v", err)
+			}
+
+			before := lookup(t, b, listedAddr)
+
+			p.answer(tt.status, tt.body)
+
+			err = b.download(t.Context(), b.lists[0])
+			if err == nil {
+				t.Fatal("second download returned no error")
+			}
+
+			after := lookup(t, b, listedAddr)
+			if len(after) != 1 || len(before) != 1 ||
+				!after[0].Updated.Equal(before[0].Updated) {
+				t.Errorf("listings %+v after the failure, want %+v", after, before)
+			}
+
+			checkGoodCopyOnDisk(t, dir)
+		})
+	}
+}
+
+// checkGoodCopyOnDisk checks that dir holds the good version of the test list
+// and no part of a failed download.
+func checkGoodCopyOnDisk(t *testing.T, dir string) {
+	t.Helper()
+
+	copies := os.DirFS(dir)
+
+	onDisk, err := fs.ReadFile(copies, testListName)
+	if err != nil || string(onDisk) != goodListBody {
+		t.Errorf("copy on disk %q (%v), want %q", onDisk, err, goodListBody)
+	}
+
+	_, err = fs.Stat(copies, testListName+downloadSuffix)
+	if !errors.Is(err, fs.ErrNotExist) {
+		t.Errorf("the failed download's file is still there (%v)", err)
+	}
+}
+
+// commentLines is an endless io.Reader of comment lines. It lets a test send
+// a large list file without holding the file in memory.
+type commentLines struct {
+	offset int
+}
+
+func (c *commentLines) Read(p []byte) (int, error) {
+	const line = "# a comment line that pads a large list file\n"
+
+	for filled := 0; filled < len(p); {
+		copied := copy(p[filled:], line[c.offset:])
+		filled += copied
+		c.offset = (c.offset + copied) % len(line)
+	}
+
+	return len(p), nil
+}
+
+// largeListFile returns a reader of a list file of size bytes: comment lines,
+// then newListedAddr on the last line.
+func largeListFile(size int64) io.Reader {
+	lastLine := "\n" + newListedAddr + "\n"
+
+	return io.MultiReader(
+		io.LimitReader(&commentLines{}, size-int64(len(lastLine))),
+		strings.NewReader(lastLine),
+	)
+}
+
+// TestLargeFileUnderTheSizeLimitIsAccepted replaces a list with a 199 MiB
+// file. The address on its last line is listed, so the whole file was read.
+func TestLargeFileUnderTheSizeLimitIsAccepted(t *testing.T) {
+	t.Parallel()
+
+	const size = 199 << 20
+
+	b := newTestIPLists(t.TempDir(), testList("", FormatLines))
+
+	err := b.replaceCopy(b.lists[0], largeListFile(size), validators{})
+	if err != nil {
+		t.Fatalf("replaceCopy: %v", err)
+	}
+
+	if listings := lookup(t, b, newListedAddr); len(listings) != 1 {
+		t.Errorf("listings %+v, want one", listings)
+	}
+}
+
+// TestFileOverTheSizeLimitKeepsThePreviousCopy downloads a good version of a
+// list, then replaces it with a file one byte over the limit. The attempt
+// fails, and the good copy stays in memory and on disk.
+func TestFileOverTheSizeLimitKeepsThePreviousCopy(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	_, url := newPublisher(t, goodListBody)
+	b := newTestIPLists(dir, testList(url, FormatLines))
+
+	err := b.download(t.Context(), b.lists[0])
+	if err != nil {
+		t.Fatalf("download: %v", err)
+	}
+
+	err = b.replaceCopy(b.lists[0], largeListFile(maxDownloadSize+1), validators{})
+	if !errors.Is(err, errTooLarge) {
+		t.Fatalf("replaceCopy: %v, want %v", err, errTooLarge)
+	}
+
+	if listings := lookup(t, b, listedAddr); len(listings) != 1 {
+		t.Errorf("listings of %s %+v, want one", listedAddr, listings)
+	}
+
+	checkGoodCopyOnDisk(t, dir)
+}
+
+// TestRestartLoadsCopiesFromDisk downloads a list, then starts a new
+// IPLists on the same directory, as a restart does. The new one answers
+// from the copy on disk at once, and does not download the list again while
+// the copy is younger than the list's interval.
+func TestRestartLoadsCopiesFromDisk(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	p, url := newPublisher(t, goodListBody)
+	list := testList(url, FormatLines)
+
+	first := newTestIPLists(dir, list)
+
+	err := first.download(t.Context(), list)
+	if err != nil {
+		t.Fatalf("download: %v", err)
+	}
+
+	downloaded := lookup(t, first, listedAddr)[0].Updated
+
+	second := newTestIPLists(dir, list)
+	second.Start(t.Context())
+	second.Stop(t.Context())
+
+	listings := lookup(t, second, listedAddr)
+	if len(listings) != 1 ||
+		listings[0].Updated.Sub(downloaded).Abs() > time.Second {
+		t.Errorf("after the restart, listings %+v, want one updated at %s",
+			listings, downloaded)
+	}
+
+	if got := len(p.requests()); got != 1 {
+		t.Errorf("the publisher had %d requests, want 1", got)
+	}
+}
+
+// TestStartRefreshesAnOldCopy starts on a copy older than its list's
+// interval. The list is downloaded at once, with routewatch's User-Agent and
+// the Last-Modified and ETag the publisher sent with the copy, kept on disk.
+// The publisher answers that the copy is still current, so the copy is kept
+// and marked as updated now.
+func TestStartRefreshesAnOldCopy(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	p, url := newPublisher(t, goodListBody)
+	list := testList(url, FormatLines)
+
+	lastModified := time.Now().Add(-3 * list.Interval).UTC().Truncate(time.Second)
+	p.publish(goodListBody, lastModified, testETag)
+
+	err := newTestIPLists(dir, list).download(t.Context(), list)
+	if err != nil {
+		t.Fatalf("download: %v", err)
+	}
+
+	old := time.Now().Add(-2 * list.Interval).UTC().Truncate(time.Second)
+
+	err = os.Chtimes(filepath.Join(dir, testListName), old, old)
+	if err != nil {
+		t.Fatalf("Chtimes: %v", err)
+	}
+
+	b := newTestIPLists(dir, list)
+	b.Start(t.Context())
+
+	deadline := time.Now().Add(maxWait)
+	for !lookup(t, b, listedAddr)[0].Updated.After(old) {
+		if time.Now().After(deadline) {
+			t.Fatalf("the copy was not marked updated within %s", maxWait)
+		}
+
+		time.Sleep(time.Millisecond)
+	}
+
+	b.Stop(t.Context())
+
+	requests := p.requests()
+	last := requests[len(requests)-1]
+
+	want := lastModified.Format(http.TimeFormat)
+	if got := last.Get(ifModifiedSinceHeader); got != want {
+		t.Errorf("If-Modified-Since %q, want %q", got, want)
+	}
+
+	if got := last.Get(ifNoneMatchHeader); got != testETag {
+		t.Errorf("If-None-Match %q, want %q", got, testETag)
+	}
+
+	if got := last.Get(userAgentHeader); got != userAgent {
+		t.Errorf("User-Agent %q, want %q", got, userAgent)
+	}
+}
+
+// TestNewVersionOlderThanTheLastDownloadReplacesTheCopy downloads a list.
+// The publisher then puts out a new version whose Last-Modified is earlier
+// than that download, as when a cache in front of the publisher passes the
+// version on late. The next download sends the Last-Modified of the copy,
+// not the time of the download, so it gets the new version.
+func TestNewVersionOlderThanTheLastDownloadReplacesTheCopy(t *testing.T) {
+	t.Parallel()
+
+	p, url := newPublisher(t, goodListBody)
+	list := testList(url, FormatLines)
+	b := newTestIPLists(t.TempDir(), list)
+
+	firstVersion := time.Now().Add(-2 * list.Interval).UTC().Truncate(time.Second)
+	p.publish(goodListBody, firstVersion, "")
+
+	err := b.download(t.Context(), list)
+	if err != nil {
+		t.Fatalf("first download: %v", err)
+	}
+
+	p.publish(newListBody, firstVersion.Add(list.Interval), "")
+
+	err = b.download(t.Context(), list)
+	if err != nil {
+		t.Fatalf("second download: %v", err)
+	}
+
+	if listings := lookup(t, b, newListedAddr); len(listings) != 1 {
+		t.Errorf("%s: listings %+v, want one from the new version",
+			newListedAddr, listings)
+	}
+
+	want := firstVersion.Format(http.TimeFormat)
+	if got := p.requests()[1].Get(ifModifiedSinceHeader); got != want {
+		t.Errorf("If-Modified-Since %q, want %q", got, want)
+	}
+}
+
+// TestDownloadWithoutLastModifiedOrETagSendsNoCondition downloads a list
+// twice from a publisher that sends neither Last-Modified nor ETag. Neither
+// request carries If-Modified-Since or If-None-Match, so the second download
+// gets the whole list again.
+func TestDownloadWithoutLastModifiedOrETagSendsNoCondition(t *testing.T) {
+	t.Parallel()
+
+	p, url := newPublisher(t, goodListBody)
+	list := testList(url, FormatLines)
+	b := newTestIPLists(t.TempDir(), list)
+
+	err := b.download(t.Context(), list)
+	if err != nil {
+		t.Fatalf("first download: %v", err)
+	}
+
+	err = b.download(t.Context(), list)
+	if err != nil {
+		t.Fatalf("second download: %v", err)
+	}
+
+	for i, request := range p.requests() {
+		if request.Get(ifModifiedSinceHeader) != "" ||
+			request.Get(ifNoneMatchHeader) != "" {
+			t.Errorf("request %d has a condition: %v", i, request)
+		}
+	}
+}
+
+// TestStopCancelsADownloadInProgress stops while the publisher has not
+// answered. Stop cancels the download and returns once the download
+// goroutine has stopped.
+func TestStopCancelsADownloadInProgress(t *testing.T) {
+	t.Parallel()
+
+	arrived := make(chan struct{})
+
+	var once sync.Once
+
+	srv := httptest.NewServer(http.HandlerFunc(
+		func(_ http.ResponseWriter, r *http.Request) {
+			once.Do(func() { close(arrived) })
+			<-r.Context().Done()
+		}))
+	t.Cleanup(srv.Close)
+
+	b := newTestIPLists(t.TempDir(), testList(srv.URL, FormatLines))
+	b.Start(t.Context())
+
+	select {
+	case <-arrived:
+	case <-time.After(maxWait):
+		t.Fatalf("no download began within %s", maxWait)
+	}
+
+	ctx, cancel := context.WithTimeout(t.Context(), maxWait)
+	defer cancel()
+
+	b.Stop(ctx)
+
+	select {
+	case <-b.done:
+	default:
+		t.Error("Stop returned before the download goroutine stopped")
+	}
+}
+
+// TestPublicListsAreComplete checks that every public list has what a
+// download and a lookup answer need, and a unique name.
+func TestPublicListsAreComplete(t *testing.T) {
+	t.Parallel()
+
+	names := make(map[string]bool)
+
+	for _, list := range Lists() {
+		if list.Name == "" || list.Description == "" || list.URL == "" ||
+			list.Interval <= 0 || names[list.Name] {
+			t.Errorf("list %+v is incomplete or has a name used before", list)
+		}
+
+		names[list.Name] = true
+	}
+}