routewatch/internal/iplist/iplist_internal_test.go

package iplist

import (
	"context"
	"errors"
	"io"
	"io/fs"
	"log/slog"
	"net/http"
	"net/http/httptest"
	"net/netip"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"
)

// The test list's name and description, the address its good version lists,
// and the address a newer version lists instead.
const (
	testListName        = "test_list"
	testListDescription = "On the test list"
	listedAddr          = "192.0.2.1"
	goodListBody        = "# A test list\n" + listedAddr + "\n"
	newListedAddr       = "192.0.2.2"
	newListBody         = "# A test list\n" + newListedAddr + "\n"
)

// testETag is the ETag a publisher sends with a version of the test list.
const testETag = `"v1"`

// maxWait is how long a test waits for the download goroutine.
const maxWait = 10 * time.Second

// publisher stands in for a list's publisher. It answers every request with
// its status and body, and keeps the headers of each request. A 200 answer
// carries lastModified and etag, when set, as Last-Modified and ETag. As
// from a real publisher, a request whose If-None-Match or If-Modified-Since
// matches them is answered "not modified".
type publisher struct {
	mu           sync.Mutex
	status       int
	body         string
	lastModified time.Time
	etag         string
	headers      []http.Header
}

// newPublisher starts a publisher that answers 200 with body, and returns it
// and its URL.
func newPublisher(t *testing.T, body string) (*publisher, string) {
	t.Helper()

	p := &publisher{status: http.StatusOK, body: body}
	srv := httptest.NewServer(p)
	t.Cleanup(srv.Close)

	return p, srv.URL
}

func (p *publisher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	p.mu.Lock()
	defer p.mu.Unlock()

	p.headers = append(p.headers, r.Header.Clone())

	if p.status != http.StatusOK {
		w.WriteHeader(p.status)
		_, _ = io.WriteString(w, p.body)

		return
	}

	if p.etag != "" {
		w.Header().Set(etagHeader, p.etag)
	}

	// ServeContent sends Last-Modified unless lastModified is zero, and
	// answers "not modified" when the request's conditions match.
	http.ServeContent(w, r, testListName, p.lastModified,
		strings.NewReader(p.body))
}

// answer makes the publisher answer later requests with status and body.
func (p *publisher) answer(status int, body string) {
	p.mu.Lock()
	defer p.mu.Unlock()

	p.status, p.body = status, body
}

// publish makes the publisher answer later requests with a version of the
// list: body, with lastModified and etag. A zero lastModified or an empty
// etag is not sent.
func (p *publisher) publish(body string, lastModified time.Time, etag string) {
	p.mu.Lock()
	defer p.mu.Unlock()

	p.status, p.body = http.StatusOK, body
	p.lastModified, p.etag = lastModified, etag
}

// requests returns the headers of the requests the publisher has had.
func (p *publisher) requests() []http.Header {
	p.mu.Lock()
	defer p.mu.Unlock()

	return append([]http.Header(nil), p.headers...)
}

// testList returns a list served at url, written in format.
func testList(url string, format Format) List {
	return List{
		Name:        testListName,
		Description: testListDescription,
		URL:         url,
		Interval:    time.Hour,
		Format:      format,
	}
}

func newTestIPLists(dir string, list List) *IPLists {
	return New(Params{
		Dir:    dir,
		Lists:  []List{list},
		Logger: slog.New(slog.DiscardHandler),
	})
}

// lookup looks addr up in b.
func lookup(t *testing.T, b *IPLists, addr string) []Listing {
	t.Helper()

	return b.Lookup(netip.MustParseAddr(addr))
}

// TestLookupFindsListedAddresses downloads a list in each format and looks
// up addresses on it, inside its networks, and off it, in IPv4 and IPv6.
// A listed address has one listing that names the list; any other has an
// empty list of listings.
func TestLookupFindsListedAddresses(t *testing.T) {
	t.Parallel()

	tests := []struct {
		name     string
		format   Format
		body     string
		listed   []string
		unlisted []string
	}{
		{
			name:     "single addresses",
			format:   FormatLines,
			body:     "# comment\n192.0.2.1\n\n2001:db8::1\n",
			listed:   []string{"192.0.2.1", "::ffff:192.0.2.1", "2001:db8::1"},
			unlisted: []string{"192.0.2.2", "2001:db8::2"},
		},
		{
			name:     "networks",
			format:   FormatLines,
			body:     "# comment\n198.51.100.0/24\n2001:db8:1::/48\n",
			listed:   []string{"198.51.100.0", "198.51.100.255", "2001:db8:1::5"},
			unlisted: []string{"198.51.101.0", "2001:db8:2::1"},
		},
		{
			name:   "Spamhaus networks",
			format: FormatSpamhausJSON,
			body: `{"cidr":"203.0.113.0/25","sblid":"SBL1","rir":"arin"}` + "\n" +
				`{"cidr":"2001:db8:3::/48","sblid":"SBL2","rir":"ripencc"}` + "\n" +
				`{"type":"metadata","timestamp":1793074442,"records":2}` + "\n",
			listed:   []string{"203.0.113.1", "2001:db8:3::1"},
			unlisted: []string{"203.0.113.200", "2001:db8:4::1"},
		},
		{
			name:   "DShield networks",
			format: FormatDShield,
			body: "#   DShield.org Recommended Block List\n" +
				"192.0.2.0\t192.0.2.255\t24\t327\tEXAMPLE\tUS\tNone\n",
			listed:   []string{"192.0.2.0", "192.0.2.77"},
			unlisted: []string{"192.0.3.1", "2001:db8::1"},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			_, url := newPublisher(t, tt.body)
			b := newTestIPLists(t.TempDir(), testList(url, tt.format))

			err := b.download(t.Context(), b.lists[0])
			if err != nil {
				t.Fatalf("download: %v", err)
			}

			for _, addr := range tt.listed {
				listings := lookup(t, b, addr)
				if len(listings) != 1 || listings[0].Name != testListName ||
					listings[0].Description != testListDescription ||
					listings[0].Updated.IsZero() {
					t.Errorf("%s: listings %+v, want one for %s", addr,
						listings, testListName)
				}
			}

			for _, addr := range tt.unlisted {
				listings := lookup(t, b, addr)
				if listings == nil || len(listings) != 0 {
					t.Errorf("%s: listings %#v, want an empty list", addr, listings)
				}
			}
		})
	}
}

// TestFailedDownloadKeepsThePreviousCopy downloads a good version of a list,
// then one that fails. The good copy stays in memory and on disk, and no
// part of the failed download is left beside it.
func TestFailedDownloadKeepsThePreviousCopy(t *testing.T) {
	t.Parallel()

	tests := []struct {
		name   string
		status int
		body   string
	}{
		{"server error", http.StatusInternalServerError, "unavailable"},
		{"not a list", http.StatusOK, "<!DOCTYPE html>\n<html></html>\n"},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			dir := t.TempDir()
			p, url := newPublisher(t, goodListBody)
			b := newTestIPLists(dir, testList(url, FormatLines))

			err := b.download(t.Context(), b.lists[0])
			if err != nil {
				t.Fatalf("first download: %v", err)
			}

			before := lookup(t, b, listedAddr)

			p.answer(tt.status, tt.body)

			err = b.download(t.Context(), b.lists[0])
			if err == nil {
				t.Fatal("second download returned no error")
			}

			after := lookup(t, b, listedAddr)
			if len(after) != 1 || len(before) != 1 ||
				!after[0].Updated.Equal(before[0].Updated) {
				t.Errorf("listings %+v after the failure, want %+v", after, before)
			}

			checkGoodCopyOnDisk(t, dir)
		})
	}
}

// checkGoodCopyOnDisk checks that dir holds the good version of the test list
// and no part of a failed download.
func checkGoodCopyOnDisk(t *testing.T, dir string) {
	t.Helper()

	copies := os.DirFS(dir)

	onDisk, err := fs.ReadFile(copies, testListName)
	if err != nil || string(onDisk) != goodListBody {
		t.Errorf("copy on disk %q (%v), want %q", onDisk, err, goodListBody)
	}

	_, err = fs.Stat(copies, testListName+downloadSuffix)
	if !errors.Is(err, fs.ErrNotExist) {
		t.Errorf("the failed download's file is still there (%v)", err)
	}
}

// commentLines is an endless io.Reader of comment lines. It lets a test send
// a large list file without holding the file in memory.
type commentLines struct {
	offset int
}

func (c *commentLines) Read(p []byte) (int, error) {
	const line = "# a comment line that pads a large list file\n"

	for filled := 0; filled < len(p); {
		copied := copy(p[filled:], line[c.offset:])
		filled += copied
		c.offset = (c.offset + copied) % len(line)
	}

	return len(p), nil
}

// largeListFile returns a reader of a list file of size bytes: comment lines,
// then newListedAddr on the last line.
func largeListFile(size int64) io.Reader {
	lastLine := "\n" + newListedAddr + "\n"

	return io.MultiReader(
		io.LimitReader(&commentLines{}, size-int64(len(lastLine))),
		strings.NewReader(lastLine),
	)
}

// TestLargeFileUnderTheSizeLimitIsAccepted replaces a list with a 199 MiB
// file. The address on its last line is listed, so the whole file was read.
func TestLargeFileUnderTheSizeLimitIsAccepted(t *testing.T) {
	t.Parallel()

	const size = 199 << 20

	b := newTestIPLists(t.TempDir(), testList("", FormatLines))

	err := b.replaceCopy(b.lists[0], largeListFile(size), validators{})
	if err != nil {
		t.Fatalf("replaceCopy: %v", err)
	}

	if listings := lookup(t, b, newListedAddr); len(listings) != 1 {
		t.Errorf("listings %+v, want one", listings)
	}
}

// TestFileOverTheSizeLimitKeepsThePreviousCopy downloads a good version of a
// list, then replaces it with a file one byte over the limit. The attempt
// fails, and the good copy stays in memory and on disk.
func TestFileOverTheSizeLimitKeepsThePreviousCopy(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()
	_, url := newPublisher(t, goodListBody)
	b := newTestIPLists(dir, testList(url, FormatLines))

	err := b.download(t.Context(), b.lists[0])
	if err != nil {
		t.Fatalf("download: %v", err)
	}

	err = b.replaceCopy(b.lists[0], largeListFile(maxDownloadSize+1), validators{})
	if !errors.Is(err, errTooLarge) {
		t.Fatalf("replaceCopy: %v, want %v", err, errTooLarge)
	}

	if listings := lookup(t, b, listedAddr); len(listings) != 1 {
		t.Errorf("listings of %s %+v, want one", listedAddr, listings)
	}

	checkGoodCopyOnDisk(t, dir)
}

// TestRestartLoadsCopiesFromDisk downloads a list, then starts a new
// IPLists on the same directory, as a restart does. The new one answers
// from the copy on disk at once, and does not download the list again while
// the copy is younger than the list's interval.
func TestRestartLoadsCopiesFromDisk(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()
	p, url := newPublisher(t, goodListBody)
	list := testList(url, FormatLines)

	first := newTestIPLists(dir, list)

	err := first.download(t.Context(), list)
	if err != nil {
		t.Fatalf("download: %v", err)
	}

	downloaded := lookup(t, first, listedAddr)[0].Updated

	second := newTestIPLists(dir, list)
	second.Start(t.Context())
	second.Stop(t.Context())

	listings := lookup(t, second, listedAddr)
	if len(listings) != 1 ||
		listings[0].Updated.Sub(downloaded).Abs() > time.Second {
		t.Errorf("after the restart, listings %+v, want one updated at %s",
			listings, downloaded)
	}

	if got := len(p.requests()); got != 1 {
		t.Errorf("the publisher had %d requests, want 1", got)
	}
}

// TestStartRefreshesAnOldCopy starts on a copy older than its list's
// interval. The list is downloaded at once, with routewatch's User-Agent and
// the Last-Modified and ETag the publisher sent with the copy, kept on disk.
// The publisher answers that the copy is still current, so the copy is kept
// and marked as updated now.
func TestStartRefreshesAnOldCopy(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()
	p, url := newPublisher(t, goodListBody)
	list := testList(url, FormatLines)

	lastModified := time.Now().Add(-3 * list.Interval).UTC().Truncate(time.Second)
	p.publish(goodListBody, lastModified, testETag)

	err := newTestIPLists(dir, list).download(t.Context(), list)
	if err != nil {
		t.Fatalf("download: %v", err)
	}

	old := time.Now().Add(-2 * list.Interval).UTC().Truncate(time.Second)

	err = os.Chtimes(filepath.Join(dir, testListName), old, old)
	if err != nil {
		t.Fatalf("Chtimes: %v", err)
	}

	b := newTestIPLists(dir, list)
	b.Start(t.Context())

	deadline := time.Now().Add(maxWait)
	for !lookup(t, b, listedAddr)[0].Updated.After(old) {
		if time.Now().After(deadline) {
			t.Fatalf("the copy was not marked updated within %s", maxWait)
		}

		time.Sleep(time.Millisecond)
	}

	b.Stop(t.Context())

	requests := p.requests()
	last := requests[len(requests)-1]

	want := lastModified.Format(http.TimeFormat)
	if got := last.Get(ifModifiedSinceHeader); got != want {
		t.Errorf("If-Modified-Since %q, want %q", got, want)
	}

	if got := last.Get(ifNoneMatchHeader); got != testETag {
		t.Errorf("If-None-Match %q, want %q", got, testETag)
	}

	if got := last.Get(userAgentHeader); got != userAgent {
		t.Errorf("User-Agent %q, want %q", got, userAgent)
	}
}

// TestNewVersionOlderThanTheLastDownloadReplacesTheCopy downloads a list.
// The publisher then puts out a new version whose Last-Modified is earlier
// than that download, as when a cache in front of the publisher passes the
// version on late. The next download sends the Last-Modified of the copy,
// not the time of the download, so it gets the new version.
func TestNewVersionOlderThanTheLastDownloadReplacesTheCopy(t *testing.T) {
	t.Parallel()

	p, url := newPublisher(t, goodListBody)
	list := testList(url, FormatLines)
	b := newTestIPLists(t.TempDir(), list)

	firstVersion := time.Now().Add(-2 * list.Interval).UTC().Truncate(time.Second)
	p.publish(goodListBody, firstVersion, "")

	err := b.download(t.Context(), list)
	if err != nil {
		t.Fatalf("first download: %v", err)
	}

	p.publish(newListBody, firstVersion.Add(list.Interval), "")

	err = b.download(t.Context(), list)
	if err != nil {
		t.Fatalf("second download: %v", err)
	}

	if listings := lookup(t, b, newListedAddr); len(listings) != 1 {
		t.Errorf("%s: listings %+v, want one from the new version",
			newListedAddr, listings)
	}

	want := firstVersion.Format(http.TimeFormat)
	if got := p.requests()[1].Get(ifModifiedSinceHeader); got != want {
		t.Errorf("If-Modified-Since %q, want %q", got, want)
	}
}

// TestDownloadWithoutLastModifiedOrETagSendsNoCondition downloads a list
// twice from a publisher that sends neither Last-Modified nor ETag. Neither
// request carries If-Modified-Since or If-None-Match, so the second download
// gets the whole list again.
func TestDownloadWithoutLastModifiedOrETagSendsNoCondition(t *testing.T) {
	t.Parallel()

	p, url := newPublisher(t, goodListBody)
	list := testList(url, FormatLines)
	b := newTestIPLists(t.TempDir(), list)

	err := b.download(t.Context(), list)
	if err != nil {
		t.Fatalf("first download: %v", err)
	}

	err = b.download(t.Context(), list)
	if err != nil {
		t.Fatalf("second download: %v", err)
	}

	for i, request := range p.requests() {
		if request.Get(ifModifiedSinceHeader) != "" ||
			request.Get(ifNoneMatchHeader) != "" {
			t.Errorf("request %d has a condition: %v", i, request)
		}
	}
}

// TestStopCancelsADownloadInProgress stops while the publisher has not
// answered. Stop cancels the download and returns once the download
// goroutine has stopped.
func TestStopCancelsADownloadInProgress(t *testing.T) {
	t.Parallel()

	arrived := make(chan struct{})

	var once sync.Once

	srv := httptest.NewServer(http.HandlerFunc(
		func(_ http.ResponseWriter, r *http.Request) {
			once.Do(func() { close(arrived) })
			<-r.Context().Done()
		}))
	t.Cleanup(srv.Close)

	b := newTestIPLists(t.TempDir(), testList(srv.URL, FormatLines))
	b.Start(t.Context())

	select {
	case <-arrived:
	case <-time.After(maxWait):
		t.Fatalf("no download began within %s", maxWait)
	}

	ctx, cancel := context.WithTimeout(t.Context(), maxWait)
	defer cancel()

	b.Stop(ctx)

	select {
	case <-b.done:
	default:
		t.Error("Stop returned before the download goroutine stopped")
	}
}

// TestPublicListsAreComplete checks that every public list has what a
// download and a lookup answer need, and a unique name.
func TestPublicListsAreComplete(t *testing.T) {
	t.Parallel()

	names := make(map[string]bool)

	for _, list := range Lists() {
		if list.Name == "" || list.Description == "" || list.URL == "" ||
			list.Interval <= 0 || names[list.Name] {
			t.Errorf("list %+v is incomplete or has a name used before", list)
		}

		names[list.Name] = true
	}
}