routewatch/internal/blocklist/blocklist_internal_test.go

package blocklist

import (
	"context"
	"errors"
	"io"
	"io/fs"
	"log/slog"
	"net/http"
	"net/http/httptest"
	"net/netip"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"
)

// The test list's name and description, the address its good version lists,
// and the address a newer version lists instead.
const (
	testListName        = "test_list"
	testListDescription = "On the test list"
	listedAddr          = "192.0.2.1"
	goodListBody        = "# A test list\n" + listedAddr + "\n"
	newListedAddr       = "192.0.2.2"
	newListBody         = "# A test list\n" + newListedAddr + "\n"
)

// testETag is the ETag a publisher sends with a version of the test list.
const testETag = `"v1"`

// maxWait is how long a test waits for the download goroutine.
const maxWait = 10 * time.Second

// publisher stands in for a list's publisher. It answers every request with
// its status and body, and keeps the headers of each request. A 200 answer
// carries lastModified and etag, when set, as Last-Modified and ETag. As
// from a real publisher, a request whose If-None-Match or If-Modified-Since
// matches them is answered "not modified".
type publisher struct {
	mu           sync.Mutex
	status       int
	body         string
	lastModified time.Time
	etag         string
	headers      []http.Header
}

// newPublisher starts a publisher that answers 200 with body, and returns it
// and its URL.
func newPublisher(t *testing.T, body string) (*publisher, string) {
	t.Helper()

	p := &publisher{status: http.StatusOK, body: body}
	srv := httptest.NewServer(p)
	t.Cleanup(srv.Close)

	return p, srv.URL
}

func (p *publisher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	p.mu.Lock()
	defer p.mu.Unlock()

	p.headers = append(p.headers, r.Header.Clone())

	if p.status != http.StatusOK {
		w.WriteHeader(p.status)
		_, _ = io.WriteString(w, p.body)

		return
	}

	if p.etag != "" {
		w.Header().Set(etagHeader, p.etag)
	}

	// ServeContent sends Last-Modified unless lastModified is zero, and
	// answers "not modified" when the request's conditions match.
	http.ServeContent(w, r, testListName, p.lastModified,
		strings.NewReader(p.body))
}

// answer makes the publisher answer later requests with status and body.
func (p *publisher) answer(status int, body string) {
	p.mu.Lock()
	defer p.mu.Unlock()

	p.status, p.body = status, body
}

// publish makes the publisher answer later requests with a version of the
// list: body, with lastModified and etag. A zero lastModified or an empty
// etag is not sent.
func (p *publisher) publish(body string, lastModified time.Time, etag string) {
	p.mu.Lock()
	defer p.mu.Unlock()

	p.status, p.body = http.StatusOK, body
	p.lastModified, p.etag = lastModified, etag
}

// requests returns the headers of the requests the publisher has had.
func (p *publisher) requests() []http.Header {
	p.mu.Lock()
	defer p.mu.Unlock()

	return append([]http.Header(nil), p.headers...)
}

// testList returns a list served at url, written in format.
func testList(url string, format Format) List {
	return List{
		Name:        testListName,
		Description: testListDescription,
		URL:         url,
		Interval:    time.Hour,
		Format:      format,
	}
}

func newTestBlocklists(dir string, list List) *Blocklists {
	return New(Params{
		Dir:    dir,
		Lists:  []List{list},
		Logger: slog.New(slog.DiscardHandler),
	})
}

// lookup looks addr up in b.
func lookup(t *testing.T, b *Blocklists, addr string) []Listing {
	t.Helper()

	return b.Lookup(netip.MustParseAddr(addr))
}

// TestLookupFindsListedAddresses downloads a list in each format and looks
// up addresses on it, inside its networks, and off it, in IPv4 and IPv6.
// A listed address has one listing that names the list; any other has an
// empty list of listings.
func TestLookupFindsListedAddresses(t *testing.T) {
	t.Parallel()

	tests := []struct {
		name     string
		format   Format
		body     string
		listed   []string
		unlisted []string
	}{
		{
			name:     "single addresses",
			format:   FormatLines,
			body:     "# comment\n192.0.2.1\n\n2001:db8::1\n",
			listed:   []string{"192.0.2.1", "::ffff:192.0.2.1", "2001:db8::1"},
			unlisted: []string{"192.0.2.2", "2001:db8::2"},
		},
		{
			name:     "networks",
			format:   FormatLines,
			body:     "# comment\n198.51.100.0/24\n2001:db8:1::/48\n",
			listed:   []string{"198.51.100.0", "198.51.100.255", "2001:db8:1::5"},
			unlisted: []string{"198.51.101.0", "2001:db8:2::1"},
		},
		{
			name:   "Spamhaus networks",
			format: FormatSpamhausJSON,
			body: `{"cidr":"203.0.113.0/25","sblid":"SBL1","rir":"arin"}` + "\n" +
				`{"cidr":"2001:db8:3::/48","sblid":"SBL2","rir":"ripencc"}` + "\n" +
				`{"type":"metadata","timestamp":1793074442,"records":2}` + "\n",
			listed:   []string{"203.0.113.1", "2001:db8:3::1"},
			unlisted: []string{"203.0.113.200", "2001:db8:4::1"},
		},
		{
			name:   "DShield networks",
			format: FormatDShield,
			body: "#   DShield.org Recommended Block List\n" +
				"192.0.2.0\t192.0.2.255\t24\t327\tEXAMPLE\tUS\tNone\n",
			listed:   []string{"192.0.2.0", "192.0.2.77"},
			unlisted: []string{"192.0.3.1", "2001:db8::1"},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			_, url := newPublisher(t, tt.body)
			b := newTestBlocklists(t.TempDir(), testList(url, tt.format))

			err := b.download(t.Context(), b.lists[0])
			if err != nil {
				t.Fatalf("download: %v", err)
			}

			for _, addr := range tt.listed {
				listings := lookup(t, b, addr)
				if len(listings) != 1 || listings[0].Name != testListName ||
					listings[0].Description != testListDescription ||
					listings[0].Updated.IsZero() {
					t.Errorf("%s: listings %+v, want one for %s", addr,
						listings, testListName)
				}
			}

			for _, addr := range tt.unlisted {
				listings := lookup(t, b, addr)
				if listings == nil || len(listings) != 0 {
					t.Errorf("%s: listings %#v, want an empty list", addr, listings)
				}
			}
		})
	}
}

// TestFailedDownloadKeepsThePreviousCopy downloads a good version of a list,
// then one that fails. The good copy stays in memory and on disk, and no
// part of the failed download is left beside it.
func TestFailedDownloadKeepsThePreviousCopy(t *testing.T) {
	t.Parallel()

	tests := []struct {
		name   string
		status int
		body   string
	}{
		{"server error", http.StatusInternalServerError, "unavailable"},
		{"not a list", http.StatusOK, "<!DOCTYPE html>\n<html></html>\n"},
		{
			"over the size limit", http.StatusOK,
			strings.Repeat(listedAddr+"\n", maxDownloadSize/len(listedAddr)),
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			dir := t.TempDir()
			p, url := newPublisher(t, goodListBody)
			b := newTestBlocklists(dir, testList(url, FormatLines))

			err := b.download(t.Context(), b.lists[0])
			if err != nil {
				t.Fatalf("first download: %v", err)
			}

			before := lookup(t, b, listedAddr)

			p.answer(tt.status, tt.body)

			err = b.download(t.Context(), b.lists[0])
			if err == nil {
				t.Fatal("second download returned no error")
			}

			after := lookup(t, b, listedAddr)
			if len(after) != 1 || len(before) != 1 ||
				!after[0].Updated.Equal(before[0].Updated) {
				t.Errorf("listings %+v after the failure, want %+v", after, before)
			}

			copies := os.DirFS(dir)

			onDisk, err := fs.ReadFile(copies, testListName)
			if err != nil || string(onDisk) != goodListBody {
				t.Errorf("copy on disk %q (%v), want %q", onDisk, err, goodListBody)
			}

			_, err = fs.Stat(copies, testListName+downloadSuffix)
			if !errors.Is(err, fs.ErrNotExist) {
				t.Errorf("the failed download's file is still there (%v)", err)
			}
		})
	}
}

// TestRestartLoadsCopiesFromDisk downloads a list, then starts a new
// Blocklists on the same directory, as a restart does. The new one answers
// from the copy on disk at once, and does not download the list again while
// the copy is younger than the list's interval.
func TestRestartLoadsCopiesFromDisk(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()
	p, url := newPublisher(t, goodListBody)
	list := testList(url, FormatLines)

	first := newTestBlocklists(dir, list)

	err := first.download(t.Context(), list)
	if err != nil {
		t.Fatalf("download: %v", err)
	}

	downloaded := lookup(t, first, listedAddr)[0].Updated

	second := newTestBlocklists(dir, list)
	second.Start(t.Context())
	second.Stop(t.Context())

	// The download takes its time after writing the file, and the restart
	// reads the file's time. A busy host can put seconds between the two.
	const maxUpdatedDifference = 15 * time.Second

	listings := lookup(t, second, listedAddr)
	if len(listings) != 1 ||
		listings[0].Updated.Sub(downloaded).Abs() > maxUpdatedDifference {
		t.Errorf("after the restart, listings %+v, want one updated at %s",
			listings, downloaded)
	}

	if got := len(p.requests()); got != 1 {
		t.Errorf("the publisher had %d requests, want 1", got)
	}
}

// TestStartRefreshesAnOldCopy starts on a copy older than its list's
// interval. The list is downloaded at once, with routewatch's User-Agent and
// the Last-Modified and ETag the publisher sent with the copy, kept on disk.
// The publisher answers that the copy is still current, so the copy is kept
// and marked as updated now.
func TestStartRefreshesAnOldCopy(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()
	p, url := newPublisher(t, goodListBody)
	list := testList(url, FormatLines)

	lastModified := time.Now().Add(-3 * list.Interval).UTC().Truncate(time.Second)
	p.publish(goodListBody, lastModified, testETag)

	err := newTestBlocklists(dir, list).download(t.Context(), list)
	if err != nil {
		t.Fatalf("download: %v", err)
	}

	old := time.Now().Add(-2 * list.Interval).UTC().Truncate(time.Second)

	err = os.Chtimes(filepath.Join(dir, testListName), old, old)
	if err != nil {
		t.Fatalf("Chtimes: %v", err)
	}

	b := newTestBlocklists(dir, list)
	b.Start(t.Context())

	deadline := time.Now().Add(maxWait)
	for !lookup(t, b, listedAddr)[0].Updated.After(old) {
		if time.Now().After(deadline) {
			t.Fatalf("the copy was not marked updated within %s", maxWait)
		}

		time.Sleep(time.Millisecond)
	}

	b.Stop(t.Context())

	requests := p.requests()
	last := requests[len(requests)-1]

	want := lastModified.Format(http.TimeFormat)
	if got := last.Get(ifModifiedSinceHeader); got != want {
		t.Errorf("If-Modified-Since %q, want %q", got, want)
	}

	if got := last.Get(ifNoneMatchHeader); got != testETag {
		t.Errorf("If-None-Match %q, want %q", got, testETag)
	}

	if got := last.Get(userAgentHeader); got != userAgent {
		t.Errorf("User-Agent %q, want %q", got, userAgent)
	}
}

// TestNewVersionOlderThanTheLastDownloadReplacesTheCopy downloads a list.
// The publisher then puts out a new version whose Last-Modified is earlier
// than that download, as when a cache in front of the publisher passes the
// version on late. The next download sends the Last-Modified of the copy,
// not the time of the download, so it gets the new version.
func TestNewVersionOlderThanTheLastDownloadReplacesTheCopy(t *testing.T) {
	t.Parallel()

	p, url := newPublisher(t, goodListBody)
	list := testList(url, FormatLines)
	b := newTestBlocklists(t.TempDir(), list)

	firstVersion := time.Now().Add(-2 * list.Interval).UTC().Truncate(time.Second)
	p.publish(goodListBody, firstVersion, "")

	err := b.download(t.Context(), list)
	if err != nil {
		t.Fatalf("first download: %v", err)
	}

	p.publish(newListBody, firstVersion.Add(list.Interval), "")

	err = b.download(t.Context(), list)
	if err != nil {
		t.Fatalf("second download: %v", err)
	}

	if listings := lookup(t, b, newListedAddr); len(listings) != 1 {
		t.Errorf("%s: listings %+v, want one from the new version",
			newListedAddr, listings)
	}

	want := firstVersion.Format(http.TimeFormat)
	if got := p.requests()[1].Get(ifModifiedSinceHeader); got != want {
		t.Errorf("If-Modified-Since %q, want %q", got, want)
	}
}

// TestDownloadWithoutLastModifiedOrETagSendsNoCondition downloads a list
// twice from a publisher that sends neither Last-Modified nor ETag. Neither
// request carries If-Modified-Since or If-None-Match, so the second download
// gets the whole list again.
func TestDownloadWithoutLastModifiedOrETagSendsNoCondition(t *testing.T) {
	t.Parallel()

	p, url := newPublisher(t, goodListBody)
	list := testList(url, FormatLines)
	b := newTestBlocklists(t.TempDir(), list)

	err := b.download(t.Context(), list)
	if err != nil {
		t.Fatalf("first download: %v", err)
	}

	err = b.download(t.Context(), list)
	if err != nil {
		t.Fatalf("second download: %v", err)
	}

	for i, request := range p.requests() {
		if request.Get(ifModifiedSinceHeader) != "" ||
			request.Get(ifNoneMatchHeader) != "" {
			t.Errorf("request %d has a condition: %v", i, request)
		}
	}
}

// TestStopCancelsADownloadInProgress stops while the publisher has not
// answered. Stop cancels the download and returns once the download
// goroutine has stopped.
func TestStopCancelsADownloadInProgress(t *testing.T) {
	t.Parallel()

	arrived := make(chan struct{})

	var once sync.Once

	srv := httptest.NewServer(http.HandlerFunc(
		func(_ http.ResponseWriter, r *http.Request) {
			once.Do(func() { close(arrived) })
			<-r.Context().Done()
		}))
	t.Cleanup(srv.Close)

	b := newTestBlocklists(t.TempDir(), testList(srv.URL, FormatLines))
	b.Start(t.Context())

	select {
	case <-arrived:
	case <-time.After(maxWait):
		t.Fatalf("no download began within %s", maxWait)
	}

	ctx, cancel := context.WithTimeout(t.Context(), maxWait)
	defer cancel()

	b.Stop(ctx)

	select {
	case <-b.done:
	default:
		t.Error("Stop returned before the download goroutine stopped")
	}
}

// TestPublicListsAreComplete checks that every public list has what a
// download and a lookup answer need, and a unique name.
func TestPublicListsAreComplete(t *testing.T) {
	t.Parallel()

	names := make(map[string]bool)

	for _, list := range Lists() {
		if list.Name == "" || list.Description == "" || list.URL == "" ||
			list.Interval <= 0 || names[list.Name] {
			t.Errorf("list %+v is incomplete or has a name used before", list)
		}

		names[list.Name] = true
	}
}