routewatch/internal/blocklist/blocklist_internal_test.go

package blocklist

import (
	"context"
	"errors"
	"io"
	"io/fs"
	"log/slog"
	"net/http"
	"net/http/httptest"
	"net/netip"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"
)

// The test list's name and description, and the address its good version
// lists.
const (
	testListName        = "test_list"
	testListDescription = "On the test list"
	listedAddr          = "192.0.2.1"
	goodListBody        = "# A test list\n" + listedAddr + "\n"
)

// maxWait is how long a test waits for the download goroutine.
const maxWait = 10 * time.Second

// publisher stands in for a list's publisher. It answers every request with
// its status and body, and keeps the headers of each request.
type publisher struct {
	mu      sync.Mutex
	status  int
	body    string
	headers []http.Header
}

// newPublisher starts a publisher that answers 200 with body, and returns it
// and its URL.
func newPublisher(t *testing.T, body string) (*publisher, string) {
	t.Helper()

	p := &publisher{status: http.StatusOK, body: body}
	srv := httptest.NewServer(p)
	t.Cleanup(srv.Close)

	return p, srv.URL
}

func (p *publisher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	p.mu.Lock()
	defer p.mu.Unlock()

	p.headers = append(p.headers, r.Header.Clone())
	w.WriteHeader(p.status)
	_, _ = io.WriteString(w, p.body)
}

// answer makes the publisher answer later requests with status and body.
func (p *publisher) answer(status int, body string) {
	p.mu.Lock()
	defer p.mu.Unlock()

	p.status, p.body = status, body
}

// requests returns the headers of the requests the publisher has had.
func (p *publisher) requests() []http.Header {
	p.mu.Lock()
	defer p.mu.Unlock()

	return append([]http.Header(nil), p.headers...)
}

// testList returns a list served at url, written in format.
func testList(url string, format Format) List {
	return List{
		Name:        testListName,
		Description: testListDescription,
		URL:         url,
		Interval:    time.Hour,
		Format:      format,
	}
}

func newTestBlocklists(dir string, list List) *Blocklists {
	return New(Params{
		Dir:    dir,
		Lists:  []List{list},
		Logger: slog.New(slog.DiscardHandler),
	})
}

// lookup looks addr up in b.
func lookup(t *testing.T, b *Blocklists, addr string) []Listing {
	t.Helper()

	return b.Lookup(netip.MustParseAddr(addr))
}

// TestLookupFindsListedAddresses downloads a list in each format and looks
// up addresses on it, inside its networks, and off it, in IPv4 and IPv6.
// A listed address has one listing that names the list; any other has an
// empty list of listings.
func TestLookupFindsListedAddresses(t *testing.T) {
	t.Parallel()

	tests := []struct {
		name     string
		format   Format
		body     string
		listed   []string
		unlisted []string
	}{
		{
			name:     "single addresses",
			format:   FormatLines,
			body:     "# comment\n192.0.2.1\n\n2001:db8::1\n",
			listed:   []string{"192.0.2.1", "::ffff:192.0.2.1", "2001:db8::1"},
			unlisted: []string{"192.0.2.2", "2001:db8::2"},
		},
		{
			name:     "networks",
			format:   FormatLines,
			body:     "# comment\n198.51.100.0/24\n2001:db8:1::/48\n",
			listed:   []string{"198.51.100.0", "198.51.100.255", "2001:db8:1::5"},
			unlisted: []string{"198.51.101.0", "2001:db8:2::1"},
		},
		{
			name:   "Spamhaus networks",
			format: FormatSpamhausJSON,
			body: `{"cidr":"203.0.113.0/25","sblid":"SBL1","rir":"arin"}` + "\n" +
				`{"cidr":"2001:db8:3::/48","sblid":"SBL2","rir":"ripencc"}` + "\n" +
				`{"type":"metadata","timestamp":1793074442,"records":2}` + "\n",
			listed:   []string{"203.0.113.1", "2001:db8:3::1"},
			unlisted: []string{"203.0.113.200", "2001:db8:4::1"},
		},
		{
			name:   "DShield networks",
			format: FormatDShield,
			body: "#   DShield.org Recommended Block List\n" +
				"192.0.2.0\t192.0.2.255\t24\t327\tEXAMPLE\tUS\tNone\n",
			listed:   []string{"192.0.2.0", "192.0.2.77"},
			unlisted: []string{"192.0.3.1", "2001:db8::1"},
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			_, url := newPublisher(t, tt.body)
			b := newTestBlocklists(t.TempDir(), testList(url, tt.format))

			err := b.download(t.Context(), b.lists[0])
			if err != nil {
				t.Fatalf("download: %v", err)
			}

			for _, addr := range tt.listed {
				listings := lookup(t, b, addr)
				if len(listings) != 1 || listings[0].Name != testListName ||
					listings[0].Description != testListDescription ||
					listings[0].Updated.IsZero() {
					t.Errorf("%s: listings %+v, want one for %s", addr,
						listings, testListName)
				}
			}

			for _, addr := range tt.unlisted {
				listings := lookup(t, b, addr)
				if listings == nil || len(listings) != 0 {
					t.Errorf("%s: listings %#v, want an empty list", addr, listings)
				}
			}
		})
	}
}

// TestFailedDownloadKeepsThePreviousCopy downloads a good version of a list,
// then one that fails. The good copy stays in memory and on disk, and no
// part of the failed download is left beside it.
func TestFailedDownloadKeepsThePreviousCopy(t *testing.T) {
	t.Parallel()

	tests := []struct {
		name   string
		status int
		body   string
	}{
		{"server error", http.StatusInternalServerError, "unavailable"},
		{"not a list", http.StatusOK, "<!DOCTYPE html>\n<html></html>\n"},
		{
			"over the size limit", http.StatusOK,
			strings.Repeat(listedAddr+"\n", maxDownloadSize/len(listedAddr)),
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()

			dir := t.TempDir()
			p, url := newPublisher(t, goodListBody)
			b := newTestBlocklists(dir, testList(url, FormatLines))

			err := b.download(t.Context(), b.lists[0])
			if err != nil {
				t.Fatalf("first download: %v", err)
			}

			before := lookup(t, b, listedAddr)

			p.answer(tt.status, tt.body)

			err = b.download(t.Context(), b.lists[0])
			if err == nil {
				t.Fatal("second download returned no error")
			}

			after := lookup(t, b, listedAddr)
			if len(after) != 1 || len(before) != 1 ||
				!after[0].Updated.Equal(before[0].Updated) {
				t.Errorf("listings %+v after the failure, want %+v", after, before)
			}

			copies := os.DirFS(dir)

			onDisk, err := fs.ReadFile(copies, testListName)
			if err != nil || string(onDisk) != goodListBody {
				t.Errorf("copy on disk %q (%v), want %q", onDisk, err, goodListBody)
			}

			_, err = fs.Stat(copies, testListName+downloadSuffix)
			if !errors.Is(err, fs.ErrNotExist) {
				t.Errorf("the failed download's file is still there (%v)", err)
			}
		})
	}
}

// TestRestartLoadsCopiesFromDisk downloads a list, then starts a new
// Blocklists on the same directory, as a restart does. The new one answers
// from the copy on disk at once, and does not download the list again while
// the copy is younger than the list's interval.
func TestRestartLoadsCopiesFromDisk(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()
	p, url := newPublisher(t, goodListBody)
	list := testList(url, FormatLines)

	first := newTestBlocklists(dir, list)

	err := first.download(t.Context(), list)
	if err != nil {
		t.Fatalf("download: %v", err)
	}

	downloaded := lookup(t, first, listedAddr)[0].Updated

	second := newTestBlocklists(dir, list)
	second.Start(t.Context())
	second.Stop(t.Context())

	listings := lookup(t, second, listedAddr)
	if len(listings) != 1 ||
		listings[0].Updated.Sub(downloaded).Abs() > time.Second {
		t.Errorf("after the restart, listings %+v, want one updated at %s",
			listings, downloaded)
	}

	if got := len(p.requests()); got != 1 {
		t.Errorf("the publisher had %d requests, want 1", got)
	}
}

// TestStartRefreshesAnOldCopy starts on a copy older than its list's
// interval. The list is downloaded at once, with routewatch's User-Agent and
// the copy's time in If-Modified-Since. The publisher answers that the copy
// is still current, so the copy is kept and marked as updated now.
func TestStartRefreshesAnOldCopy(t *testing.T) {
	t.Parallel()

	dir := t.TempDir()
	p, url := newPublisher(t, goodListBody)
	list := testList(url, FormatLines)

	err := newTestBlocklists(dir, list).download(t.Context(), list)
	if err != nil {
		t.Fatalf("download: %v", err)
	}

	old := time.Now().Add(-2 * list.Interval).UTC().Truncate(time.Second)

	err = os.Chtimes(filepath.Join(dir, testListName), old, old)
	if err != nil {
		t.Fatalf("Chtimes: %v", err)
	}

	p.answer(http.StatusNotModified, "")

	b := newTestBlocklists(dir, list)
	b.Start(t.Context())

	deadline := time.Now().Add(maxWait)
	for !lookup(t, b, listedAddr)[0].Updated.After(old) {
		if time.Now().After(deadline) {
			t.Fatalf("the copy was not marked updated within %s", maxWait)
		}

		time.Sleep(time.Millisecond)
	}

	b.Stop(t.Context())

	requests := p.requests()
	last := requests[len(requests)-1]

	if got := last.Get(ifModifiedSinceHeader); got != old.Format(http.TimeFormat) {
		t.Errorf("If-Modified-Since %q, want %q", got,
			old.Format(http.TimeFormat))
	}

	if got := last.Get(userAgentHeader); got != userAgent {
		t.Errorf("User-Agent %q, want %q", got, userAgent)
	}
}

// TestStopCancelsADownloadInProgress stops while the publisher has not
// answered. Stop cancels the download and returns once the download
// goroutine has stopped.
func TestStopCancelsADownloadInProgress(t *testing.T) {
	t.Parallel()

	arrived := make(chan struct{})

	var once sync.Once

	srv := httptest.NewServer(http.HandlerFunc(
		func(_ http.ResponseWriter, r *http.Request) {
			once.Do(func() { close(arrived) })
			<-r.Context().Done()
		}))
	t.Cleanup(srv.Close)

	b := newTestBlocklists(t.TempDir(), testList(srv.URL, FormatLines))
	b.Start(t.Context())

	select {
	case <-arrived:
	case <-time.After(maxWait):
		t.Fatalf("no download began within %s", maxWait)
	}

	ctx, cancel := context.WithTimeout(t.Context(), maxWait)
	defer cancel()

	b.Stop(ctx)

	select {
	case <-b.done:
	default:
		t.Error("Stop returned before the download goroutine stopped")
	}
}

// TestPublicListsAreComplete checks that every public list has what a
// download and a lookup answer need, and a unique name.
func TestPublicListsAreComplete(t *testing.T) {
	t.Parallel()

	names := make(map[string]bool)

	for _, list := range Lists() {
		if list.Name == "" || list.Description == "" || list.URL == "" ||
			list.Interval <= 0 || names[list.Name] {
			t.Errorf("list %+v is incomplete or has a name used before", list)
		}

		names[list.Name] = true
	}
}