package blocklist import ( "context" "errors" "io" "io/fs" "log/slog" "net/http" "net/http/httptest" "net/netip" "os" "path/filepath" "strings" "sync" "testing" "time" ) // The test list's name and description, and the address its good version // lists. const ( testListName = "test_list" testListDescription = "On the test list" listedAddr = "192.0.2.1" goodListBody = "# A test list\n" + listedAddr + "\n" ) // maxWait is how long a test waits for the download goroutine. const maxWait = 10 * time.Second // publisher stands in for a list's publisher. It answers every request with // its status and body, and keeps the headers of each request. type publisher struct { mu sync.Mutex status int body string headers []http.Header } // newPublisher starts a publisher that answers 200 with body, and returns it // and its URL. func newPublisher(t *testing.T, body string) (*publisher, string) { t.Helper() p := &publisher{status: http.StatusOK, body: body} srv := httptest.NewServer(p) t.Cleanup(srv.Close) return p, srv.URL } func (p *publisher) ServeHTTP(w http.ResponseWriter, r *http.Request) { p.mu.Lock() defer p.mu.Unlock() p.headers = append(p.headers, r.Header.Clone()) w.WriteHeader(p.status) _, _ = io.WriteString(w, p.body) } // answer makes the publisher answer later requests with status and body. func (p *publisher) answer(status int, body string) { p.mu.Lock() defer p.mu.Unlock() p.status, p.body = status, body } // requests returns the headers of the requests the publisher has had. func (p *publisher) requests() []http.Header { p.mu.Lock() defer p.mu.Unlock() return append([]http.Header(nil), p.headers...) } // testList returns a list served at url, written in format. func testList(url string, format Format) List { return List{ Name: testListName, Description: testListDescription, URL: url, Interval: time.Hour, Format: format, } } func newTestBlocklists(dir string, list List) *Blocklists { return New(Params{ Dir: dir, Lists: []List{list}, Logger: slog.New(slog.DiscardHandler), }) } // lookup looks addr up in b. func lookup(t *testing.T, b *Blocklists, addr string) []Listing { t.Helper() return b.Lookup(netip.MustParseAddr(addr)) } // TestLookupFindsListedAddresses downloads a list in each format and looks // up addresses on it, inside its networks, and off it, in IPv4 and IPv6. // A listed address has one listing that names the list; any other has an // empty list of listings. func TestLookupFindsListedAddresses(t *testing.T) { t.Parallel() tests := []struct { name string format Format body string listed []string unlisted []string }{ { name: "single addresses", format: FormatLines, body: "# comment\n192.0.2.1\n\n2001:db8::1\n", listed: []string{"192.0.2.1", "::ffff:192.0.2.1", "2001:db8::1"}, unlisted: []string{"192.0.2.2", "2001:db8::2"}, }, { name: "networks", format: FormatLines, body: "# comment\n198.51.100.0/24\n2001:db8:1::/48\n", listed: []string{"198.51.100.0", "198.51.100.255", "2001:db8:1::5"}, unlisted: []string{"198.51.101.0", "2001:db8:2::1"}, }, { name: "Spamhaus networks", format: FormatSpamhausJSON, body: `{"cidr":"203.0.113.0/25","sblid":"SBL1","rir":"arin"}` + "\n" + `{"cidr":"2001:db8:3::/48","sblid":"SBL2","rir":"ripencc"}` + "\n" + `{"type":"metadata","timestamp":1793074442,"records":2}` + "\n", listed: []string{"203.0.113.1", "2001:db8:3::1"}, unlisted: []string{"203.0.113.200", "2001:db8:4::1"}, }, { name: "DShield networks", format: FormatDShield, body: "# DShield.org Recommended Block List\n" + "192.0.2.0\t192.0.2.255\t24\t327\tEXAMPLE\tUS\tNone\n", listed: []string{"192.0.2.0", "192.0.2.77"}, unlisted: []string{"192.0.3.1", "2001:db8::1"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() _, url := newPublisher(t, tt.body) b := newTestBlocklists(t.TempDir(), testList(url, tt.format)) err := b.download(t.Context(), b.lists[0]) if err != nil { t.Fatalf("download: %v", err) } for _, addr := range tt.listed { listings := lookup(t, b, addr) if len(listings) != 1 || listings[0].Name != testListName || listings[0].Description != testListDescription || listings[0].Updated.IsZero() { t.Errorf("%s: listings %+v, want one for %s", addr, listings, testListName) } } for _, addr := range tt.unlisted { listings := lookup(t, b, addr) if listings == nil || len(listings) != 0 { t.Errorf("%s: listings %#v, want an empty list", addr, listings) } } }) } } // TestFailedDownloadKeepsThePreviousCopy downloads a good version of a list, // then one that fails. The good copy stays in memory and on disk, and no // part of the failed download is left beside it. func TestFailedDownloadKeepsThePreviousCopy(t *testing.T) { t.Parallel() tests := []struct { name string status int body string }{ {"server error", http.StatusInternalServerError, "unavailable"}, {"not a list", http.StatusOK, "\n\n"}, { "over the size limit", http.StatusOK, strings.Repeat(listedAddr+"\n", maxDownloadSize/len(listedAddr)), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() dir := t.TempDir() p, url := newPublisher(t, goodListBody) b := newTestBlocklists(dir, testList(url, FormatLines)) err := b.download(t.Context(), b.lists[0]) if err != nil { t.Fatalf("first download: %v", err) } before := lookup(t, b, listedAddr) p.answer(tt.status, tt.body) err = b.download(t.Context(), b.lists[0]) if err == nil { t.Fatal("second download returned no error") } after := lookup(t, b, listedAddr) if len(after) != 1 || len(before) != 1 || !after[0].Updated.Equal(before[0].Updated) { t.Errorf("listings %+v after the failure, want %+v", after, before) } copies := os.DirFS(dir) onDisk, err := fs.ReadFile(copies, testListName) if err != nil || string(onDisk) != goodListBody { t.Errorf("copy on disk %q (%v), want %q", onDisk, err, goodListBody) } _, err = fs.Stat(copies, testListName+downloadSuffix) if !errors.Is(err, fs.ErrNotExist) { t.Errorf("the failed download's file is still there (%v)", err) } }) } } // TestRestartLoadsCopiesFromDisk downloads a list, then starts a new // Blocklists on the same directory, as a restart does. The new one answers // from the copy on disk at once, and does not download the list again while // the copy is younger than the list's interval. func TestRestartLoadsCopiesFromDisk(t *testing.T) { t.Parallel() dir := t.TempDir() p, url := newPublisher(t, goodListBody) list := testList(url, FormatLines) first := newTestBlocklists(dir, list) err := first.download(t.Context(), list) if err != nil { t.Fatalf("download: %v", err) } downloaded := lookup(t, first, listedAddr)[0].Updated second := newTestBlocklists(dir, list) second.Start(t.Context()) second.Stop(t.Context()) listings := lookup(t, second, listedAddr) if len(listings) != 1 || listings[0].Updated.Sub(downloaded).Abs() > time.Second { t.Errorf("after the restart, listings %+v, want one updated at %s", listings, downloaded) } if got := len(p.requests()); got != 1 { t.Errorf("the publisher had %d requests, want 1", got) } } // TestStartRefreshesAnOldCopy starts on a copy older than its list's // interval. The list is downloaded at once, with routewatch's User-Agent and // the copy's time in If-Modified-Since. The publisher answers that the copy // is still current, so the copy is kept and marked as updated now. func TestStartRefreshesAnOldCopy(t *testing.T) { t.Parallel() dir := t.TempDir() p, url := newPublisher(t, goodListBody) list := testList(url, FormatLines) err := newTestBlocklists(dir, list).download(t.Context(), list) if err != nil { t.Fatalf("download: %v", err) } old := time.Now().Add(-2 * list.Interval).UTC().Truncate(time.Second) err = os.Chtimes(filepath.Join(dir, testListName), old, old) if err != nil { t.Fatalf("Chtimes: %v", err) } p.answer(http.StatusNotModified, "") b := newTestBlocklists(dir, list) b.Start(t.Context()) deadline := time.Now().Add(maxWait) for !lookup(t, b, listedAddr)[0].Updated.After(old) { if time.Now().After(deadline) { t.Fatalf("the copy was not marked updated within %s", maxWait) } time.Sleep(time.Millisecond) } b.Stop(t.Context()) requests := p.requests() last := requests[len(requests)-1] if got := last.Get(ifModifiedSinceHeader); got != old.Format(http.TimeFormat) { t.Errorf("If-Modified-Since %q, want %q", got, old.Format(http.TimeFormat)) } if got := last.Get(userAgentHeader); got != userAgent { t.Errorf("User-Agent %q, want %q", got, userAgent) } } // TestStopCancelsADownloadInProgress stops while the publisher has not // answered. Stop cancels the download and returns once the download // goroutine has stopped. func TestStopCancelsADownloadInProgress(t *testing.T) { t.Parallel() arrived := make(chan struct{}) var once sync.Once srv := httptest.NewServer(http.HandlerFunc( func(_ http.ResponseWriter, r *http.Request) { once.Do(func() { close(arrived) }) <-r.Context().Done() })) t.Cleanup(srv.Close) b := newTestBlocklists(t.TempDir(), testList(srv.URL, FormatLines)) b.Start(t.Context()) select { case <-arrived: case <-time.After(maxWait): t.Fatalf("no download began within %s", maxWait) } ctx, cancel := context.WithTimeout(t.Context(), maxWait) defer cancel() b.Stop(ctx) select { case <-b.done: default: t.Error("Stop returned before the download goroutine stopped") } } // TestPublicListsAreComplete checks that every public list has what a // download and a lookup answer need, and a unique name. func TestPublicListsAreComplete(t *testing.T) { t.Parallel() names := make(map[string]bool) for _, list := range Lists() { if list.Name == "" || list.Description == "" || list.URL == "" || list.Interval <= 0 || names[list.Name] { t.Errorf("list %+v is incomplete or has a name used before", list) } names[list.Name] = true } }