Report which public blocklists list an address in the IP lookup (closes #158)

A new package, internal/blocklist, downloads seven public lists on a
schedule: DShield, Feodo Tracker, FireHOL abusers and level 3, Spamhaus
DROP for IPv4 and IPv6, and the Tor exit list. It keeps the last good
copy of each in memory and in the state directory's blocklists
directory, and loads the copies at startup. Each list's interval follows
its publisher's page, cited in the code.

Each IP lookup result carries blocklists, one entry per list the address
is on. A successful lookup answer carries Cache-Control, private for the
caller's own address. The address sets come from go4.org/netipx.

Model: opus-5-5

internal/blocklist/blocklist.go

@@ -0,0 +1,172 @@
+// Package blocklist downloads public IP blocklists on a schedule, keeps the
+// last good copy of each on disk and in memory, and answers which lists
+// contain an address.
+package blocklist
+
+import (
+	"context"
+	"log/slog"
+	"net/netip"
+	"sync"
+	"time"
+
+	"go4.org/netipx"
+)
+
+// Format is how a list's file is written.
+type Format int
+
+const (
+	// FormatLines has one address or network per line, such as 192.0.2.1 or
+	// 192.0.2.0/24. Lines that start with # are comments.
+	FormatLines Format = iota
+
+	// FormatSpamhausJSON has one JSON object per line, with a network in its
+	// cidr field. The last line holds the file's metadata and no network.
+	FormatSpamhausJSON
+
+	// FormatDShield has one network per line, as tab-separated fields that
+	// start with the network's first and last address. Lines that start
+	// with # are comments.
+	FormatDShield
+)
+
+// List is one public blocklist.
+type List struct {
+	// Name names the list in lookup answers, and names its copy on disk.
+	Name string
+
+	// Description says in one line what being on the list means.
+	Description string
+
+	// URL is where the publisher serves the whole list.
+	URL string
+
+	// Interval is how long a copy is used before the list is downloaded
+	// again.
+	Interval time.Duration
+
+	// Format is how the list's file is written.
+	Format Format
+}
+
+// Listing says that a list contains an address.
+type Listing struct {
+	Name        string `json:"name"`
+	Description string `json:"description"`
+
+	// Updated is when routewatch last downloaded the list, or last heard from
+	// its publisher that its copy was still current.
+	Updated time.Time `json:"updated"`
+}
+
+// listCopy is a list's last good copy in memory.
+type listCopy struct {
+	addresses *netipx.IPSet
+	updated   time.Time
+}
+
+// Params is what New needs.
+type Params struct {
+	// Dir is the directory that holds the copies on disk.
+	Dir string
+
+	// Lists are the lists to download and look addresses up in.
+	Lists []List
+
+	Logger *slog.Logger
+}
+
+// Blocklists keeps a copy of each list and answers which lists contain an
+// address. Start loads the copies from disk and starts downloading the lists.
+type Blocklists struct {
+	dir    string
+	lists  []List
+	logger *slog.Logger
+
+	// copies holds each list's last good copy, by list name. A list with no
+	// good copy yet has none.
+	mu     sync.RWMutex
+	copies map[string]listCopy
+
+	// stop ends the goroutine Start runs, and done is closed when it returns.
+	stop context.CancelFunc
+	done chan struct{}
+}
+
+// New returns a Blocklists with no copies. Start loads them.
+func New(params Params) *Blocklists {
+	return &Blocklists{
+		dir:    params.Dir,
+		lists:  params.Lists,
+		logger: params.Logger.With("component", "blocklists"),
+		copies: make(map[string]listCopy),
+		done:   make(chan struct{}),
+	}
+}
+
+// Lookup returns a Listing for each list whose copy contains addr, in the
+// order of the lists given to New. It makes no outside request. When no list
+// contains addr, it returns an empty list, not nil.
+func (b *Blocklists) Lookup(addr netip.Addr) []Listing {
+	b.mu.RLock()
+	defer b.mu.RUnlock()
+
+	listings := []Listing{}
+
+	for _, list := range b.lists {
+		c, ok := b.copies[list.Name]
+		if ok && c.addresses.Contains(addr.Unmap()) {
+			listings = append(listings, Listing{
+				Name:        list.Name,
+				Description: list.Description,
+				Updated:     c.updated,
+			})
+		}
+	}
+
+	return listings
+}
+
+// ShortestInterval returns the shortest Interval of the lists. That is the
+// soonest a copy can change, and so how long a lookup answer stays current.
+func (b *Blocklists) ShortestInterval() time.Duration {
+	var shortest time.Duration
+
+	for i, list := range b.lists {
+		if i == 0 || list.Interval < shortest {
+			shortest = list.Interval
+		}
+	}
+
+	return shortest
+}
+
+// updated returns when the named list's copy was last downloaded or found
+// current, or the zero time when the list has no copy.
+func (b *Blocklists) updated(name string) time.Time {
+	b.mu.RLock()
+	defer b.mu.RUnlock()
+
+	return b.copies[name].updated
+}
+
+// setCopy makes c the named list's copy.
+func (b *Blocklists) setCopy(name string, c listCopy) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	b.copies[name] = c
+}
+
+// setUpdated records that the named list's copy was found current at t.
+func (b *Blocklists) setUpdated(name string, t time.Time) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	c, ok := b.copies[name]
+	if ok {
+		c.updated = t
+		b.copies[name] = c
+	}
+}

internal/blocklist/blocklist_internal_test.go

@@ -0,0 +1,386 @@
+package blocklist
+
+import (
+	"context"
+	"errors"
+	"io"
+	"io/fs"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// The test list's name and description, and the address its good version
+// lists.
+const (
+	testListName        = "test_list"
+	testListDescription = "On the test list"
+	listedAddr          = "192.0.2.1"
+	goodListBody        = "# A test list\n" + listedAddr + "\n"
+)
+
+// maxWait is how long a test waits for the download goroutine.
+const maxWait = 10 * time.Second
+
+// publisher stands in for a list's publisher. It answers every request with
+// its status and body, and keeps the headers of each request.
+type publisher struct {
+	mu      sync.Mutex
+	status  int
+	body    string
+	headers []http.Header
+}
+
+// newPublisher starts a publisher that answers 200 with body, and returns it
+// and its URL.
+func newPublisher(t *testing.T, body string) (*publisher, string) {
+	t.Helper()
+
+	p := &publisher{status: http.StatusOK, body: body}
+	srv := httptest.NewServer(p)
+	t.Cleanup(srv.Close)
+
+	return p, srv.URL
+}
+
+func (p *publisher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	p.headers = append(p.headers, r.Header.Clone())
+	w.WriteHeader(p.status)
+	_, _ = io.WriteString(w, p.body)
+}
+
+// answer makes the publisher answer later requests with status and body.
+func (p *publisher) answer(status int, body string) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	p.status, p.body = status, body
+}
+
+// requests returns the headers of the requests the publisher has had.
+func (p *publisher) requests() []http.Header {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	return append([]http.Header(nil), p.headers...)
+}
+
+// testList returns a list served at url, written in format.
+func testList(url string, format Format) List {
+	return List{
+		Name:        testListName,
+		Description: testListDescription,
+		URL:         url,
+		Interval:    time.Hour,
+		Format:      format,
+	}
+}
+
+func newTestBlocklists(dir string, list List) *Blocklists {
+	return New(Params{
+		Dir:    dir,
+		Lists:  []List{list},
+		Logger: slog.New(slog.DiscardHandler),
+	})
+}
+
+// lookup looks addr up in b.
+func lookup(t *testing.T, b *Blocklists, addr string) []Listing {
+	t.Helper()
+
+	return b.Lookup(netip.MustParseAddr(addr))
+}
+
+// TestLookupFindsListedAddresses downloads a list in each format and looks
+// up addresses on it, inside its networks, and off it, in IPv4 and IPv6.
+// A listed address has one listing that names the list; any other has an
+// empty list of listings.
+func TestLookupFindsListedAddresses(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name     string
+		format   Format
+		body     string
+		listed   []string
+		unlisted []string
+	}{
+		{
+			name:     "single addresses",
+			format:   FormatLines,
+			body:     "# comment\n192.0.2.1\n\n2001:db8::1\n",
+			listed:   []string{"192.0.2.1", "::ffff:192.0.2.1", "2001:db8::1"},
+			unlisted: []string{"192.0.2.2", "2001:db8::2"},
+		},
+		{
+			name:     "networks",
+			format:   FormatLines,
+			body:     "# comment\n198.51.100.0/24\n2001:db8:1::/48\n",
+			listed:   []string{"198.51.100.0", "198.51.100.255", "2001:db8:1::5"},
+			unlisted: []string{"198.51.101.0", "2001:db8:2::1"},
+		},
+		{
+			name:   "Spamhaus networks",
+			format: FormatSpamhausJSON,
+			body: `{"cidr":"203.0.113.0/25","sblid":"SBL1","rir":"arin"}` + "\n" +
+				`{"cidr":"2001:db8:3::/48","sblid":"SBL2","rir":"ripencc"}` + "\n" +
+				`{"type":"metadata","timestamp":1793074442,"records":2}` + "\n",
+			listed:   []string{"203.0.113.1", "2001:db8:3::1"},
+			unlisted: []string{"203.0.113.200", "2001:db8:4::1"},
+		},
+		{
+			name:   "DShield networks",
+			format: FormatDShield,
+			body: "#   DShield.org Recommended Block List\n" +
+				"192.0.2.0\t192.0.2.255\t24\t327\tEXAMPLE\tUS\tNone\n",
+			listed:   []string{"192.0.2.0", "192.0.2.77"},
+			unlisted: []string{"192.0.3.1", "2001:db8::1"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			_, url := newPublisher(t, tt.body)
+			b := newTestBlocklists(t.TempDir(), testList(url, tt.format))
+
+			err := b.download(t.Context(), b.lists[0])
+			if err != nil {
+				t.Fatalf("download: %v", err)
+			}
+
+			for _, addr := range tt.listed {
+				listings := lookup(t, b, addr)
+				if len(listings) != 1 || listings[0].Name != testListName ||
+					listings[0].Description != testListDescription ||
+					listings[0].Updated.IsZero() {
+					t.Errorf("%s: listings %+v, want one for %s", addr,
+						listings, testListName)
+				}
+			}
+
+			for _, addr := range tt.unlisted {
+				listings := lookup(t, b, addr)
+				if listings == nil || len(listings) != 0 {
+					t.Errorf("%s: listings %#v, want an empty list", addr, listings)
+				}
+			}
+		})
+	}
+}
+
+// TestFailedDownloadKeepsThePreviousCopy downloads a good version of a list,
+// then one that fails. The good copy stays in memory and on disk, and no
+// part of the failed download is left beside it.
+func TestFailedDownloadKeepsThePreviousCopy(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name   string
+		status int
+		body   string
+	}{
+		{"server error", http.StatusInternalServerError, "unavailable"},
+		{"not a list", http.StatusOK, "<!DOCTYPE html>\n<html></html>\n"},
+		{
+			"over the size limit", http.StatusOK,
+			strings.Repeat(listedAddr+"\n", maxDownloadSize/len(listedAddr)),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			dir := t.TempDir()
+			p, url := newPublisher(t, goodListBody)
+			b := newTestBlocklists(dir, testList(url, FormatLines))
+
+			err := b.download(t.Context(), b.lists[0])
+			if err != nil {
+				t.Fatalf("first download: %v", err)
+			}
+
+			before := lookup(t, b, listedAddr)
+
+			p.answer(tt.status, tt.body)
+
+			err = b.download(t.Context(), b.lists[0])
+			if err == nil {
+				t.Fatal("second download returned no error")
+			}
+
+			after := lookup(t, b, listedAddr)
+			if len(after) != 1 || len(before) != 1 ||
+				!after[0].Updated.Equal(before[0].Updated) {
+				t.Errorf("listings %+v after the failure, want %+v", after, before)
+			}
+
+			copies := os.DirFS(dir)
+
+			onDisk, err := fs.ReadFile(copies, testListName)
+			if err != nil || string(onDisk) != goodListBody {
+				t.Errorf("copy on disk %q (%v), want %q", onDisk, err, goodListBody)
+			}
+
+			_, err = fs.Stat(copies, testListName+downloadSuffix)
+			if !errors.Is(err, fs.ErrNotExist) {
+				t.Errorf("the failed download's file is still there (%v)", err)
+			}
+		})
+	}
+}
+
+// TestRestartLoadsCopiesFromDisk downloads a list, then starts a new
+// Blocklists on the same directory, as a restart does. The new one answers
+// from the copy on disk at once, and does not download the list again while
+// the copy is younger than the list's interval.
+func TestRestartLoadsCopiesFromDisk(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	p, url := newPublisher(t, goodListBody)
+	list := testList(url, FormatLines)
+
+	first := newTestBlocklists(dir, list)
+
+	err := first.download(t.Context(), list)
+	if err != nil {
+		t.Fatalf("download: %v", err)
+	}
+
+	downloaded := lookup(t, first, listedAddr)[0].Updated
+
+	second := newTestBlocklists(dir, list)
+	second.Start(t.Context())
+	second.Stop(t.Context())
+
+	listings := lookup(t, second, listedAddr)
+	if len(listings) != 1 ||
+		listings[0].Updated.Sub(downloaded).Abs() > time.Second {
+		t.Errorf("after the restart, listings %+v, want one updated at %s",
+			listings, downloaded)
+	}
+
+	if got := len(p.requests()); got != 1 {
+		t.Errorf("the publisher had %d requests, want 1", got)
+	}
+}
+
+// TestStartRefreshesAnOldCopy starts on a copy older than its list's
+// interval. The list is downloaded at once, with routewatch's User-Agent and
+// the copy's time in If-Modified-Since. The publisher answers that the copy
+// is still current, so the copy is kept and marked as updated now.
+func TestStartRefreshesAnOldCopy(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	p, url := newPublisher(t, goodListBody)
+	list := testList(url, FormatLines)
+
+	err := newTestBlocklists(dir, list).download(t.Context(), list)
+	if err != nil {
+		t.Fatalf("download: %v", err)
+	}
+
+	old := time.Now().Add(-2 * list.Interval).UTC().Truncate(time.Second)
+
+	err = os.Chtimes(filepath.Join(dir, testListName), old, old)
+	if err != nil {
+		t.Fatalf("Chtimes: %v", err)
+	}
+
+	p.answer(http.StatusNotModified, "")
+
+	b := newTestBlocklists(dir, list)
+	b.Start(t.Context())
+
+	deadline := time.Now().Add(maxWait)
+	for !lookup(t, b, listedAddr)[0].Updated.After(old) {
+		if time.Now().After(deadline) {
+			t.Fatalf("the copy was not marked updated within %s", maxWait)
+		}
+
+		time.Sleep(time.Millisecond)
+	}
+
+	b.Stop(t.Context())
+
+	requests := p.requests()
+	last := requests[len(requests)-1]
+
+	if got := last.Get(ifModifiedSinceHeader); got != old.Format(http.TimeFormat) {
+		t.Errorf("If-Modified-Since %q, want %q", got,
+			old.Format(http.TimeFormat))
+	}
+
+	if got := last.Get(userAgentHeader); got != userAgent {
+		t.Errorf("User-Agent %q, want %q", got, userAgent)
+	}
+}
+
+// TestStopCancelsADownloadInProgress stops while the publisher has not
+// answered. Stop cancels the download and returns once the download
+// goroutine has stopped.
+func TestStopCancelsADownloadInProgress(t *testing.T) {
+	t.Parallel()
+
+	arrived := make(chan struct{})
+
+	var once sync.Once
+
+	srv := httptest.NewServer(http.HandlerFunc(
+		func(_ http.ResponseWriter, r *http.Request) {
+			once.Do(func() { close(arrived) })
+			<-r.Context().Done()
+		}))
+	t.Cleanup(srv.Close)
+
+	b := newTestBlocklists(t.TempDir(), testList(srv.URL, FormatLines))
+	b.Start(t.Context())
+
+	select {
+	case <-arrived:
+	case <-time.After(maxWait):
+		t.Fatalf("no download began within %s", maxWait)
+	}
+
+	ctx, cancel := context.WithTimeout(t.Context(), maxWait)
+	defer cancel()
+
+	b.Stop(ctx)
+
+	select {
+	case <-b.done:
+	default:
+		t.Error("Stop returned before the download goroutine stopped")
+	}
+}
+
+// TestPublicListsAreComplete checks that every public list has what a
+// download and a lookup answer need, and a unique name.
+func TestPublicListsAreComplete(t *testing.T) {
+	t.Parallel()
+
+	names := make(map[string]bool)
+
+	for _, list := range Lists() {
+		if list.Name == "" || list.Description == "" || list.URL == "" ||
+			list.Interval <= 0 || names[list.Name] {
+			t.Errorf("list %+v is incomplete or has a name used before", list)
+		}
+
+		names[list.Name] = true
+	}
+}

internal/blocklist/download.go

@@ -0,0 +1,284 @@
+package blocklist
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/http"
+	"os"
+	"slices"
+	"time"
+
+	"go4.org/netipx"
+)
+
+// Limits of one download.
+const (
+	// downloadTimeout is how long one download may take.
+	downloadTimeout = time.Minute
+
+	// maxDownloadSize is the size of the largest list file accepted, in
+	// bytes. The largest list, FireHOL abusers, is about 2.5 MiB.
+	maxDownloadSize = 16 << 20
+)
+
+// The request headers a download sets.
+const (
+	userAgentHeader       = "User-Agent"
+	ifModifiedSinceHeader = "If-Modified-Since"
+
+	// userAgent names routewatch to the publishers.
+	userAgent = "routewatch (+https://git.eeqj.de/sneak/routewatch)"
+)
+
+const (
+	// downloadSuffix ends the name of the file a download is written to,
+	// beside the list's copy, until it has parsed.
+	downloadSuffix = ".download"
+
+	// dirPermissions are those of the directory that holds the copies.
+	dirPermissions = 0o750
+)
+
+var (
+	errUnexpectedStatus = errors.New("unexpected HTTP status")
+	errTooLarge         = errors.New("list is over the size limit")
+)
+
+// Start loads each list's copy from disk. It then starts the goroutine that
+// downloads each list once its copy is older than its Interval, and a list
+// with no copy at once. The goroutine runs until Stop: the end of ctx does
+// not stop it.
+func (b *Blocklists) Start(ctx context.Context) {
+	err := os.MkdirAll(b.dir, dirPermissions)
+	if err != nil {
+		b.logger.Error("Failed to create the blocklists directory",
+			"dir", b.dir, "error", err)
+	}
+
+	nextDownload := make([]time.Time, len(b.lists))
+
+	for i, list := range b.lists {
+		loadErr := b.loadCopy(list)
+		if loadErr != nil && !errors.Is(loadErr, fs.ErrNotExist) {
+			b.logger.Error("Failed to load a blocklist copy",
+				"list", list.Name, "error", loadErr)
+		}
+
+		// A list with no copy has the zero time, so it is due at once.
+		nextDownload[i] = b.updated(list.Name).Add(list.Interval)
+	}
+
+	ctx, b.stop = context.WithCancel(context.WithoutCancel(ctx))
+
+	go func() {
+		b.run(ctx, nextDownload)
+	}()
+
+	b.logger.Info("Blocklist downloads started", "dir", b.dir)
+}
+
+// Stop stops the downloads, cancelling one in progress, and waits until the
+// goroutine has returned or ctx ends. The copies in memory stay, so Lookup
+// still answers.
+func (b *Blocklists) Stop(ctx context.Context) {
+	b.stop()
+
+	select {
+	case <-b.done:
+		b.logger.Info("Blocklist downloads stopped")
+	case <-ctx.Done():
+		b.logger.Warn("Stop deadline passed before the blocklist downloads " +
+			"stopped")
+	}
+}
+
+// run downloads each list when nextDownload says, until ctx ends. After each
+// try, whether it worked or not, the list's next download is an Interval
+// later.
+func (b *Blocklists) run(ctx context.Context, nextDownload []time.Time) {
+	defer close(b.done)
+
+	if len(nextDownload) == 0 {
+		return
+	}
+
+	for {
+		soonest := slices.MinFunc(nextDownload, time.Time.Compare)
+		timer := time.NewTimer(time.Until(soonest))
+
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+
+			return
+		case <-timer.C:
+		}
+
+		for i, list := range b.lists {
+			if time.Now().Before(nextDownload[i]) {
+				continue
+			}
+
+			err := b.download(ctx, list)
+			if ctx.Err() != nil {
+				return
+			}
+
+			if err != nil {
+				b.logger.Error("Blocklist download failed; keeping the "+
+					"previous copy", "list", list.Name, "error", err)
+			}
+
+			nextDownload[i] = time.Now().Add(list.Interval)
+		}
+	}
+}
+
+// download asks the publisher for list. A new version that parses becomes
+// the list's copy, on disk and in memory. An answer that the copy is still
+// current marks the copy as updated now. Anything else leaves the copy as it
+// is and returns an error.
+func (b *Blocklists) download(ctx context.Context, list List) error {
+	ctx, cancel := context.WithTimeout(ctx, downloadTimeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, list.URL, nil)
+	if err != nil {
+		return err
+	}
+
+	req.Header.Set(userAgentHeader, userAgent)
+
+	updated := b.updated(list.Name)
+	if !updated.IsZero() {
+		req.Header.Set(ifModifiedSinceHeader, updated.Format(http.TimeFormat))
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = resp.Body.Close() }()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return b.replaceCopy(list, resp.Body)
+	case http.StatusNotModified:
+		return b.markCurrent(list)
+	default:
+		return fmt.Errorf("%w: %s", errUnexpectedStatus, resp.Status)
+	}
+}
+
+// replaceCopy writes a new version of list, read from body, beside the list's
+// copy. Only when it parses does it replace the copy, on disk and in memory.
+func (b *Blocklists) replaceCopy(list List, body io.Reader) error {
+	root, err := os.OpenRoot(b.dir)
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = root.Close() }()
+
+	downloadName := list.Name + downloadSuffix
+
+	addresses, err := writeAndParse(root, downloadName, body, list.Format)
+	if err != nil {
+		_ = root.Remove(downloadName)
+
+		return err
+	}
+
+	err = root.Rename(downloadName, list.Name)
+	if err != nil {
+		return err
+	}
+
+	b.setCopy(list.Name, listCopy{addresses: addresses, updated: time.Now().UTC()})
+	b.logger.Info("Downloaded a new version of a blocklist", "list", list.Name)
+
+	return nil
+}
+
+// writeAndParse writes body to the file name in root, up to maxDownloadSize
+// bytes, and parses the file as format.
+func writeAndParse(
+	root *os.Root, name string, body io.Reader, format Format,
+) (*netipx.IPSet, error) {
+	f, err := root.Create(name)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() { _ = f.Close() }()
+
+	written, err := io.Copy(f, io.LimitReader(body, maxDownloadSize+1))
+	if err != nil {
+		return nil, err
+	}
+
+	if written > maxDownloadSize {
+		return nil, errTooLarge
+	}
+
+	_, err = f.Seek(0, io.SeekStart)
+	if err != nil {
+		return nil, err
+	}
+
+	return parse(f, format)
+}
+
+// markCurrent records that the publisher says the list's copy is still
+// current: the copy, on disk and in memory, is marked as updated now.
+func (b *Blocklists) markCurrent(list List) error {
+	root, err := os.OpenRoot(b.dir)
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = root.Close() }()
+
+	now := time.Now().UTC()
+
+	err = root.Chtimes(list.Name, now, now)
+	if err != nil {
+		return err
+	}
+
+	b.setUpdated(list.Name, now)
+	b.logger.Debug("Blocklist copy is still current", "list", list.Name)
+
+	return nil
+}
+
+// loadCopy makes the list's copy on disk its copy in memory. The file's
+// modification time is when the copy was updated.
+func (b *Blocklists) loadCopy(list List) error {
+	f, err := os.OpenInRoot(b.dir, list.Name)
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = f.Close() }()
+
+	info, err := f.Stat()
+	if err != nil {
+		return err
+	}
+
+	addresses, err := parse(f, list.Format)
+	if err != nil {
+		return err
+	}
+
+	b.setCopy(list.Name, listCopy{
+		addresses: addresses, updated: info.ModTime().UTC(),
+	})
+
+	return nil
+}

internal/blocklist/lists.go

@@ -0,0 +1,93 @@
+package blocklist
+
+import "time"
+
+// How often each list is downloaded. No list is downloaded more often than
+// its publisher updates it or allows.
+const (
+	// DShield asks that its feeds be downloaded at most once an hour:
+	// https://isc.sans.edu/feeds_doc.html
+	dshieldInterval = time.Hour
+
+	// abuse.ch makes the list every 5 minutes and recommends a download at
+	// least every 15: https://feodotracker.abuse.ch/blocklist/
+	feodoTrackerInterval = 15 * time.Minute
+
+	// FireHOL's page for each list gives how often the list changes on
+	// average: about every 5 hours for abusers and every 3 hours for level
+	// 3. FireHOL sets no download limit.
+	// https://iplists.firehol.org/?ipset=firehol_abusers_30d
+	// https://iplists.firehol.org/?ipset=firehol_level3
+	fireHOLAbusersInterval = 5 * time.Hour
+	fireHOLLevel3Interval  = 3 * time.Hour
+
+	// Spamhaus asks that DROP be downloaded at most once a day:
+	// https://www.spamhaus.org/blocklists/do-not-route-or-peer/
+	spamhausDROPInterval = 24 * time.Hour
+
+	// Tor publishes a new exit list every hour and sets no download limit:
+	// https://collector.torproject.org/recent/exit-lists/
+	torExitsInterval = time.Hour
+)
+
+// Lists returns the public blocklists routewatch downloads. Each can be
+// downloaded whole without an account or a key. Spamhaus merged its EDROP
+// list into DROP, so DROP covers both.
+func Lists() []List {
+	return []List{
+		{
+			Name: "dshield",
+			Description: "In one of the 20 networks that DShield saw scan " +
+				"the most targets in the last three days",
+			URL:      "https://feeds.dshield.org/block.txt",
+			Interval: dshieldInterval,
+			Format:   FormatDShield,
+		},
+		{
+			Name:        "feodo_tracker",
+			Description: "A botnet command-and-control server",
+			URL:         "https://feodotracker.abuse.ch/downloads/ipblocklist.txt",
+			Interval:    feodoTrackerInterval,
+			Format:      FormatLines,
+		},
+		{
+			Name: "firehol_abusers_30d",
+			Description: "Reported for comment, form or forum spam in the " +
+				"last 30 days",
+			URL:      "https://iplists.firehol.org/files/firehol_abusers_30d.netset",
+			Interval: fireHOLAbusersInterval,
+			Format:   FormatLines,
+		},
+		{
+			Name: "firehol_level3",
+			Description: "Reported for attacks, spyware or viruses in the " +
+				"last 30 days",
+			URL:      "https://iplists.firehol.org/files/firehol_level3.netset",
+			Interval: fireHOLLevel3Interval,
+			Format:   FormatLines,
+		},
+		{
+			Name: "spamhaus_drop",
+			Description: "In an IPv4 network that Spamhaus says spammers " +
+				"or cyber criminals have hijacked or lease",
+			URL:      "https://www.spamhaus.org/drop/drop_v4.json",
+			Interval: spamhausDROPInterval,
+			Format:   FormatSpamhausJSON,
+		},
+		{
+			Name: "spamhaus_dropv6",
+			Description: "In an IPv6 network that Spamhaus says spammers " +
+				"or cyber criminals have hijacked or lease",
+			URL:      "https://www.spamhaus.org/drop/drop_v6.json",
+			Interval: spamhausDROPInterval,
+			Format:   FormatSpamhausJSON,
+		},
+		{
+			Name:        "tor_exits",
+			Description: "A Tor exit node, where traffic from the Tor network leaves",
+			URL:         "https://check.torproject.org/torbulkexitlist",
+			Interval:    torExitsInterval,
+			Format:      FormatLines,
+		},
+	}
+}

internal/blocklist/parse.go

@@ -0,0 +1,158 @@
+package blocklist
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/netip"
+	"strings"
+
+	"go4.org/netipx"
+)
+
+const (
+	// commentPrefix starts a comment line in a list file.
+	commentPrefix = "#"
+
+	// networkSeparator separates the address and the mask length of a
+	// network, as in 192.0.2.0/24.
+	networkSeparator = "/"
+
+	// dshieldFieldSeparator separates the fields of a DShield line.
+	dshieldFieldSeparator = "\t"
+
+	// spamhausMetadataType is the type of the line that ends a Spamhaus
+	// file. That line holds the file's metadata and no network.
+	spamhausMetadataType = "metadata"
+)
+
+var (
+	errUnknownFormat = errors.New("unknown list format")
+	errNoNetwork     = errors.New("no network on the line")
+	errBadRange      = errors.New("first and last address make no range")
+)
+
+// parse reads a list file written in format and returns the addresses it
+// holds. A line it cannot read fails the whole file, so a file that is not
+// the list, such as an error page, never replaces a good copy.
+func parse(r io.Reader, format Format) (*netipx.IPSet, error) {
+	var builder netipx.IPSetBuilder
+
+	scanner := bufio.NewScanner(r)
+
+	for lineNumber := 1; scanner.Scan(); lineNumber++ {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, commentPrefix) {
+			continue
+		}
+
+		err := addLine(&builder, line, format)
+		if err != nil {
+			return nil, fmt.Errorf("line %d: %w", lineNumber, err)
+		}
+	}
+
+	err := scanner.Err()
+	if err != nil {
+		return nil, err
+	}
+
+	return builder.IPSet()
+}
+
+// addLine adds the address or network on one line of a list file to builder.
+func addLine(builder *netipx.IPSetBuilder, line string, format Format) error {
+	switch format {
+	case FormatLines:
+		return addAddressOrNetwork(builder, line)
+	case FormatSpamhausJSON:
+		return addSpamhausNetwork(builder, line)
+	case FormatDShield:
+		return addDShieldNetwork(builder, line)
+	default:
+		return errUnknownFormat
+	}
+}
+
+// addAddressOrNetwork adds an address, such as 192.0.2.1, or a network, such
+// as 192.0.2.0/24, to builder.
+func addAddressOrNetwork(builder *netipx.IPSetBuilder, text string) error {
+	if strings.Contains(text, networkSeparator) {
+		network, err := netip.ParsePrefix(text)
+		if err != nil {
+			return err
+		}
+
+		builder.AddPrefix(network)
+
+		return nil
+	}
+
+	addr, err := netip.ParseAddr(text)
+	if err != nil {
+		return err
+	}
+
+	builder.Add(addr)
+
+	return nil
+}
+
+// addSpamhausNetwork adds the network on a line of a Spamhaus file to
+// builder. The metadata line adds nothing.
+func addSpamhausNetwork(builder *netipx.IPSetBuilder, line string) error {
+	var entry struct {
+		CIDR string `json:"cidr"`
+		Type string `json:"type"`
+	}
+
+	err := json.Unmarshal([]byte(line), &entry)
+	if err != nil {
+		return err
+	}
+
+	if entry.Type == spamhausMetadataType {
+		return nil
+	}
+
+	network, err := netip.ParsePrefix(entry.CIDR)
+	if err != nil {
+		return err
+	}
+
+	builder.AddPrefix(network)
+
+	return nil
+}
+
+// addDShieldNetwork adds the network on a line of a DShield file to builder.
+// The line's first two fields are the network's first and last address.
+func addDShieldNetwork(builder *netipx.IPSetBuilder, line string) error {
+	firstField, rest, found := strings.Cut(line, dshieldFieldSeparator)
+	if !found {
+		return errNoNetwork
+	}
+
+	lastField, _, _ := strings.Cut(rest, dshieldFieldSeparator)
+
+	first, err := netip.ParseAddr(firstField)
+	if err != nil {
+		return err
+	}
+
+	last, err := netip.ParseAddr(lastField)
+	if err != nil {
+		return err
+	}
+
+	network := netipx.IPRangeFrom(first, last)
+	if !network.IsValid() {
+		return errBadRange
+	}
+
+	builder.AddRange(network)
+
+	return nil
+}