Rename blocklists to IP lists and add IP lookup flags (closes #233)

The answer field is now ip_lists, the package internal/iplist and the
state directory iplists. No compatibility is kept for the old names.

Every IP lookup result gains a flags object with is_tor and is_bogon.
The largest list file accepted is now 200 MiB.

Model: opus-5-5

internal/iplist/download.go

@@ -0,0 +1,379 @@
+package iplist
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"net/http"
+	"os"
+	"slices"
+	"time"
+
+	"go4.org/netipx"
+)
+
+// Limits of one download.
+const (
+	// downloadTimeout is how long one download may take.
+	downloadTimeout = time.Minute
+
+	// maxDownloadSize is the size of the largest list file accepted, in
+	// bytes. The largest list, FireHOL abusers, is about 2.5 MiB.
+	maxDownloadSize = 200 << 20
+)
+
+// The request headers a download sets.
+const (
+	userAgentHeader       = "User-Agent"
+	ifModifiedSinceHeader = "If-Modified-Since"
+	ifNoneMatchHeader     = "If-None-Match"
+
+	// userAgent names routewatch to the publishers.
+	userAgent = "routewatch (+https://git.eeqj.de/sneak/routewatch)"
+)
+
+// The response headers that identify the version of a list a publisher sent.
+const (
+	lastModifiedHeader = "Last-Modified"
+	etagHeader         = "ETag"
+)
+
+const (
+	// downloadSuffix ends the name of the file a download is written to,
+	// beside the list's copy, until it has parsed.
+	downloadSuffix = ".download"
+
+	// validatorsSuffix ends the name of the file beside a list's copy that
+	// keeps the copy's validators.
+	validatorsSuffix = ".validators"
+
+	// dirPermissions are those of the directory that holds the copies.
+	dirPermissions = 0o750
+
+	// validatorsPermissions are those of the file that keeps a copy's
+	// validators.
+	validatorsPermissions = 0o600
+)
+
+// validators are the Last-Modified and ETag headers a publisher sent with a
+// version of a list. A download sends them back, as If-Modified-Since and
+// If-None-Match, so that the publisher sends the list only when it has
+// changed. They are kept as the publisher wrote them.
+//
+//nolint:tagliatelle // snake_case keys, as the Go style guide asks
+type validators struct {
+	LastModified string `json:"last_modified,omitempty"`
+	ETag         string `json:"etag,omitempty"`
+}
+
+var (
+	errUnexpectedStatus = errors.New("unexpected HTTP status")
+	errTooLarge         = errors.New("list is over the size limit")
+)
+
+// Start loads each list's copy from disk. It then starts the goroutine that
+// downloads each list once its copy is older than its Interval, and a list
+// with no copy at once. The goroutine runs until Stop: the end of ctx does
+// not stop it.
+func (b *IPLists) Start(ctx context.Context) {
+	err := os.MkdirAll(b.dir, dirPermissions)
+	if err != nil {
+		b.logger.Error("Failed to create the IP lists directory",
+			"dir", b.dir, "error", err)
+	}
+
+	nextDownload := make([]time.Time, len(b.lists))
+
+	for i, list := range b.lists {
+		loadErr := b.loadCopy(list)
+		if loadErr != nil && !errors.Is(loadErr, fs.ErrNotExist) {
+			b.logger.Error("Failed to load an IP list copy",
+				"list", list.Name, "error", loadErr)
+		}
+
+		// A list with no copy has the zero time, so it is due at once.
+		nextDownload[i] = b.updated(list.Name).Add(list.Interval)
+	}
+
+	ctx, b.stop = context.WithCancel(context.WithoutCancel(ctx))
+
+	go func() {
+		b.run(ctx, nextDownload)
+	}()
+
+	b.logger.Info("IP list downloads started", "dir", b.dir)
+}
+
+// Stop stops the downloads, cancelling one in progress, and waits until the
+// goroutine has returned or ctx ends. The copies in memory stay, so Lookup
+// still answers.
+func (b *IPLists) Stop(ctx context.Context) {
+	b.stop()
+
+	select {
+	case <-b.done:
+		b.logger.Info("IP list downloads stopped")
+	case <-ctx.Done():
+		b.logger.Warn("Stop deadline passed before the IP list downloads " +
+			"stopped")
+	}
+}
+
+// run downloads each list when nextDownload says, until ctx ends. After each
+// try, whether it worked or not, the list's next download is an Interval
+// later.
+func (b *IPLists) run(ctx context.Context, nextDownload []time.Time) {
+	defer close(b.done)
+
+	if len(nextDownload) == 0 {
+		return
+	}
+
+	for {
+		soonest := slices.MinFunc(nextDownload, time.Time.Compare)
+		timer := time.NewTimer(time.Until(soonest))
+
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+
+			return
+		case <-timer.C:
+		}
+
+		for i, list := range b.lists {
+			if time.Now().Before(nextDownload[i]) {
+				continue
+			}
+
+			err := b.download(ctx, list)
+			if ctx.Err() != nil {
+				return
+			}
+
+			if err != nil {
+				b.logger.Error("IP list download failed; keeping the "+
+					"previous copy", "list", list.Name, "error", err)
+			}
+
+			nextDownload[i] = time.Now().Add(list.Interval)
+		}
+	}
+}
+
+// download asks the publisher for list, sending the validators of the list's
+// copy when it has them. A new version that parses becomes the list's copy,
+// on disk and in memory. An answer that the copy is still current marks the
+// copy as updated now. Anything else leaves the copy as it is and returns an
+// error.
+func (b *IPLists) download(ctx context.Context, list List) error {
+	ctx, cancel := context.WithTimeout(ctx, downloadTimeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, list.URL, nil)
+	if err != nil {
+		return err
+	}
+
+	req.Header.Set(userAgentHeader, userAgent)
+
+	held := b.copyValidators(list.Name)
+	if held.LastModified != "" {
+		req.Header.Set(ifModifiedSinceHeader, held.LastModified)
+	}
+
+	if held.ETag != "" {
+		req.Header.Set(ifNoneMatchHeader, held.ETag)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = resp.Body.Close() }()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return b.replaceCopy(list, resp.Body, validators{
+			LastModified: resp.Header.Get(lastModifiedHeader),
+			ETag:         resp.Header.Get(etagHeader),
+		})
+	case http.StatusNotModified:
+		return b.markCurrent(list)
+	default:
+		return fmt.Errorf("%w: %s", errUnexpectedStatus, resp.Status)
+	}
+}
+
+// replaceCopy writes a new version of list, read from body, beside the list's
+// copy. Only when it parses does it replace the copy, on disk and in memory.
+// sent are the validators the publisher sent with the new version.
+func (b *IPLists) replaceCopy(list List, body io.Reader, sent validators) error {
+	root, err := os.OpenRoot(b.dir)
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = root.Close() }()
+
+	downloadName := list.Name + downloadSuffix
+
+	addresses, err := writeAndParse(root, downloadName, body, list.Format)
+	if err != nil {
+		_ = root.Remove(downloadName)
+
+		return err
+	}
+
+	// The old copy's validators are removed before the copy is replaced, so
+	// that they are never kept with the new copy.
+	err = root.Remove(list.Name + validatorsSuffix)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		return err
+	}
+
+	err = root.Rename(downloadName, list.Name)
+	if err != nil {
+		return err
+	}
+
+	b.setCopy(list.Name, listCopy{
+		addresses: addresses, updated: time.Now().UTC(), validators: sent,
+	})
+	b.logger.Info("Downloaded a new version of an IP list", "list", list.Name)
+
+	// Without this file, the first download after a restart asks for the
+	// whole list.
+	err = writeValidators(root, list.Name, sent)
+	if err != nil {
+		b.logger.Error("Failed to keep the validators of an IP list copy",
+			"list", list.Name, "error", err)
+	}
+
+	return nil
+}
+
+// writeAndParse writes body to the file name in root, up to maxDownloadSize
+// bytes, and parses the file as format.
+func writeAndParse(
+	root *os.Root, name string, body io.Reader, format Format,
+) (*netipx.IPSet, error) {
+	f, err := root.Create(name)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() { _ = f.Close() }()
+
+	written, err := io.Copy(f, io.LimitReader(body, maxDownloadSize+1))
+	if err != nil {
+		return nil, err
+	}
+
+	if written > maxDownloadSize {
+		return nil, errTooLarge
+	}
+
+	_, err = f.Seek(0, io.SeekStart)
+	if err != nil {
+		return nil, err
+	}
+
+	return parse(f, format)
+}
+
+// markCurrent records that the publisher says the list's copy is still
+// current: the copy, on disk and in memory, is marked as updated now.
+func (b *IPLists) markCurrent(list List) error {
+	root, err := os.OpenRoot(b.dir)
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = root.Close() }()
+
+	now := time.Now().UTC()
+
+	err = root.Chtimes(list.Name, now, now)
+	if err != nil {
+		return err
+	}
+
+	b.setUpdated(list.Name, now)
+	b.logger.Debug("IP list copy is still current", "list", list.Name)
+
+	return nil
+}
+
+// loadCopy makes the list's copy on disk, with the validators kept beside it,
+// its copy in memory. The file's modification time is when the copy was
+// updated.
+func (b *IPLists) loadCopy(list List) error {
+	root, err := os.OpenRoot(b.dir)
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = root.Close() }()
+
+	f, err := root.Open(list.Name)
+	if err != nil {
+		return err
+	}
+
+	defer func() { _ = f.Close() }()
+
+	info, err := f.Stat()
+	if err != nil {
+		return err
+	}
+
+	addresses, err := parse(f, list.Format)
+	if err != nil {
+		return err
+	}
+
+	held, err := readValidators(root, list.Name)
+	if err != nil {
+		return err
+	}
+
+	b.setCopy(list.Name, listCopy{
+		addresses: addresses, updated: info.ModTime().UTC(), validators: held,
+	})
+
+	return nil
+}
+
+// writeValidators keeps held in the file beside the named list's copy.
+func writeValidators(root *os.Root, name string, held validators) error {
+	data, err := json.Marshal(held)
+	if err != nil {
+		return err
+	}
+
+	return root.WriteFile(name+validatorsSuffix, data, validatorsPermissions)
+}
+
+// readValidators returns the validators kept beside the named list's copy.
+// They are empty when there is no such file.
+func readValidators(root *os.Root, name string) (validators, error) {
+	var held validators
+
+	data, err := root.ReadFile(name + validatorsSuffix)
+	if errors.Is(err, fs.ErrNotExist) {
+		return held, nil
+	}
+
+	if err != nil {
+		return held, err
+	}
+
+	err = json.Unmarshal(data, &held)
+
+	return held, err
+}

internal/iplist/iplist.go

@@ -0,0 +1,184 @@
+// Package iplist downloads public IP lists on a schedule, keeps the last good
+// copy of each on disk and in memory, and answers which lists contain an
+// address.
+package iplist
+
+import (
+	"context"
+	"log/slog"
+	"net/netip"
+	"sync"
+	"time"
+
+	"go4.org/netipx"
+)
+
+// Format is how a list's file is written.
+type Format int
+
+const (
+	// FormatLines has one address or network per line, such as 192.0.2.1 or
+	// 192.0.2.0/24. Lines that start with # are comments.
+	FormatLines Format = iota
+
+	// FormatSpamhausJSON has one JSON object per line, with a network in its
+	// cidr field. The last line holds the file's metadata and no network.
+	FormatSpamhausJSON
+
+	// FormatDShield has one network per line, as tab-separated fields that
+	// start with the network's first and last address. Lines that start
+	// with # are comments.
+	FormatDShield
+)
+
+// List is one public IP list.
+type List struct {
+	// Name names the list in lookup answers, and names its copy on disk.
+	Name string
+
+	// Description says in one line what being on the list means.
+	Description string
+
+	// URL is where the publisher serves the whole list.
+	URL string
+
+	// Interval is how long a copy is used before the list is downloaded
+	// again.
+	Interval time.Duration
+
+	// Format is how the list's file is written.
+	Format Format
+}
+
+// Listing says that a list contains an address.
+type Listing struct {
+	Name        string `json:"name"`
+	Description string `json:"description"`
+
+	// Updated is when routewatch last downloaded the list, or last heard from
+	// its publisher that its copy was still current.
+	Updated time.Time `json:"updated"`
+}
+
+// listCopy is a list's last good copy in memory.
+type listCopy struct {
+	addresses *netipx.IPSet
+	updated   time.Time
+
+	// validators identify the version of the list this copy holds.
+	validators validators
+}
+
+// Params is what New needs.
+type Params struct {
+	// Dir is the directory that holds the copies on disk.
+	Dir string
+
+	// Lists are the lists to download and look addresses up in.
+	Lists []List
+
+	Logger *slog.Logger
+}
+
+// IPLists keeps a copy of each list and answers which lists contain an
+// address. Start loads the copies from disk and starts downloading the lists.
+type IPLists struct {
+	dir    string
+	lists  []List
+	logger *slog.Logger
+
+	// copies holds each list's last good copy, by list name. A list with no
+	// good copy yet has none.
+	mu     sync.RWMutex
+	copies map[string]listCopy
+
+	// stop ends the goroutine Start runs, and done is closed when it returns.
+	stop context.CancelFunc
+	done chan struct{}
+}
+
+// New returns an IPLists with no copies. Start loads them.
+func New(params Params) *IPLists {
+	return &IPLists{
+		dir:    params.Dir,
+		lists:  params.Lists,
+		logger: params.Logger.With("component", "iplists"),
+		copies: make(map[string]listCopy),
+		done:   make(chan struct{}),
+	}
+}
+
+// Lookup returns a Listing for each list whose copy contains addr, in the
+// order of the lists given to New. It makes no outside request. When no list
+// contains addr, it returns an empty list, not nil.
+func (b *IPLists) Lookup(addr netip.Addr) []Listing {
+	b.mu.RLock()
+	defer b.mu.RUnlock()
+
+	listings := []Listing{}
+
+	for _, list := range b.lists {
+		c, ok := b.copies[list.Name]
+		if ok && c.addresses.Contains(addr.Unmap()) {
+			listings = append(listings, Listing{
+				Name:        list.Name,
+				Description: list.Description,
+				Updated:     c.updated,
+			})
+		}
+	}
+
+	return listings
+}
+
+// ShortestInterval returns the shortest Interval of the lists. That is the
+// soonest a copy can change, and so how long a lookup answer stays current.
+func (b *IPLists) ShortestInterval() time.Duration {
+	var shortest time.Duration
+
+	for i, list := range b.lists {
+		if i == 0 || list.Interval < shortest {
+			shortest = list.Interval
+		}
+	}
+
+	return shortest
+}
+
+// updated returns when the named list's copy was last downloaded or found
+// current, or the zero time when the list has no copy.
+func (b *IPLists) updated(name string) time.Time {
+	b.mu.RLock()
+	defer b.mu.RUnlock()
+
+	return b.copies[name].updated
+}
+
+// copyValidators returns the validators of the named list's copy. They are
+// empty when the list has no copy.
+func (b *IPLists) copyValidators(name string) validators {
+	b.mu.RLock()
+	defer b.mu.RUnlock()
+
+	return b.copies[name].validators
+}
+
+// setCopy makes c the named list's copy.
+func (b *IPLists) setCopy(name string, c listCopy) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	b.copies[name] = c
+}
+
+// setUpdated records that the named list's copy was found current at t.
+func (b *IPLists) setUpdated(name string, t time.Time) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	c, ok := b.copies[name]
+	if ok {
+		c.updated = t
+		b.copies[name] = c
+	}
+}

internal/iplist/iplist_internal_test.go

@@ -0,0 +1,573 @@
+package iplist
+
+import (
+	"context"
+	"errors"
+	"io"
+	"io/fs"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+// The test list's name and description, the address its good version lists,
+// and the address a newer version lists instead.
+const (
+	testListName        = "test_list"
+	testListDescription = "On the test list"
+	listedAddr          = "192.0.2.1"
+	goodListBody        = "# A test list\n" + listedAddr + "\n"
+	newListedAddr       = "192.0.2.2"
+	newListBody         = "# A test list\n" + newListedAddr + "\n"
+)
+
+// testETag is the ETag a publisher sends with a version of the test list.
+const testETag = `"v1"`
+
+// maxWait is how long a test waits for the download goroutine.
+const maxWait = 10 * time.Second
+
+// publisher stands in for a list's publisher. It answers every request with
+// its status and body, and keeps the headers of each request. A 200 answer
+// carries lastModified and etag, when set, as Last-Modified and ETag. As
+// from a real publisher, a request whose If-None-Match or If-Modified-Since
+// matches them is answered "not modified".
+type publisher struct {
+	mu           sync.Mutex
+	status       int
+	body         string
+	lastModified time.Time
+	etag         string
+	headers      []http.Header
+}
+
+// newPublisher starts a publisher that answers 200 with body, and returns it
+// and its URL.
+func newPublisher(t *testing.T, body string) (*publisher, string) {
+	t.Helper()
+
+	p := &publisher{status: http.StatusOK, body: body}
+	srv := httptest.NewServer(p)
+	t.Cleanup(srv.Close)
+
+	return p, srv.URL
+}
+
+func (p *publisher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	p.headers = append(p.headers, r.Header.Clone())
+
+	if p.status != http.StatusOK {
+		w.WriteHeader(p.status)
+		_, _ = io.WriteString(w, p.body)
+
+		return
+	}
+
+	if p.etag != "" {
+		w.Header().Set(etagHeader, p.etag)
+	}
+
+	// ServeContent sends Last-Modified unless lastModified is zero, and
+	// answers "not modified" when the request's conditions match.
+	http.ServeContent(w, r, testListName, p.lastModified,
+		strings.NewReader(p.body))
+}
+
+// answer makes the publisher answer later requests with status and body.
+func (p *publisher) answer(status int, body string) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	p.status, p.body = status, body
+}
+
+// publish makes the publisher answer later requests with a version of the
+// list: body, with lastModified and etag. A zero lastModified or an empty
+// etag is not sent.
+func (p *publisher) publish(body string, lastModified time.Time, etag string) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	p.status, p.body = http.StatusOK, body
+	p.lastModified, p.etag = lastModified, etag
+}
+
+// requests returns the headers of the requests the publisher has had.
+func (p *publisher) requests() []http.Header {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	return append([]http.Header(nil), p.headers...)
+}
+
+// testList returns a list served at url, written in format.
+func testList(url string, format Format) List {
+	return List{
+		Name:        testListName,
+		Description: testListDescription,
+		URL:         url,
+		Interval:    time.Hour,
+		Format:      format,
+	}
+}
+
+func newTestIPLists(dir string, list List) *IPLists {
+	return New(Params{
+		Dir:    dir,
+		Lists:  []List{list},
+		Logger: slog.New(slog.DiscardHandler),
+	})
+}
+
+// lookup looks addr up in b.
+func lookup(t *testing.T, b *IPLists, addr string) []Listing {
+	t.Helper()
+
+	return b.Lookup(netip.MustParseAddr(addr))
+}
+
+// TestLookupFindsListedAddresses downloads a list in each format and looks
+// up addresses on it, inside its networks, and off it, in IPv4 and IPv6.
+// A listed address has one listing that names the list; any other has an
+// empty list of listings.
+func TestLookupFindsListedAddresses(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name     string
+		format   Format
+		body     string
+		listed   []string
+		unlisted []string
+	}{
+		{
+			name:     "single addresses",
+			format:   FormatLines,
+			body:     "# comment\n192.0.2.1\n\n2001:db8::1\n",
+			listed:   []string{"192.0.2.1", "::ffff:192.0.2.1", "2001:db8::1"},
+			unlisted: []string{"192.0.2.2", "2001:db8::2"},
+		},
+		{
+			name:     "networks",
+			format:   FormatLines,
+			body:     "# comment\n198.51.100.0/24\n2001:db8:1::/48\n",
+			listed:   []string{"198.51.100.0", "198.51.100.255", "2001:db8:1::5"},
+			unlisted: []string{"198.51.101.0", "2001:db8:2::1"},
+		},
+		{
+			name:   "Spamhaus networks",
+			format: FormatSpamhausJSON,
+			body: `{"cidr":"203.0.113.0/25","sblid":"SBL1","rir":"arin"}` + "\n" +
+				`{"cidr":"2001:db8:3::/48","sblid":"SBL2","rir":"ripencc"}` + "\n" +
+				`{"type":"metadata","timestamp":1793074442,"records":2}` + "\n",
+			listed:   []string{"203.0.113.1", "2001:db8:3::1"},
+			unlisted: []string{"203.0.113.200", "2001:db8:4::1"},
+		},
+		{
+			name:   "DShield networks",
+			format: FormatDShield,
+			body: "#   DShield.org Recommended Block List\n" +
+				"192.0.2.0\t192.0.2.255\t24\t327\tEXAMPLE\tUS\tNone\n",
+			listed:   []string{"192.0.2.0", "192.0.2.77"},
+			unlisted: []string{"192.0.3.1", "2001:db8::1"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			_, url := newPublisher(t, tt.body)
+			b := newTestIPLists(t.TempDir(), testList(url, tt.format))
+
+			err := b.download(t.Context(), b.lists[0])
+			if err != nil {
+				t.Fatalf("download: %v", err)
+			}
+
+			for _, addr := range tt.listed {
+				listings := lookup(t, b, addr)
+				if len(listings) != 1 || listings[0].Name != testListName ||
+					listings[0].Description != testListDescription ||
+					listings[0].Updated.IsZero() {
+					t.Errorf("%s: listings %+v, want one for %s", addr,
+						listings, testListName)
+				}
+			}
+
+			for _, addr := range tt.unlisted {
+				listings := lookup(t, b, addr)
+				if listings == nil || len(listings) != 0 {
+					t.Errorf("%s: listings %#v, want an empty list", addr, listings)
+				}
+			}
+		})
+	}
+}
+
+// TestFailedDownloadKeepsThePreviousCopy downloads a good version of a list,
+// then one that fails. The good copy stays in memory and on disk, and no
+// part of the failed download is left beside it.
+func TestFailedDownloadKeepsThePreviousCopy(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name   string
+		status int
+		body   string
+	}{
+		{"server error", http.StatusInternalServerError, "unavailable"},
+		{"not a list", http.StatusOK, "<!DOCTYPE html>\n<html></html>\n"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			dir := t.TempDir()
+			p, url := newPublisher(t, goodListBody)
+			b := newTestIPLists(dir, testList(url, FormatLines))
+
+			err := b.download(t.Context(), b.lists[0])
+			if err != nil {
+				t.Fatalf("first download: %v", err)
+			}
+
+			before := lookup(t, b, listedAddr)
+
+			p.answer(tt.status, tt.body)
+
+			err = b.download(t.Context(), b.lists[0])
+			if err == nil {
+				t.Fatal("second download returned no error")
+			}
+
+			after := lookup(t, b, listedAddr)
+			if len(after) != 1 || len(before) != 1 ||
+				!after[0].Updated.Equal(before[0].Updated) {
+				t.Errorf("listings %+v after the failure, want %+v", after, before)
+			}
+
+			checkGoodCopyOnDisk(t, dir)
+		})
+	}
+}
+
+// checkGoodCopyOnDisk checks that dir holds the good version of the test list
+// and no part of a failed download.
+func checkGoodCopyOnDisk(t *testing.T, dir string) {
+	t.Helper()
+
+	copies := os.DirFS(dir)
+
+	onDisk, err := fs.ReadFile(copies, testListName)
+	if err != nil || string(onDisk) != goodListBody {
+		t.Errorf("copy on disk %q (%v), want %q", onDisk, err, goodListBody)
+	}
+
+	_, err = fs.Stat(copies, testListName+downloadSuffix)
+	if !errors.Is(err, fs.ErrNotExist) {
+		t.Errorf("the failed download's file is still there (%v)", err)
+	}
+}
+
+// commentLines is an endless io.Reader of comment lines. It lets a test send
+// a large list file without holding the file in memory.
+type commentLines struct {
+	offset int
+}
+
+func (c *commentLines) Read(p []byte) (int, error) {
+	const line = "# a comment line that pads a large list file\n"
+
+	for filled := 0; filled < len(p); {
+		copied := copy(p[filled:], line[c.offset:])
+		filled += copied
+		c.offset = (c.offset + copied) % len(line)
+	}
+
+	return len(p), nil
+}
+
+// largeListFile returns a reader of a list file of size bytes: comment lines,
+// then newListedAddr on the last line.
+func largeListFile(size int64) io.Reader {
+	lastLine := "\n" + newListedAddr + "\n"
+
+	return io.MultiReader(
+		io.LimitReader(&commentLines{}, size-int64(len(lastLine))),
+		strings.NewReader(lastLine),
+	)
+}
+
+// TestLargeFileUnderTheSizeLimitIsAccepted replaces a list with a 199 MiB
+// file. The address on its last line is listed, so the whole file was read.
+func TestLargeFileUnderTheSizeLimitIsAccepted(t *testing.T) {
+	t.Parallel()
+
+	const size = 199 << 20
+
+	b := newTestIPLists(t.TempDir(), testList("", FormatLines))
+
+	err := b.replaceCopy(b.lists[0], largeListFile(size), validators{})
+	if err != nil {
+		t.Fatalf("replaceCopy: %v", err)
+	}
+
+	if listings := lookup(t, b, newListedAddr); len(listings) != 1 {
+		t.Errorf("listings %+v, want one", listings)
+	}
+}
+
+// TestFileOverTheSizeLimitKeepsThePreviousCopy downloads a good version of a
+// list, then replaces it with a file one byte over the limit. The attempt
+// fails, and the good copy stays in memory and on disk.
+func TestFileOverTheSizeLimitKeepsThePreviousCopy(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	_, url := newPublisher(t, goodListBody)
+	b := newTestIPLists(dir, testList(url, FormatLines))
+
+	err := b.download(t.Context(), b.lists[0])
+	if err != nil {
+		t.Fatalf("download: %v", err)
+	}
+
+	err = b.replaceCopy(b.lists[0], largeListFile(maxDownloadSize+1), validators{})
+	if !errors.Is(err, errTooLarge) {
+		t.Fatalf("replaceCopy: %v, want %v", err, errTooLarge)
+	}
+
+	if listings := lookup(t, b, listedAddr); len(listings) != 1 {
+		t.Errorf("listings of %s %+v, want one", listedAddr, listings)
+	}
+
+	checkGoodCopyOnDisk(t, dir)
+}
+
+// TestRestartLoadsCopiesFromDisk downloads a list, then starts a new
+// IPLists on the same directory, as a restart does. The new one answers
+// from the copy on disk at once, and does not download the list again while
+// the copy is younger than the list's interval.
+func TestRestartLoadsCopiesFromDisk(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	p, url := newPublisher(t, goodListBody)
+	list := testList(url, FormatLines)
+
+	first := newTestIPLists(dir, list)
+
+	err := first.download(t.Context(), list)
+	if err != nil {
+		t.Fatalf("download: %v", err)
+	}
+
+	downloaded := lookup(t, first, listedAddr)[0].Updated
+
+	second := newTestIPLists(dir, list)
+	second.Start(t.Context())
+	second.Stop(t.Context())
+
+	listings := lookup(t, second, listedAddr)
+	if len(listings) != 1 ||
+		listings[0].Updated.Sub(downloaded).Abs() > time.Second {
+		t.Errorf("after the restart, listings %+v, want one updated at %s",
+			listings, downloaded)
+	}
+
+	if got := len(p.requests()); got != 1 {
+		t.Errorf("the publisher had %d requests, want 1", got)
+	}
+}
+
+// TestStartRefreshesAnOldCopy starts on a copy older than its list's
+// interval. The list is downloaded at once, with routewatch's User-Agent and
+// the Last-Modified and ETag the publisher sent with the copy, kept on disk.
+// The publisher answers that the copy is still current, so the copy is kept
+// and marked as updated now.
+func TestStartRefreshesAnOldCopy(t *testing.T) {
+	t.Parallel()
+
+	dir := t.TempDir()
+	p, url := newPublisher(t, goodListBody)
+	list := testList(url, FormatLines)
+
+	lastModified := time.Now().Add(-3 * list.Interval).UTC().Truncate(time.Second)
+	p.publish(goodListBody, lastModified, testETag)
+
+	err := newTestIPLists(dir, list).download(t.Context(), list)
+	if err != nil {
+		t.Fatalf("download: %v", err)
+	}
+
+	old := time.Now().Add(-2 * list.Interval).UTC().Truncate(time.Second)
+
+	err = os.Chtimes(filepath.Join(dir, testListName), old, old)
+	if err != nil {
+		t.Fatalf("Chtimes: %v", err)
+	}
+
+	b := newTestIPLists(dir, list)
+	b.Start(t.Context())
+
+	deadline := time.Now().Add(maxWait)
+	for !lookup(t, b, listedAddr)[0].Updated.After(old) {
+		if time.Now().After(deadline) {
+			t.Fatalf("the copy was not marked updated within %s", maxWait)
+		}
+
+		time.Sleep(time.Millisecond)
+	}
+
+	b.Stop(t.Context())
+
+	requests := p.requests()
+	last := requests[len(requests)-1]
+
+	want := lastModified.Format(http.TimeFormat)
+	if got := last.Get(ifModifiedSinceHeader); got != want {
+		t.Errorf("If-Modified-Since %q, want %q", got, want)
+	}
+
+	if got := last.Get(ifNoneMatchHeader); got != testETag {
+		t.Errorf("If-None-Match %q, want %q", got, testETag)
+	}
+
+	if got := last.Get(userAgentHeader); got != userAgent {
+		t.Errorf("User-Agent %q, want %q", got, userAgent)
+	}
+}
+
+// TestNewVersionOlderThanTheLastDownloadReplacesTheCopy downloads a list.
+// The publisher then puts out a new version whose Last-Modified is earlier
+// than that download, as when a cache in front of the publisher passes the
+// version on late. The next download sends the Last-Modified of the copy,
+// not the time of the download, so it gets the new version.
+func TestNewVersionOlderThanTheLastDownloadReplacesTheCopy(t *testing.T) {
+	t.Parallel()
+
+	p, url := newPublisher(t, goodListBody)
+	list := testList(url, FormatLines)
+	b := newTestIPLists(t.TempDir(), list)
+
+	firstVersion := time.Now().Add(-2 * list.Interval).UTC().Truncate(time.Second)
+	p.publish(goodListBody, firstVersion, "")
+
+	err := b.download(t.Context(), list)
+	if err != nil {
+		t.Fatalf("first download: %v", err)
+	}
+
+	p.publish(newListBody, firstVersion.Add(list.Interval), "")
+
+	err = b.download(t.Context(), list)
+	if err != nil {
+		t.Fatalf("second download: %v", err)
+	}
+
+	if listings := lookup(t, b, newListedAddr); len(listings) != 1 {
+		t.Errorf("%s: listings %+v, want one from the new version",
+			newListedAddr, listings)
+	}
+
+	want := firstVersion.Format(http.TimeFormat)
+	if got := p.requests()[1].Get(ifModifiedSinceHeader); got != want {
+		t.Errorf("If-Modified-Since %q, want %q", got, want)
+	}
+}
+
+// TestDownloadWithoutLastModifiedOrETagSendsNoCondition downloads a list
+// twice from a publisher that sends neither Last-Modified nor ETag. Neither
+// request carries If-Modified-Since or If-None-Match, so the second download
+// gets the whole list again.
+func TestDownloadWithoutLastModifiedOrETagSendsNoCondition(t *testing.T) {
+	t.Parallel()
+
+	p, url := newPublisher(t, goodListBody)
+	list := testList(url, FormatLines)
+	b := newTestIPLists(t.TempDir(), list)
+
+	err := b.download(t.Context(), list)
+	if err != nil {
+		t.Fatalf("first download: %v", err)
+	}
+
+	err = b.download(t.Context(), list)
+	if err != nil {
+		t.Fatalf("second download: %v", err)
+	}
+
+	for i, request := range p.requests() {
+		if request.Get(ifModifiedSinceHeader) != "" ||
+			request.Get(ifNoneMatchHeader) != "" {
+			t.Errorf("request %d has a condition: %v", i, request)
+		}
+	}
+}
+
+// TestStopCancelsADownloadInProgress stops while the publisher has not
+// answered. Stop cancels the download and returns once the download
+// goroutine has stopped.
+func TestStopCancelsADownloadInProgress(t *testing.T) {
+	t.Parallel()
+
+	arrived := make(chan struct{})
+
+	var once sync.Once
+
+	srv := httptest.NewServer(http.HandlerFunc(
+		func(_ http.ResponseWriter, r *http.Request) {
+			once.Do(func() { close(arrived) })
+			<-r.Context().Done()
+		}))
+	t.Cleanup(srv.Close)
+
+	b := newTestIPLists(t.TempDir(), testList(srv.URL, FormatLines))
+	b.Start(t.Context())
+
+	select {
+	case <-arrived:
+	case <-time.After(maxWait):
+		t.Fatalf("no download began within %s", maxWait)
+	}
+
+	ctx, cancel := context.WithTimeout(t.Context(), maxWait)
+	defer cancel()
+
+	b.Stop(ctx)
+
+	select {
+	case <-b.done:
+	default:
+		t.Error("Stop returned before the download goroutine stopped")
+	}
+}
+
+// TestPublicListsAreComplete checks that every public list has what a
+// download and a lookup answer need, and a unique name.
+func TestPublicListsAreComplete(t *testing.T) {
+	t.Parallel()
+
+	names := make(map[string]bool)
+
+	for _, list := range Lists() {
+		if list.Name == "" || list.Description == "" || list.URL == "" ||
+			list.Interval <= 0 || names[list.Name] {
+			t.Errorf("list %+v is incomplete or has a name used before", list)
+		}
+
+		names[list.Name] = true
+	}
+}

internal/iplist/lists.go

@@ -0,0 +1,96 @@
+package iplist
+
+import "time"
+
+// TorExitsList is the name of the list of Tor exit nodes.
+const TorExitsList = "tor_exits"
+
+// How often each list is downloaded. No list is downloaded more often than
+// its publisher updates it or allows.
+const (
+	// DShield asks that its feeds be downloaded at most once an hour:
+	// https://isc.sans.edu/feeds_doc.html
+	dshieldInterval = time.Hour
+
+	// abuse.ch makes the list every 5 minutes and recommends a download at
+	// least every 15: https://feodotracker.abuse.ch/blocklist/
+	feodoTrackerInterval = 15 * time.Minute
+
+	// FireHOL's page for each list gives how often the list changes on
+	// average: about every 5 hours for abusers and every 3 hours for level
+	// 3. FireHOL sets no download limit.
+	// https://iplists.firehol.org/?ipset=firehol_abusers_30d
+	// https://iplists.firehol.org/?ipset=firehol_level3
+	fireHOLAbusersInterval = 5 * time.Hour
+	fireHOLLevel3Interval  = 3 * time.Hour
+
+	// Spamhaus asks that DROP be downloaded at most once a day:
+	// https://www.spamhaus.org/blocklists/do-not-route-or-peer/
+	spamhausDROPInterval = 24 * time.Hour
+
+	// Tor publishes a new exit list every hour and sets no download limit:
+	// https://collector.torproject.org/recent/exit-lists/
+	torExitsInterval = time.Hour
+)
+
+// Lists returns the public IP lists routewatch downloads. Each can be
+// downloaded whole without an account or a key. Spamhaus merged its EDROP
+// list into DROP, so DROP covers both.
+func Lists() []List {
+	return []List{
+		{
+			Name: "dshield",
+			Description: "In one of the 20 networks that DShield saw scan " +
+				"the most targets in the last three days",
+			URL:      "https://feeds.dshield.org/block.txt",
+			Interval: dshieldInterval,
+			Format:   FormatDShield,
+		},
+		{
+			Name:        "feodo_tracker",
+			Description: "A botnet command-and-control server",
+			URL:         "https://feodotracker.abuse.ch/downloads/ipblocklist.txt",
+			Interval:    feodoTrackerInterval,
+			Format:      FormatLines,
+		},
+		{
+			Name: "firehol_abusers_30d",
+			Description: "Reported for comment, form or forum spam in the " +
+				"last 30 days",
+			URL:      "https://iplists.firehol.org/files/firehol_abusers_30d.netset",
+			Interval: fireHOLAbusersInterval,
+			Format:   FormatLines,
+		},
+		{
+			Name: "firehol_level3",
+			Description: "Reported for attacks, spyware or viruses in the " +
+				"last 30 days",
+			URL:      "https://iplists.firehol.org/files/firehol_level3.netset",
+			Interval: fireHOLLevel3Interval,
+			Format:   FormatLines,
+		},
+		{
+			Name: "spamhaus_drop",
+			Description: "In an IPv4 network that Spamhaus says spammers " +
+				"or cyber criminals have hijacked or lease",
+			URL:      "https://www.spamhaus.org/drop/drop_v4.json",
+			Interval: spamhausDROPInterval,
+			Format:   FormatSpamhausJSON,
+		},
+		{
+			Name: "spamhaus_dropv6",
+			Description: "In an IPv6 network that Spamhaus says spammers " +
+				"or cyber criminals have hijacked or lease",
+			URL:      "https://www.spamhaus.org/drop/drop_v6.json",
+			Interval: spamhausDROPInterval,
+			Format:   FormatSpamhausJSON,
+		},
+		{
+			Name:        TorExitsList,
+			Description: "A Tor exit node, where traffic from the Tor network leaves",
+			URL:         "https://check.torproject.org/torbulkexitlist",
+			Interval:    torExitsInterval,
+			Format:      FormatLines,
+		},
+	}
+}

internal/iplist/parse.go

@@ -0,0 +1,158 @@
+package iplist
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/netip"
+	"strings"
+
+	"go4.org/netipx"
+)
+
+const (
+	// commentPrefix starts a comment line in a list file.
+	commentPrefix = "#"
+
+	// networkSeparator separates the address and the mask length of a
+	// network, as in 192.0.2.0/24.
+	networkSeparator = "/"
+
+	// dshieldFieldSeparator separates the fields of a DShield line.
+	dshieldFieldSeparator = "\t"
+
+	// spamhausMetadataType is the type of the line that ends a Spamhaus
+	// file. That line holds the file's metadata and no network.
+	spamhausMetadataType = "metadata"
+)
+
+var (
+	errUnknownFormat = errors.New("unknown list format")
+	errNoNetwork     = errors.New("no network on the line")
+	errBadRange      = errors.New("first and last address make no range")
+)
+
+// parse reads a list file written in format and returns the addresses it
+// holds. A line it cannot read fails the whole file, so a file that is not
+// the list, such as an error page, never replaces a good copy.
+func parse(r io.Reader, format Format) (*netipx.IPSet, error) {
+	var builder netipx.IPSetBuilder
+
+	scanner := bufio.NewScanner(r)
+
+	for lineNumber := 1; scanner.Scan(); lineNumber++ {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" || strings.HasPrefix(line, commentPrefix) {
+			continue
+		}
+
+		err := addLine(&builder, line, format)
+		if err != nil {
+			return nil, fmt.Errorf("line %d: %w", lineNumber, err)
+		}
+	}
+
+	err := scanner.Err()
+	if err != nil {
+		return nil, err
+	}
+
+	return builder.IPSet()
+}
+
+// addLine adds the address or network on one line of a list file to builder.
+func addLine(builder *netipx.IPSetBuilder, line string, format Format) error {
+	switch format {
+	case FormatLines:
+		return addAddressOrNetwork(builder, line)
+	case FormatSpamhausJSON:
+		return addSpamhausNetwork(builder, line)
+	case FormatDShield:
+		return addDShieldNetwork(builder, line)
+	default:
+		return errUnknownFormat
+	}
+}
+
+// addAddressOrNetwork adds an address, such as 192.0.2.1, or a network, such
+// as 192.0.2.0/24, to builder.
+func addAddressOrNetwork(builder *netipx.IPSetBuilder, text string) error {
+	if strings.Contains(text, networkSeparator) {
+		network, err := netip.ParsePrefix(text)
+		if err != nil {
+			return err
+		}
+
+		builder.AddPrefix(network)
+
+		return nil
+	}
+
+	addr, err := netip.ParseAddr(text)
+	if err != nil {
+		return err
+	}
+
+	builder.Add(addr)
+
+	return nil
+}
+
+// addSpamhausNetwork adds the network on a line of a Spamhaus file to
+// builder. The metadata line adds nothing.
+func addSpamhausNetwork(builder *netipx.IPSetBuilder, line string) error {
+	var entry struct {
+		CIDR string `json:"cidr"`
+		Type string `json:"type"`
+	}
+
+	err := json.Unmarshal([]byte(line), &entry)
+	if err != nil {
+		return err
+	}
+
+	if entry.Type == spamhausMetadataType {
+		return nil
+	}
+
+	network, err := netip.ParsePrefix(entry.CIDR)
+	if err != nil {
+		return err
+	}
+
+	builder.AddPrefix(network)
+
+	return nil
+}
+
+// addDShieldNetwork adds the network on a line of a DShield file to builder.
+// The line's first two fields are the network's first and last address.
+func addDShieldNetwork(builder *netipx.IPSetBuilder, line string) error {
+	firstField, rest, found := strings.Cut(line, dshieldFieldSeparator)
+	if !found {
+		return errNoNetwork
+	}
+
+	lastField, _, _ := strings.Cut(rest, dshieldFieldSeparator)
+
+	first, err := netip.ParseAddr(firstField)
+	if err != nil {
+		return err
+	}
+
+	last, err := netip.ParseAddr(lastField)
+	if err != nil {
+		return err
+	}
+
+	network := netipx.IPRangeFrom(first, last)
+	if !network.IsValid() {
+		return errBadRange
+	}
+
+	builder.AddRange(network)
+
+	return nil
+}