Report which public blocklists list an address in the IP lookup (closes #158)

A new package, internal/blocklist, downloads seven public lists on a
schedule: DShield, Feodo Tracker, FireHOL abusers and level 3, Spamhaus
DROP for IPv4 and IPv6, and the Tor exit list. It keeps the last good
copy of each in memory and on disk. A download sends back the
Last-Modified and ETag the publisher gave for the copy, also kept on
disk.

Every IP lookup result carries blocklists. An address with no live route
now gets a result too, without route fields, so the lookup no longer
answers 404. A successful answer carries Cache-Control, private for the
caller's own address. The address sets come from go4.org/netipx.

Model: opus-5-5

internal/routewatch/app.go

@@ -8,9 +8,11 @@ package routewatch
 import (
 	"context"
 	"errors"
+	"path/filepath"
 	"sync"
 	"time"
 
+	"git.eeqj.de/sneak/routewatch/internal/blocklist"
 	"git.eeqj.de/sneak/routewatch/internal/config"
 	"git.eeqj.de/sneak/routewatch/internal/database"
 	"git.eeqj.de/sneak/routewatch/internal/logger"
@@ -52,8 +54,13 @@ type RouteWatch struct {
 	peeringHandler *PeeringHandler
 	asnFetcher     *ASNFetcher
 	dbMaintainer   *DBMaintainer
+	blocklists     *blocklist.Blocklists
 }
 
+// blocklistsDirName is the directory in the state directory that holds the
+// blocklist copies.
+const blocklistsDirName = "blocklists"
+
 // New creates a new RouteWatch instance
 func New(deps Dependencies) *RouteWatch {
 	rw := &RouteWatch{
@@ -122,6 +129,18 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
 	rw.asnFetcher.Start(ctx)
 	rw.server.SetASNFetcher(rw.asnFetcher)
 
+	// Load the blocklist copies from disk and start downloading the lists.
+	// The HTTP handlers read the server's blocklists without a lock, so they
+	// are set before the handlers run.
+	blocklists := blocklist.New(blocklist.Params{
+		Dir:    filepath.Join(rw.config.StateDir, blocklistsDirName),
+		Lists:  blocklist.Lists(),
+		Logger: rw.logger.Logger,
+	})
+	blocklists.Start(ctx)
+	rw.blocklists = blocklists
+	rw.server.SetBlocklists(blocklists)
+
 	// Start HTTP server
 	err = rw.server.Start(ctx)
 	if err != nil {
@@ -139,11 +158,12 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
 }
 
 // Shutdown stops all services in order: the streamer, the handlers, which
-// write what they hold, the WHOIS fetcher, the database maintainer and the
-// HTTP server, and last the database, which it checkpoints and closes. Each
-// stage waits at most until ctx ends, so Shutdown returns soon after: a
-// handler logs what it could not write by then, and the HTTP server closes
-// the connections of requests still running.
+// write what they hold, the WHOIS fetcher, the database maintainer, the
+// blocklist downloads and the HTTP server, and last the database, which it
+// checkpoints and closes. Each stage waits at most until ctx ends, so
+// Shutdown returns soon after: a handler logs what it could not write by
+// then, and the HTTP server closes the connections of requests still
+// running.
 func (rw *RouteWatch) Shutdown(ctx context.Context) {
 	rw.mu.Lock()
 	if rw.shutdown {
@@ -190,6 +210,10 @@ func (rw *RouteWatch) Shutdown(ctx context.Context) {
 		rw.dbMaintainer.Stop(ctx)
 	}
 
+	if rw.blocklists != nil {
+		rw.blocklists.Stop(ctx)
+	}
+
 	err := rw.server.Stop(ctx)
 	if err != nil {
 		rw.logger.Error("Stopped the HTTP server before every request finished",

internal/routewatch/app_integration_internal_test.go

@@ -466,9 +466,9 @@ func TestRouteWatchLiveFeed(t *testing.T) {
 	// Create streamer
 	s := streamer.New(logger, metricsTracker)
 
-	// Create test config with empty state dir (no snapshot loading)
+	// The state directory holds the blocklist copies Run downloads.
 	cfg := &config.Config{
-		StateDir:                    "",
+		StateDir:                    t.TempDir(),
 		MaxRuntime:                  5 * time.Second,
 		EnableBatchedDatabaseWrites: true,
 	}