Report which public blocklists list an address in the IP lookup (closes #158)

A new package, internal/blocklist, downloads seven public lists on a
schedule: DShield, Feodo Tracker, FireHOL abusers and level 3, Spamhaus
DROP for IPv4 and IPv6, and the Tor exit list. It keeps the last good
copy of each in memory and on disk. A download sends back the
Last-Modified and ETag the publisher gave for the copy, also kept on
disk.

Every IP lookup result carries blocklists. An address with no live route
now gets a result too, without route fields, so the lookup no longer
answers 404. A successful answer carries Cache-Control, private for the
caller's own address. The address sets come from go4.org/netipx.

Model: opus-5-5

internal/server/handlers.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 	"time"
 
+	"git.eeqj.de/sneak/routewatch/internal/blocklist"
 	"git.eeqj.de/sneak/routewatch/internal/database"
 	"git.eeqj.de/sneak/routewatch/internal/metrics"
 	"git.eeqj.de/sneak/routewatch/internal/ristypes"
@@ -45,6 +46,15 @@ const (
 	ptrLookupTimeout = 2 * time.Second
 )
 
+// The Cache-Control header of an IP lookup answer, such as
+// "public, max-age=900".
+const (
+	cacheControlHeader  = "Cache-Control"
+	cacheControlPublic  = "public"
+	cacheControlPrivate = "private"
+	cacheControlMaxAge  = ", max-age="
+)
+
 // Keys of the JSON envelope the handlers and middleware write, and the status
 // value of an error answer.
 const (
@@ -693,9 +703,24 @@ func (s *Server) handleIPLookup() http.HandlerFunc {
 
 // IPLookupResponse is the standard response for IP/hostname lookups.
 type IPLookupResponse struct {
-	Query   string             `json:"query"`
-	Results []*database.IPInfo `json:"results"`
-	Errors  []string           `json:"errors,omitempty"`
+	Query   string           `json:"query"`
+	Results []IPLookupResult `json:"results"`
+	Errors  []string         `json:"errors,omitempty"`
+}
+
+// IPLookupResult is the answer for one address: its PTR names, its live route
+// and that route's AS, and the blocklists that list it. IPInfo is nil when
+// the address has no live route, and the answer then has no route or AS
+// fields. IP hides the IPInfo field of the same name, so that an address with
+// no route has it too.
+type IPLookupResult struct {
+	*database.IPInfo
+
+	IP  string   `json:"ip"`
+	PTR []string `json:"ptr,omitempty"`
+
+	// Blocklists has one entry for each blocklist the address is on.
+	Blocklists []blocklist.Listing `json:"blocklists"`
 }
 
 // handleIPInfo returns a handler that provides comprehensive IP information.
@@ -704,7 +729,7 @@ type IPLookupResponse struct {
 // Always returns the same response structure with PTR records for each IP.
 func (s *Server) handleIPInfo() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		target := lookupTarget(r)
+		target, ownAddress := lookupTarget(r)
 		if target == "" {
 			s.writeJSONError(w, http.StatusBadRequest, "Could not determine IP address")
 
@@ -712,7 +737,7 @@ func (s *Server) handleIPInfo() http.HandlerFunc {
 		}
 
 		ctx := r.Context()
-		response := IPLookupResponse{Query: target, Results: make([]*database.IPInfo, 0)}
+		response := IPLookupResponse{Query: target, Results: make([]IPLookupResult, 0)}
 
 		// Collect all IPs to look up
 		ipsToLookup, resolveErrors, err := s.resolveLookupTargets(ctx, target)
@@ -749,7 +774,10 @@ func (s *Server) handleIPInfo() http.HandlerFunc {
 			}
 
 			switch {
-			case errors.Is(err, database.ErrNoRoute), errors.Is(err, database.ErrInvalidIP):
+			case errors.Is(err, database.ErrNoRoute):
+				// The address still gets a result, without route fields.
+				ipInfo = nil
+			case errors.Is(err, database.ErrInvalidIP):
 				response.Errors = append(response.Errors, ip+": "+err.Error())
 
 				continue
@@ -768,13 +796,14 @@ func (s *Server) handleIPInfo() http.HandlerFunc {
 				return
 			}
 
-			if len(ptrs) > 0 {
-				ipInfo.PTR = ptrs
-			}
+			response.Results = append(response.Results, IPLookupResult{
+				IP:         ip,
+				PTR:        ptrs,
+				IPInfo:     ipInfo,
+				Blocklists: s.blocklistListings(ip),
+			})
 
-			response.Results = append(response.Results, ipInfo)
-
-			if ipInfo.NeedsWHOISRefresh {
+			if ipInfo != nil && ipInfo.NeedsWHOISRefresh {
 				refreshASNs[ipInfo.ASN] = true
 			}
 		}
@@ -782,10 +811,40 @@ func (s *Server) handleIPInfo() http.HandlerFunc {
 		// Queue WHOIS refresh for stale ASNs (non-blocking)
 		s.queueWHOISRefresh(refreshASNs)
 
-		s.writeIPLookupAnswer(w, response)
+		s.writeIPLookupAnswer(w, response, ownAddress)
 	}
 }
 
+// blocklistListings returns the blocklists that list ip. It returns an empty
+// list, never nil, so that an answer's blocklists is [] rather than null.
+func (s *Server) blocklistListings(ip string) []blocklist.Listing {
+	addr, err := netip.ParseAddr(ip)
+	if s.blocklists == nil || err != nil {
+		return []blocklist.Listing{}
+	}
+
+	return s.blocklists.Lookup(addr)
+}
+
+// setIPLookupCacheControl lets caches keep an IP lookup answer for the
+// shortest blocklist interval, the soonest a blocklist copy can change. The
+// answer about the caller's own address is private, so that a shared cache
+// does not give it to another caller. Without blocklists it sets nothing.
+func (s *Server) setIPLookupCacheControl(w http.ResponseWriter, ownAddress bool) {
+	if s.blocklists == nil {
+		return
+	}
+
+	visibility := cacheControlPublic
+	if ownAddress {
+		visibility = cacheControlPrivate
+	}
+
+	maxAgeSeconds := int(s.blocklists.ShortestInterval().Seconds())
+	w.Header().Set(cacheControlHeader,
+		visibility+cacheControlMaxAge+strconv.Itoa(maxAgeSeconds))
+}
+
 // noTargetsMessage returns the message of the 400 an IP lookup answers when
 // no address is left to look up, followed by the first of resolveErrors, which
 // says why, when there is one.
@@ -799,16 +858,12 @@ func noTargetsMessage(resolveErrors []string) string {
 	return message + ": " + resolveErrors[0]
 }
 
-// writeIPLookupAnswer sends response, or 404 with its first error when no
-// address had a route.
+// writeIPLookupAnswer sends response with its Cache-Control header.
+// ownAddress says the request asked about the caller's own address.
 func (s *Server) writeIPLookupAnswer(
-	w http.ResponseWriter, response IPLookupResponse,
+	w http.ResponseWriter, response IPLookupResponse, ownAddress bool,
 ) {
-	if len(response.Results) == 0 && len(response.Errors) > 0 {
-		s.writeJSONError(w, http.StatusNotFound, "No routes found: "+response.Errors[0])
-
-		return
-	}
+	s.setIPLookupCacheControl(w, ownAddress)
 
 	err := writeJSONSuccess(w, response)
 	if err != nil {
@@ -817,8 +872,9 @@ func (s *Server) writeIPLookupAnswer(
 }
 
 // lookupTarget returns what an IP lookup request asks for: the ip or addr URL
-// parameter, or else the client's own address.
-func lookupTarget(r *http.Request) string {
+// parameter, or else the client's own address. The bool is true for the
+// client's own address.
+func lookupTarget(r *http.Request) (string, bool) {
 	target := chi.URLParam(r, "ip")
 	if target == "" {
 		target = chi.URLParam(r, "addr")
@@ -826,10 +882,10 @@ func lookupTarget(r *http.Request) string {
 
 	if target == "" {
 		// Use client IP (RealIP middleware has already processed this)
-		target = extractClientIP(r)
+		return extractClientIP(r), true
 	}
 
-	return target
+	return target, false
 }
 
 // queueWHOISRefresh asks the WHOIS fetcher, when there is one, to refresh