Resolve PTR names over DNS-over-HTTPS and cache them for the record TTL (closes #238)

`internal/ptr` asks Google, Cloudflare, AdGuard and Quad9 in turn, 2 seconds in all per address. It keeps answers in memory and writes them through to `cache.db`, opened by the new `internal/cachedb`. A reserved address is not sent to the resolvers.

Judgement calls:
- Each resolver gets an equal share of the time left, so a hung resolver does not use up the 2 seconds.
- The negative TTL is the smaller of the SOA minimum and the SOA's own TTL (RFC 2308).
- Names that are not host names are dropped, as `net.Resolver.LookupAddr` does.

Model: opus-5-5

internal/server/handlers.go

@@ -39,12 +39,9 @@ const (
 const (
 	// maxLookupTargets is the most addresses and host names an IP lookup may
 	// list, and the most addresses it may have once the names are resolved:
-	// each address costs two database reads and a PTR lookup.
+	// each address costs two database reads and a PTR lookup. Ten PTR lookups
+	// that run to ptr.LookupTimeout fit in the request's 30-second limit.
 	maxLookupTargets = 10
-
-	// ptrLookupTimeout is each PTR lookup's own time limit, so that ten
-	// addresses whose PTR lookups never answer fit in the 30-second limit.
-	ptrLookupTimeout = 2 * time.Second
 )
 
 // The Cache-Control header of an IP lookup answer, such as
@@ -1194,15 +1191,17 @@ func publicAddresses(addrs []string) []string {
 }
 
 // lookupPTRNames returns the PTR records of ip without their trailing dots. It
-// returns an empty list, never nil, when there are none, when the lookup fails
-// and when it runs past ptrLookupTimeout, so that an answer's ptr is [] rather
-// than null.
+// returns an empty list, never nil, when there are none and when the lookup
+// fails, so that an answer's ptr is [] rather than null.
 func (s *Server) lookupPTRNames(ctx context.Context, ip string) []string {
-	ctx, cancel := context.WithTimeout(ctx, ptrLookupTimeout)
-	defer cancel()
+	ptrs, err := s.ptrResolver.LookupAddr(ctx, ip)
+	if err != nil {
+		s.logger.Warn("PTR lookup failed", "ip", ip, "error", err)
 
-	ptrs, err := s.resolver.LookupAddr(ctx, ip)
-	if err != nil || ptrs == nil {
+		return []string{}
+	}
+
+	if ptrs == nil {
 		return []string{}
 	}
 

internal/server/handlers_internal_test.go

@@ -24,6 +24,7 @@ import (
 	"git.eeqj.de/sneak/routewatch/internal/iplist"
 	"git.eeqj.de/sneak/routewatch/internal/logger"
 	"git.eeqj.de/sneak/routewatch/internal/metrics"
+	"git.eeqj.de/sneak/routewatch/internal/ptr"
 	"git.eeqj.de/sneak/routewatch/internal/ristypes"
 	"git.eeqj.de/sneak/routewatch/internal/routetable"
 	"git.eeqj.de/sneak/routewatch/internal/streamer"
@@ -523,8 +524,11 @@ func newServerWithTable(t *testing.T, table *routetable.Table) *Server {
 
 	t.Cleanup(func() { _ = db.Close() })
 
-	return New(db, table, streamer.New(logger.New(), metrics.New()), logger.New(),
+	s := New(db, table, streamer.New(logger.New(), metrics.New()), logger.New(),
 		&config.Config{})
+	useFakeResolver(s, &fakeResolver{})
+
+	return s
 }
 
 // TestPrefixDetailHandlersParsePrefix checks that both prefix pages find a
@@ -1213,7 +1217,7 @@ func TestIPLookupAnswers500WhenDatabaseReadFails(t *testing.T) {
 
 	s := New(failingIPLookupDB{}, newTestTable(),
 		streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
-	s.resolver = &fakeResolver{}
+	useFakeResolver(s, &fakeResolver{})
 
 	tests := []struct {
 		path     string
@@ -1255,8 +1259,8 @@ var errNoSuchHost = errors.New("no such host")
 
 // fakeResolver answers a host name lookup with the addresses hosts lists for
 // the name, and a PTR lookup with no names or, when ptrNeverAnswers is set,
-// not at all: it waits until the lookup's context ends. hostLookups counts the
-// host name lookups.
+// with a failure after ptr.LookupTimeout, as the real one does when no
+// resolver answers. hostLookups counts the host name lookups.
 type fakeResolver struct {
 	hosts           map[string][]string
 	ptrNeverAnswers bool
@@ -1277,6 +1281,9 @@ func (r *fakeResolver) LookupHost(_ context.Context, host string) ([]string, err
 
 func (r *fakeResolver) LookupAddr(ctx context.Context, _ string) ([]string, error) {
 	if r.ptrNeverAnswers {
+		ctx, cancel := context.WithTimeout(ctx, ptr.LookupTimeout)
+		defer cancel()
+
 		<-ctx.Done()
 
 		return nil, ctx.Err()
@@ -1285,6 +1292,12 @@ func (r *fakeResolver) LookupAddr(ctx context.Context, _ string) ([]string, erro
 	return nil, errNoSuchHost
 }
 
+// useFakeResolver makes r answer the server's host name and PTR lookups.
+func useFakeResolver(s *Server, r *fakeResolver) {
+	s.resolver = r
+	s.ptrResolver = r
+}
+
 // countingIPLookupDB embeds database.Store (left nil) and answers the IP
 // lookup's reads with nothing stored. It counts the lookups, as the reads of a
 // prefix's first-seen time. Every other Store method is unused and would panic
@@ -1353,7 +1366,7 @@ func TestIPLookupRefusesTooManyAddresses(t *testing.T) {
 		s := New(db, newTestTable(), streamer.New(logger.New(), metrics.New()),
 			logger.New(), &config.Config{})
 		resolver := &fakeResolver{hosts: hosts}
-		s.resolver = resolver
+		useFakeResolver(s, resolver)
 
 		rec := httptest.NewRecorder()
 		s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
@@ -1395,10 +1408,10 @@ func TestIPLookupKeepsRouteWhenPTRLookupDoesNotAnswer(t *testing.T) {
 	t.Parallel()
 
 	s := newServerWithTestRoutes(t)
-	s.resolver = &fakeResolver{ptrNeverAnswers: true}
+	useFakeResolver(s, &fakeResolver{ptrNeverAnswers: true})
 
 	// The request's time limit passes after the PTR lookup's.
-	ctx, cancel := context.WithTimeout(t.Context(), 2*ptrLookupTimeout)
+	ctx, cancel := context.WithTimeout(t.Context(), 2*ptr.LookupTimeout)
 	defer cancel()
 
 	rec := httptest.NewRecorder()
@@ -1476,7 +1489,7 @@ func TestIPLookupFlags(t *testing.T) {
 	t.Parallel()
 
 	s := newServerWithTestRoutes(t)
-	s.resolver = &fakeResolver{}
+	useFakeResolver(s, &fakeResolver{})
 	s.SetIPLists(fakeIPLists{})
 
 	tests := []struct {
@@ -1532,7 +1545,7 @@ func TestIPLookupReportsIPLists(t *testing.T) {
 	t.Parallel()
 
 	s := newServerWithTestRoutes(t)
-	s.resolver = &fakeResolver{}
+	useFakeResolver(s, &fakeResolver{})
 	s.SetIPLists(fakeIPLists{})
 
 	rec := httptest.NewRecorder()
@@ -1609,7 +1622,7 @@ func TestIPLookupReportsIPListsOfAnAddressWithNoRoute(t *testing.T) {
 	t.Parallel()
 
 	s := newServerWithTestRoutes(t)
-	s.resolver = &fakeResolver{}
+	useFakeResolver(s, &fakeResolver{})
 	s.SetIPLists(fakeIPLists{})
 
 	// routeFields are the fields of a result that come from the address's
@@ -1687,9 +1700,9 @@ func TestIPLookupResultsHaveTheSameFields(t *testing.T) {
 	t.Parallel()
 
 	s := newServerWithTestRoutes(t)
-	s.resolver = &fakeResolver{hosts: map[string][]string{
+	useFakeResolver(s, &fakeResolver{hosts: map[string][]string{
 		twoAddressHostName: {addrInTestPrefixV4, listedUnroutedAddr},
-	}}
+	}})
 	s.SetIPLists(fakeIPLists{})
 
 	// want is the fields of the first result.
@@ -1722,7 +1735,7 @@ func TestIPLookupOfAnAddressWithoutPTRNamesOrStoredNetblock(t *testing.T) {
 	t.Parallel()
 
 	s := newServerWithTestRoutes(t)
-	s.resolver = &fakeResolver{}
+	useFakeResolver(s, &fakeResolver{})
 
 	path := "/api/v1/ip/" + addrInTestPrefixV4
 
@@ -1752,7 +1765,7 @@ func TestIPLookupAnswersMayBeCached(t *testing.T) {
 	t.Parallel()
 
 	s := newServerWithTestRoutes(t)
-	s.resolver = &fakeResolver{}
+	useFakeResolver(s, &fakeResolver{})
 	s.SetIPLists(fakeIPLists{})
 
 	tests := []struct {
@@ -1803,10 +1816,10 @@ func TestIPLookupHidesPrivateAddresses(t *testing.T) {
 	t.Parallel()
 
 	s := newServerWithTestRoutes(t)
-	s.resolver = &fakeResolver{hosts: map[string][]string{
+	useFakeResolver(s, &fakeResolver{hosts: map[string][]string{
 		internalHostName: {internalHostAddr},
 		"mixed.test":     {internalHostAddr, addrInTestPrefixV4},
-	}}
+	}})
 
 	notPublic := internalHostName + ": " + hostNotPublicMessage
 

internal/server/handlers_table_internal_test.go

@@ -85,7 +85,7 @@ func TestIPLookupGivesTheOriginMostPeersCarry(t *testing.T) {
 	t.Parallel()
 
 	s := newServerWithTable(t, newTwoOriginTable())
-	s.resolver = &fakeResolver{}
+	useFakeResolver(s, &fakeResolver{})
 
 	firstSeen := testRouteTime(-1)
 

internal/server/middleware_internal_test.go

@@ -114,7 +114,7 @@ func TestJSONAnswersPutNoUnitInAString(t *testing.T) {
 		`^([0-9]+(\.[0-9]+)? ?(ns|µs|ms|s|m|h|B|kB|MB|GB|TB))+$`)
 
 	s := newServerWithTestRoutes(t)
-	s.resolver = &fakeResolver{}
+	useFakeResolver(s, &fakeResolver{})
 	s.SetASNFetcher(fixedIntervalASNFetcher{interval: time.Second})
 
 	paths := []string{
@@ -174,7 +174,7 @@ func TestJSONAnswersCarryMeta(t *testing.T) {
 	)
 
 	s := newServerWithTestRoutes(t)
-	s.resolver = &fakeResolver{}
+	useFakeResolver(s, &fakeResolver{})
 	s.streamer.GetMetricsTracker().SetConnected(true)
 	s.router.Get(writesNothingPath, JSONValidationMiddleware(http.HandlerFunc(
 		func(http.ResponseWriter, *http.Request) {})).ServeHTTP)
@@ -239,7 +239,7 @@ func TestJSONAnswersCarryMeta(t *testing.T) {
 	// handlers of the other routes 500.
 	failing := New(failingDB{}, newTestTable(),
 		streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
-	failing.resolver = &fakeResolver{}
+	useFakeResolver(failing, &fakeResolver{})
 
 	checkAnswerCarriesMeta(t, failing, httptest.NewRequestWithContext(
 		t.Context(), http.MethodGet, healthCheckPath, nil),
@@ -514,7 +514,7 @@ func newServerWithConfig(
 
 	s := New(db, newTestTable(), streamer.New(logger.New(), metrics.New()),
 		&logger.Logger{Logger: slog.New(slog.NewJSONHandler(requestLog, nil))}, cfg)
-	s.resolver = &fakeResolver{}
+	useFakeResolver(s, &fakeResolver{})
 
 	return s
 }

internal/server/server.go

@@ -52,10 +52,14 @@ type IPLists interface {
 	ShortestInterval() time.Duration
 }
 
-// dnsResolver looks up the addresses of host names and the names of addresses
-// in DNS, as *net.Resolver does.
+// dnsResolver looks up the addresses of host names in DNS, as *net.Resolver
+// does.
 type dnsResolver interface {
 	LookupHost(ctx context.Context, host string) ([]string, error)
+}
+
+// ptrResolver looks up the PTR names of addresses, as *ptr.Cache does.
+type ptrResolver interface {
 	LookupAddr(ctx context.Context, addr string) ([]string, error)
 }
 
@@ -75,10 +79,13 @@ type Server struct {
 	// RealIPMiddleware believes
 	trustedProxies []netip.Prefix
 
-	// resolver makes the IP lookup's DNS lookups: net.DefaultResolver, or a
+	// resolver looks up the IP lookup's host names: net.DefaultResolver, or a
 	// fake in tests
 	resolver dnsResolver
 
+	// ptrResolver looks up the IP lookup's PTR names. SetPTRResolver sets it.
+	ptrResolver ptrResolver
+
 	// cards is the copy of the home page's cards that the pages serve, nil
 	// until the server first works them out
 	cards atomic.Pointer[homeCards]
@@ -196,6 +203,12 @@ func (s *Server) Stop(ctx context.Context) error {
 	return nil
 }
 
+// SetPTRResolver sets what looks up the IP lookup's PTR names. Call it before
+// Start: the handlers read it without a lock.
+func (s *Server) SetPTRResolver(resolver ptrResolver) {
+	s.ptrResolver = resolver
+}
+
 // SetASNFetcher sets the ASN WHOIS fetcher for on-demand lookups. Call it
 // before Start: the handlers read the fetcher without a lock.
 func (s *Server) SetASNFetcher(fetcher ASNFetcher) {