Resolve PTR names over DNS-over-HTTPS and cache them for the record TTL (closes #238)
check / check (push) Successful in 1m58s

`internal/ptr` asks Google, Cloudflare, AdGuard and Quad9 in turn, 2 seconds in all per address. It keeps answers in memory and writes them through to `cache.db`, opened by the new `internal/cachedb`. A reserved address is not sent to the resolvers. The cache holds at most 250,000 addresses.

Judgement calls:
- Each resolver gets an equal share of the time left, so a hung resolver does not use up the 2 seconds.
- The negative TTL is the smaller of the SOA minimum and the SOA's own TTL (RFC 2308).
- Names that are not host names are dropped, as `net.Resolver.LookupAddr` does.

Model: opus-5-5
This commit is contained in:
sneak committed 2026-11-10 12:13:46 +00:00
1 parent 641798bb22
commit 848271b5c8
21 files changed
+1973 -69

No files matched your search

+11 -2
View File
@@ -1,5 +1,5 @@
// Package bogon tells whether a prefix or an AS number is reserved: set aside
// for a special purpose, so that no network should announce it on the
// Package bogon tells whether a prefix, an address or an AS number is reserved:
// set aside for a special purpose, so that no network should announce it on the
// internet. A route is a bogon announcement when its prefix is reserved or its
// origin AS is.
//
@@ -42,6 +42,15 @@ func Prefix(prefix netip.Prefix) (string, bool) {
return holder.name + " (" + holder.prefix.String() + ")", true
}
// Addr returns why addr is reserved, and true, when it lies inside a reserved
// block, as Prefix decides. An IPv4 address written in IPv6 form counts as the
// IPv4 address it holds. It returns "" and false when addr is not reserved.
func Addr(addr netip.Addr) (string, bool) {
addr = addr.Unmap()
return Prefix(netip.PrefixFrom(addr, addr.BitLen()))
}
// ASN returns why asn is reserved, and true, when it is. It returns "" and
// false when asn is not reserved.
func ASN(asn int) (string, bool) {
+38 -2
View File
@@ -8,6 +8,9 @@ import (
"git.eeqj.de/sneak/routewatch/internal/bogon"
)
// privateUseTen is why a prefix or an address inside 10.0.0.0/8 is reserved.
const privateUseTen = "Private-Use (10.0.0.0/8)"
func TestPrefix(t *testing.T) {
t.Parallel()
@@ -16,8 +19,8 @@ func TestPrefix(t *testing.T) {
// want is why the prefix is reserved, or "" when it is not.
want string
}{
{"10.1.2.0/24", "Private-Use (10.0.0.0/8)"},
{"10.0.0.0/8", "Private-Use (10.0.0.0/8)"},
{"10.1.2.0/24", privateUseTen},
{"10.0.0.0/8", privateUseTen},
{"192.0.2.0/24", "Documentation (TEST-NET-1) (192.0.2.0/24)"},
{"2001:db8:1::/48", "Documentation (2001:db8::/32)"},
{"fd00::/8", "Unique-Local (fc00::/7)"},
@@ -56,6 +59,39 @@ func TestPrefix(t *testing.T) {
}
}
func TestAddr(t *testing.T) {
t.Parallel()
tests := []struct {
addr string
// want is why the address is reserved, or "" when it is not.
want string
}{
{"10.0.0.1", privateUseTen},
{"fd00::1", "Unique-Local (fc00::/7)"},
// An IPv4 address written in IPv6 form counts as the IPv4 address.
{"::ffff:10.0.0.1", privateUseTen},
{"::ffff:8.8.8.8", ""},
// An address that may be announced, inside a reserved block.
{"192.0.0.9", ""},
// Addresses anyone may announce.
{"8.8.8.8", ""},
{"2001:4860:4860::8888", ""},
}
for _, tc := range tests {
t.Run(tc.addr, func(t *testing.T) {
t.Parallel()
reason, reserved := bogon.Addr(netip.MustParseAddr(tc.addr))
if reason != tc.want || reserved != (tc.want != "") {
t.Errorf("Addr(%s) = %q, %v; want %q", tc.addr, reason,
reserved, tc.want)
}
})
}
}
func TestASN(t *testing.T) {
t.Parallel()
+51
View File
@@ -0,0 +1,51 @@
// Package cachedb opens cache.db, the SQLite file in the state directory that
// holds RouteWatch's caches of answers from outside services. It is separate
// from the routes database so that a cache write never queues behind a write
// to that database.
package cachedb
import (
"context"
"database/sql"
_ "embed" // for schema.sql
"fmt"
"path/filepath"
_ "github.com/mattn/go-sqlite3" // CGO SQLite driver
)
const (
// fileName is the database's file in the state directory.
fileName = "cache.db"
// busyTimeoutMs is how long a write waits for another write to finish.
busyTimeoutMs = 5000
)
//go:embed schema.sql
var schema string
// Open opens cache.db in stateDir, creating the file and its tables when they
// do not exist. The caller closes the database.
func Open(ctx context.Context, stateDir string) (*sql.DB, error) {
dsn := fmt.Sprintf(
"file:%s?_journal_mode=WAL&_synchronous=NORMAL&_busy_timeout=%d"+
"&_txlock=immediate",
filepath.Join(stateDir, fileName),
busyTimeoutMs,
)
db, err := sql.Open("sqlite3", dsn)
if err != nil {
return nil, fmt.Errorf("opening the cache database: %w", err)
}
_, err = db.ExecContext(ctx, schema)
if err != nil {
_ = db.Close()
return nil, fmt.Errorf("creating the cache database's tables: %w", err)
}
return db, nil
}
+64
View File
@@ -0,0 +1,64 @@
package cachedb
import (
"os"
"path/filepath"
"testing"
)
// TestOpenCreatesTheFileInWALMode opens a new database, and then opens it
// again, as a restart does.
func TestOpenCreatesTheFileInWALMode(t *testing.T) {
t.Parallel()
dir := t.TempDir()
for range 2 {
db, err := Open(t.Context(), dir)
if err != nil {
t.Fatalf("Open: %v", err)
}
var mode string
err = db.QueryRowContext(t.Context(), "PRAGMA journal_mode").Scan(&mode)
if err != nil {
t.Fatalf("reading the journal mode: %v", err)
}
if mode != "wal" {
t.Errorf("journal mode %q, want wal", mode)
}
var rows int
err = db.QueryRowContext(t.Context(),
"SELECT COUNT(*) FROM hostnames").Scan(&rows)
if err != nil {
t.Fatalf("reading the hostnames table: %v", err)
}
err = db.Close()
if err != nil {
t.Fatalf("Close: %v", err)
}
}
_, err := os.Stat(filepath.Join(dir, fileName))
if err != nil {
t.Errorf("the file is not in the state directory: %v", err)
}
}
// TestOpenFailsWhenTheDirectoryDoesNotExist checks that a state directory that
// is not there is reported, not ignored.
func TestOpenFailsWhenTheDirectoryDoesNotExist(t *testing.T) {
t.Parallel()
db, err := Open(t.Context(), filepath.Join(t.TempDir(), "missing"))
if err == nil {
_ = db.Close()
t.Fatal("Open succeeded in a directory that does not exist")
}
}
+8
View File
@@ -0,0 +1,8 @@
-- hostnames holds the PTR names of addresses. names has the names separated by
-- spaces, and is empty for an address with no PTR names. expires_at is in
-- seconds since 1970.
CREATE TABLE IF NOT EXISTS hostnames (
address TEXT PRIMARY KEY,
names TEXT NOT NULL,
expires_at INTEGER NOT NULL
);
+324
View File
@@ -0,0 +1,324 @@
// Package ptr looks up the PTR names of addresses over DNS-over-HTTPS. It
// keeps answers in memory for as long as DNS allows, and in cache.db so that
// the answers survive a restart.
package ptr
import (
"context"
"database/sql"
"fmt"
"log/slog"
"maps"
"net/http"
"net/netip"
"slices"
"strings"
"sync"
"time"
"git.eeqj.de/sneak/routewatch/internal/bogon"
"github.com/miekg/dns"
)
const (
// flushInterval is how often new answers are written to cache.db.
flushInterval = 5 * time.Second
// sweepInterval is how often expired answers are deleted.
sweepInterval = 10 * time.Minute
// nameSeparator separates the names in a row of the hostnames table.
nameSeparator = " "
// maxAddresses limits how many addresses the cache holds, so that a public
// lookup cannot grow memory and cache.db without bound.
maxAddresses = 250_000
)
// Params is what New needs.
type Params struct {
// DB is the open cache.db.
DB *sql.DB
// Resolvers are the URLs of the DNS-over-HTTPS resolvers to ask, in order.
Resolvers []string
Logger *slog.Logger
}
// entry is one address's cached answer.
type entry struct {
// names are the address's PTR names, nil when it has none.
names []string
expires time.Time
}
// Cache answers PTR lookups from memory, and asks the resolvers for an address
// it holds no current answer for. Start loads it from cache.db.
type Cache struct {
db *sql.DB
resolvers []string
timeLimit time.Duration
maxEntries int
logger *slog.Logger
client *http.Client
now func() time.Time
// entries holds the kept answers by address, at most maxEntries of them.
// pending holds those that cache.db does not have yet.
mu sync.RWMutex
entries map[string]entry
pending map[string]entry
// stop ends the goroutine Start runs, and done is closed when it returns.
stop context.CancelFunc
done chan struct{}
}
// New returns an empty Cache. Start loads it.
func New(params Params) *Cache {
return &Cache{
db: params.DB,
resolvers: params.Resolvers,
timeLimit: LookupTimeout,
maxEntries: maxAddresses,
logger: params.Logger.With("component", "ptr"),
client: &http.Client{},
now: time.Now,
entries: make(map[string]entry),
pending: make(map[string]entry),
done: make(chan struct{}),
}
}
// LookupAddr returns the PTR names of addr, fully qualified, or none when it
// has none. A reserved address has none, and is not sent to the resolvers. A
// lookup takes at most LookupTimeout. A cached answer makes no outside request.
// It returns an error when no resolver answered, and nothing is cached then.
func (c *Cache) LookupAddr(ctx context.Context, addr string) ([]string, error) {
parsed, err := netip.ParseAddr(addr)
if err != nil {
return nil, fmt.Errorf("parsing the address: %w", err)
}
_, reserved := bogon.Addr(parsed)
if reserved {
return []string{}, nil
}
address := parsed.Unmap().String()
cached, ok := c.current(address)
if ok {
return slices.Clone(cached.names), nil
}
reverseName, err := dns.ReverseAddr(address)
if err != nil {
return nil, fmt.Errorf("building the reverse name: %w", err)
}
result, err := c.lookup(ctx, reverseName)
if err != nil {
return nil, err
}
if result.ttl > 0 {
c.store(address, entry{names: result.names, expires: c.now().Add(result.ttl)})
}
return slices.Clone(result.names), nil
}
// Start loads the answers that have not expired from cache.db. It then starts
// the goroutine that writes new answers there and deletes expired ones. The
// goroutine runs until Stop: the end of ctx does not stop it.
func (c *Cache) Start(ctx context.Context) error {
err := c.load(ctx)
if err != nil {
return err
}
c.logger.Info("PTR cache loaded", "entries", len(c.entries))
ctx, c.stop = context.WithCancel(context.WithoutCancel(ctx))
go c.run(ctx)
return nil
}
// Stop ends the goroutine, which first writes the answers cache.db does not
// have yet, and waits until it has returned or ctx ends.
func (c *Cache) Stop(ctx context.Context) {
c.stop()
select {
case <-c.done:
c.logger.Info("PTR cache stopped")
case <-ctx.Done():
c.logger.Warn("Stop deadline passed before the PTR cache was written")
}
}
// current returns the answer held for address, unless it has expired.
func (c *Cache) current(address string) (entry, bool) {
c.mu.RLock()
defer c.mu.RUnlock()
found, ok := c.entries[address]
if !ok || !c.now().Before(found.expires) {
return entry{}, false
}
return found, true
}
// store keeps an answer in memory and queues it for cache.db. Once entries
// holds maxEntries addresses it keeps no new address. It still replaces the
// answer of an address already held.
func (c *Cache) store(address string, answer entry) {
c.mu.Lock()
defer c.mu.Unlock()
_, held := c.entries[address]
if !held && len(c.entries) >= c.maxEntries {
return
}
c.entries[address] = answer
c.pending[address] = answer
}
// run writes new answers and deletes expired ones until ctx ends.
func (c *Cache) run(ctx context.Context) {
defer close(c.done)
flushes := time.NewTicker(flushInterval)
defer flushes.Stop()
sweeps := time.NewTicker(sweepInterval)
defer sweeps.Stop()
for {
select {
case <-ctx.Done():
c.flush(context.WithoutCancel(ctx))
return
case <-flushes.C:
c.flush(ctx)
case <-sweeps.C:
c.sweep(ctx)
}
}
}
// load reads the rows that have not expired into memory.
func (c *Cache) load(ctx context.Context) error {
rows, err := c.db.QueryContext(ctx,
"SELECT address, names, expires_at FROM hostnames WHERE expires_at > ?",
c.now().Unix())
if err != nil {
return fmt.Errorf("reading the PTR cache: %w", err)
}
defer func() { _ = rows.Close() }()
for rows.Next() {
var (
address, names string
expires int64
)
err = rows.Scan(&address, &names, &expires)
if err != nil {
return fmt.Errorf("reading a PTR cache row: %w", err)
}
c.entries[address] = entry{
names: strings.Fields(names),
expires: time.Unix(expires, 0).UTC(),
}
}
err = rows.Err()
if err != nil {
return fmt.Errorf("reading the PTR cache: %w", err)
}
return nil
}
// flush writes the pending answers to cache.db. Those it cannot write stay
// pending, unless a newer answer for the address has arrived meanwhile.
func (c *Cache) flush(ctx context.Context) {
c.mu.Lock()
batch := c.pending
c.pending = make(map[string]entry)
c.mu.Unlock()
if len(batch) == 0 {
return
}
err := c.writeRows(ctx, batch)
if err == nil {
return
}
c.logger.Error("Failed to write the PTR cache", "error", err)
c.mu.Lock()
defer c.mu.Unlock()
for address, answer := range batch {
_, newer := c.pending[address]
if !newer {
c.pending[address] = answer
}
}
}
// writeRows writes batch to cache.db in one transaction.
func (c *Cache) writeRows(ctx context.Context, batch map[string]entry) error {
tx, err := c.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() { _ = tx.Rollback() }()
for address, answer := range batch {
_, err = tx.ExecContext(ctx,
"INSERT OR REPLACE INTO hostnames (address, names, expires_at) "+
"VALUES (?, ?, ?)",
address,
strings.Join(answer.names, nameSeparator),
answer.expires.Unix())
if err != nil {
return err
}
}
return tx.Commit()
}
// sweep deletes the expired answers from memory and cache.db.
func (c *Cache) sweep(ctx context.Context) {
now := c.now()
expired := func(_ string, answer entry) bool {
return !now.Before(answer.expires)
}
c.mu.Lock()
maps.DeleteFunc(c.entries, expired)
maps.DeleteFunc(c.pending, expired)
c.mu.Unlock()
_, err := c.db.ExecContext(ctx,
"DELETE FROM hostnames WHERE expires_at <= ?", now.Unix())
if err != nil {
c.logger.Error("Failed to delete expired PTR cache rows", "error", err)
}
}
+951
View File
@@ -0,0 +1,951 @@
package ptr
import (
"context"
"database/sql"
"errors"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"slices"
"strconv"
"sync"
"testing"
"testing/synctest"
"time"
"git.eeqj.de/sneak/routewatch/internal/cachedb"
"github.com/miekg/dns"
)
// These addresses are public, because a reserved address is not sent to the
// resolvers. That rules out the documentation ranges.
const (
testAddress = "8.8.4.4"
testReverse = "4.4.8.8.in-addr.arpa."
mappedAddress = "::ffff:" + testAddress
otherAddress = "8.8.8.8"
testV6Address = "2606:4700:4700::1111"
testV6Reverse = "1.1.1.1.0.0.0.0.0.0.0.0.0.0.0.0." +
"0.0.0.0.0.0.7.4.0.0.7.4.6.0.6.2.ip6.arpa."
)
const (
testName = "host.example.com."
otherName = "other.example.com."
// testTimeLimit replaces LookupTimeout in the tests that use a real
// server. A stall on a busy host cannot use it up, and no test waits for
// it.
testTimeLimit = 2 * time.Minute
)
// classlessOwner is the name that holds the PTR records of a CNAME chain.
const classlessOwner = "4.0-25.4.8.8.in-addr.arpa."
// fixedTime returns the time a test's clock starts at.
func fixedTime() time.Time {
return time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC)
}
// fakeResolver is a DNS-over-HTTPS server. It answers each query with reply's
// result, and counts its requests.
type fakeResolver struct {
server *httptest.Server
mu sync.Mutex
requests int
reply func(http.ResponseWriter, *http.Request, *dns.Msg)
}
// newFakeResolver starts a fake resolver that answers with reply.
func newFakeResolver(
t *testing.T,
reply func(http.ResponseWriter, *http.Request, *dns.Msg),
) *fakeResolver {
t.Helper()
resolver := &fakeResolver{reply: reply}
resolver.server = httptest.NewServer(http.HandlerFunc(resolver.serve))
t.Cleanup(resolver.server.Close)
return resolver
}
// serve checks that the request is RFC 8484 wire format and passes the
// question on to the current reply.
func (f *fakeResolver) serve(w http.ResponseWriter, r *http.Request) {
f.mu.Lock()
f.requests++
reply := f.reply
f.mu.Unlock()
body, err := io.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
var query dns.Msg
err = query.Unpack(body)
if err != nil || r.Header.Get("Content-Type") != dnsMessageType ||
r.Header.Get("Accept") != dnsMessageType || r.Method != http.MethodPost {
http.Error(w, "not an RFC 8484 request", http.StatusBadRequest)
return
}
reply(w, r, &query)
}
func (f *fakeResolver) setReply(
reply func(http.ResponseWriter, *http.Request, *dns.Msg),
) {
f.mu.Lock()
defer f.mu.Unlock()
f.reply = reply
}
func (f *fakeResolver) requestCount() int {
f.mu.Lock()
defer f.mu.Unlock()
return f.requests
}
// writeReply packs reply and writes it as the answer to query.
func writeReply(w http.ResponseWriter, query, reply *dns.Msg) {
reply.SetReply(query)
wire, err := reply.Pack()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", dnsMessageType)
_, _ = w.Write(wire)
}
// ptrReply answers with one PTR record for each name, each with the TTL ttl.
func ptrReply(
ttl uint32, names ...string,
) func(http.ResponseWriter, *http.Request, *dns.Msg) {
return func(w http.ResponseWriter, _ *http.Request, query *dns.Msg) {
reply := new(dns.Msg)
for _, name := range names {
reply.Answer = append(reply.Answer, &dns.PTR{
Hdr: dns.RR_Header{
Name: query.Question[0].Name, Rrtype: dns.TypePTR,
Class: dns.ClassINET, Ttl: ttl,
},
Ptr: name,
})
}
writeReply(w, query, reply)
}
}
// rcodeReply answers with the response code and nothing else.
func rcodeReply(rcode int) func(http.ResponseWriter, *http.Request, *dns.Msg) {
return func(w http.ResponseWriter, _ *http.Request, query *dns.Msg) {
reply := new(dns.Msg)
reply.SetRcode(query, rcode)
wire, err := reply.Pack()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
_, _ = w.Write(wire)
}
}
// soaReply answers with the response code and a SOA record with these TTLs.
func soaReply(
rcode int, soaTTL, minimum uint32,
) func(http.ResponseWriter, *http.Request, *dns.Msg) {
return func(w http.ResponseWriter, _ *http.Request, query *dns.Msg) {
reply := new(dns.Msg)
reply.SetRcode(query, rcode)
reply.Ns = []dns.RR{&dns.SOA{
Hdr: dns.RR_Header{
Name: "arpa.", Rrtype: dns.TypeSOA, Class: dns.ClassINET, Ttl: soaTTL,
},
Ns: "ns.example.com.", Mbox: "hostmaster.example.com.", Minttl: minimum,
}}
wire, err := reply.Pack()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
_, _ = w.Write(wire)
}
}
// hangingReply never answers: it waits until the client gives up.
func hangingReply(_ http.ResponseWriter, r *http.Request, _ *dns.Msg) {
<-r.Context().Done()
}
// statusReply answers with an HTTP status.
func statusReply(status int) func(http.ResponseWriter, *http.Request, *dns.Msg) {
return func(w http.ResponseWriter, _ *http.Request, _ *dns.Msg) {
w.WriteHeader(status)
}
}
// openTestDB opens a cache.db in dir.
func openTestDB(t *testing.T, dir string) *sql.DB {
t.Helper()
db, err := cachedb.Open(t.Context(), dir)
if err != nil {
t.Fatalf("opening the cache database: %v", err)
}
t.Cleanup(func() { _ = db.Close() })
return db
}
// testClock is a clock a test moves by hand.
type testClock struct {
mu sync.Mutex
now time.Time
}
func (c *testClock) time() time.Time {
c.mu.Lock()
defer c.mu.Unlock()
return c.now
}
func (c *testClock) advance(d time.Duration) {
c.mu.Lock()
defer c.mu.Unlock()
c.now = c.now.Add(d)
}
// newTestCache returns a Cache that asks the resolvers in order, reads the
// time from a clock the test holds, has testTimeLimit, and is not started.
func newTestCache(
t *testing.T, db *sql.DB, resolvers ...*fakeResolver,
) (*Cache, *testClock) {
t.Helper()
urls := make([]string, 0, len(resolvers))
for _, resolver := range resolvers {
urls = append(urls, resolver.server.URL)
}
cache := New(Params{DB: db, Resolvers: urls, Logger: slog.Default()})
clock := &testClock{now: fixedTime()}
cache.now = clock.time
cache.timeLimit = testTimeLimit
return cache, clock
}
// handlerTransport hands each request to the fake resolver for its host, with
// no network between them. A test in a synctest bubble needs this: the bubble's
// clock stands still while one of its goroutines waits on a real connection. As
// over a network, a request whose time is up is not sent.
type handlerTransport map[string]*fakeResolver
func (h handlerTransport) RoundTrip(r *http.Request) (*http.Response, error) {
recorder := httptest.NewRecorder()
if r.Context().Err() == nil {
h[r.URL.Host].serve(recorder, r)
}
err := r.Context().Err()
if err != nil {
return nil, err
}
return recorder.Result(), nil
}
// newBubbleCache returns a Cache with the real time limit, for a test in a
// synctest bubble. It has one fake resolver for each reply, in order, reached
// through a handlerTransport.
func newBubbleCache(
replies ...func(http.ResponseWriter, *http.Request, *dns.Msg),
) (*Cache, []*fakeResolver) {
transport := make(handlerTransport, len(replies))
resolvers := make([]*fakeResolver, 0, len(replies))
urls := make([]string, 0, len(replies))
for i, reply := range replies {
host := "resolver" + strconv.Itoa(i) + ".example.com"
resolver := &fakeResolver{reply: reply}
transport[host] = resolver
resolvers = append(resolvers, resolver)
urls = append(urls, "https://"+host+"/dns-query")
}
cache := New(Params{Resolvers: urls, Logger: slog.Default()})
cache.client = &http.Client{Transport: transport}
return cache, resolvers
}
// lookupOK looks up address and fails the test on an error.
func lookupOK(t *testing.T, cache *Cache, address string) []string {
t.Helper()
names, err := cache.LookupAddr(t.Context(), address)
if err != nil {
t.Fatalf("LookupAddr(%s): %v", address, err)
}
return names
}
func countRows(t *testing.T, db *sql.DB) int {
t.Helper()
var rows int
err := db.QueryRowContext(t.Context(),
"SELECT COUNT(*) FROM hostnames").Scan(&rows)
if err != nil {
t.Fatalf("counting rows: %v", err)
}
return rows
}
func TestLookupAsksForTheReverseName(t *testing.T) {
t.Parallel()
tests := []struct {
address string
want string
}{
{testAddress, testReverse},
{mappedAddress, testReverse},
{testV6Address, testV6Reverse},
}
for _, tt := range tests {
var asked string
resolver := newFakeResolver(t,
func(w http.ResponseWriter, r *http.Request, query *dns.Msg) {
asked = query.Question[0].Name
if query.Question[0].Qtype != dns.TypePTR {
t.Errorf("query type %d, want PTR", query.Question[0].Qtype)
}
ptrReply(300, testName)(w, r, query)
})
cache, _ := newTestCache(t, openTestDB(t, t.TempDir()), resolver)
names := lookupOK(t, cache, tt.address)
if !slices.Equal(names, []string{testName}) {
t.Errorf("%s: names %v, want %s", tt.address, names, testName)
}
if asked != tt.want {
t.Errorf("%s: asked for %s, want %s", tt.address, asked, tt.want)
}
}
}
// TestOnlyPublicAddressesAreSentToTheResolvers gives every resolver an answer
// to send, which a reserved address must not get.
func TestOnlyPublicAddressesAreSentToTheResolvers(t *testing.T) {
t.Parallel()
tests := []struct {
name string
address string
want []string
requests int
}{
{"private IPv4", "10.0.0.1", []string{}, 0},
{"documentation IPv4", "192.0.2.1", []string{}, 0},
{"private IPv4 in IPv6 form", "::ffff:10.0.0.1", []string{}, 0},
{"unique-local IPv6", "fd00::1", []string{}, 0},
{"documentation IPv6", "2001:db8::1", []string{}, 0},
{"public IPv4", testAddress, []string{testName}, 1},
{"public IPv4 in IPv6 form", mappedAddress, []string{testName}, 1},
{"public IPv6", testV6Address, []string{testName}, 1},
}
for _, tt := range tests {
resolver := newFakeResolver(t, ptrReply(300, testName))
cache, _ := newTestCache(t, openTestDB(t, t.TempDir()), resolver)
names := lookupOK(t, cache, tt.address)
if names == nil || !slices.Equal(names, tt.want) {
t.Errorf("%s: names %#v, want %#v", tt.name, names, tt.want)
}
if got := resolver.requestCount(); got != tt.requests {
t.Errorf("%s: %d requests, want %d", tt.name, got, tt.requests)
}
}
}
// TestResolversAreTriedInOrderUntilOneAnswers has the first two resolvers fail,
// in different ways, and the third answer. The fourth is not asked.
func TestResolversAreTriedInOrderUntilOneAnswers(t *testing.T) {
t.Parallel()
var (
mu sync.Mutex
order []int
)
asked := func(
index int, reply func(http.ResponseWriter, *http.Request, *dns.Msg),
) func(http.ResponseWriter, *http.Request, *dns.Msg) {
return func(w http.ResponseWriter, r *http.Request, query *dns.Msg) {
mu.Lock()
order = append(order, index)
mu.Unlock()
reply(w, r, query)
}
}
resolvers := []*fakeResolver{
newFakeResolver(t, asked(0, statusReply(http.StatusServiceUnavailable))),
newFakeResolver(t, asked(1, rcodeReply(dns.RcodeServerFailure))),
newFakeResolver(t, asked(2, ptrReply(300, testName))),
newFakeResolver(t, asked(3, ptrReply(300, otherName))),
}
cache, _ := newTestCache(t, openTestDB(t, t.TempDir()), resolvers...)
names := lookupOK(t, cache, testAddress)
if !slices.Equal(names, []string{testName}) {
t.Errorf("names %v, want %s", names, testName)
}
if !slices.Equal(order, []int{0, 1, 2}) {
t.Errorf("resolvers asked %v, want [0 1 2]", order)
}
}
// TestResolverThatNeverAnswersLeavesTimeForTheNext would fail if the first
// resolver could take the whole time limit. The first gets half of it, the
// share of one resolver in two. The test runs in a synctest bubble, whose clock
// is fake, so a stall on the host uses none of the limit.
func TestResolverThatNeverAnswersLeavesTimeForTheNext(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
cache, _ := newBubbleCache(hangingReply, ptrReply(300, testName))
start := time.Now()
names := lookupOK(t, cache, testAddress)
elapsed := time.Since(start)
if !slices.Equal(names, []string{testName}) {
t.Errorf("names %v, want %s", names, testName)
}
if elapsed != time.Second {
t.Errorf("took %s, want 1s", elapsed)
}
})
}
// TestLookupStopsAtTheTimeLimit has no resolver answer. The lookup fails when
// 2 seconds have passed, having asked every resolver. The test runs in a
// synctest bubble, whose clock is fake, so a stall on the host uses none of the
// limit.
func TestLookupStopsAtTheTimeLimit(t *testing.T) {
t.Parallel()
synctest.Test(t, func(t *testing.T) {
cache, resolvers := newBubbleCache(
hangingReply, hangingReply, hangingReply, hangingReply)
start := time.Now()
_, err := cache.LookupAddr(t.Context(), testAddress)
elapsed := time.Since(start)
if !errors.Is(err, context.DeadlineExceeded) {
t.Errorf("error %v, want %v", err, context.DeadlineExceeded)
}
if elapsed != 2*time.Second {
t.Errorf("took %s, want 2s", elapsed)
}
for i, resolver := range resolvers {
if resolver.requestCount() != 1 {
t.Errorf("resolver %d asked %d times, want once",
i, resolver.requestCount())
}
}
})
}
// TestAnswerIsServedFromMemoryUntilItExpires makes a second lookup within the
// TTL, which asks nobody, and a third after it, which asks again.
func TestAnswerIsServedFromMemoryUntilItExpires(t *testing.T) {
t.Parallel()
resolver := newFakeResolver(t, ptrReply(300, testName))
cache, clock := newTestCache(t, openTestDB(t, t.TempDir()), resolver)
lookupOK(t, cache, testAddress)
names := lookupOK(t, cache, testAddress)
if !slices.Equal(names, []string{testName}) {
t.Errorf("cached names %v, want %s", names, testName)
}
if got := resolver.requestCount(); got != 1 {
t.Errorf("%d requests after the second lookup, want 1", got)
}
clock.advance(300 * time.Second)
lookupOK(t, cache, testAddress)
if got := resolver.requestCount(); got != 2 {
t.Errorf("%d requests after the TTL passed, want 2", got)
}
}
// TestAnAddressHasOneCacheEntryHoweverItIsWritten looks up an IPv4 address in
// its IPv4-mapped IPv6 form, then in its usual form.
func TestAnAddressHasOneCacheEntryHoweverItIsWritten(t *testing.T) {
t.Parallel()
resolver := newFakeResolver(t, ptrReply(300, testName))
cache, _ := newTestCache(t, openTestDB(t, t.TempDir()), resolver)
lookupOK(t, cache, mappedAddress)
lookupOK(t, cache, testAddress)
if got := resolver.requestCount(); got != 1 {
t.Errorf("%d requests, want 1", got)
}
}
// TestCallerCannotChangeACachedAnswer changes the list each lookup returned, as
// the server does when it removes the trailing dots. The second lookup is the
// first answered from the cache. The third shows that the cached list did not
// change with the list the second returned.
func TestCallerCannotChangeACachedAnswer(t *testing.T) {
t.Parallel()
resolver := newFakeResolver(t, ptrReply(300, testName))
cache, _ := newTestCache(t, openTestDB(t, t.TempDir()), resolver)
for range 3 {
names := lookupOK(t, cache, testAddress)
if !slices.Equal(names, []string{testName}) {
t.Errorf("names %v, want %s", names, testName)
}
names[0] = "changed"
}
}
func TestTTLIsClamped(t *testing.T) {
t.Parallel()
tests := []struct {
name string
ttl uint32
want time.Duration
}{
{"below the minimum", 1, time.Minute},
{"zero", 0, time.Minute},
{"at the minimum", 60, time.Minute},
{"in range", 300, 300 * time.Second},
{"at the maximum", 86400, 24 * time.Hour},
{"above the maximum", 604800, 24 * time.Hour},
}
for _, tt := range tests {
resolver := newFakeResolver(t, ptrReply(tt.ttl, testName))
cache, _ := newTestCache(t, openTestDB(t, t.TempDir()), resolver)
lookupOK(t, cache, testAddress)
got := cache.entries[testAddress].expires.Sub(fixedTime())
if got != tt.want {
t.Errorf("%s: kept for %s, want %s", tt.name, got, tt.want)
}
}
}
// TestSmallestTTLOfTheChainCounts answers with a CNAME record and two PTR
// records, whose TTLs all differ.
func TestSmallestTTLOfTheChainCounts(t *testing.T) {
t.Parallel()
reply := func(w http.ResponseWriter, _ *http.Request, query *dns.Msg) {
owner := query.Question[0].Name
answer := new(dns.Msg)
answer.Answer = []dns.RR{
&dns.CNAME{
Hdr: dns.RR_Header{
Name: owner, Rrtype: dns.TypeCNAME, Class: dns.ClassINET, Ttl: 200,
},
Target: classlessOwner,
},
&dns.PTR{
Hdr: dns.RR_Header{
Name: classlessOwner, Rrtype: dns.TypePTR,
Class: dns.ClassINET, Ttl: 900,
},
Ptr: testName,
},
&dns.PTR{
Hdr: dns.RR_Header{
Name: classlessOwner, Rrtype: dns.TypePTR,
Class: dns.ClassINET, Ttl: 600,
},
Ptr: otherName,
},
}
writeReply(w, query, answer)
}
cache, _ := newTestCache(t, openTestDB(t, t.TempDir()), newFakeResolver(t, reply))
names := lookupOK(t, cache, testAddress)
if !slices.Equal(names, []string{testName, otherName}) {
t.Errorf("names %v, want both PTR names", names)
}
got := cache.entries[testAddress].expires.Sub(fixedTime())
if got != 200*time.Second {
t.Errorf("kept for %s, want 200s", got)
}
}
func TestNegativeTTL(t *testing.T) {
t.Parallel()
tests := []struct {
name string
rcode int
soaTTL uint32
minimum uint32
want time.Duration
}{
{"name does not exist", dns.RcodeNameError, 3600, 300, 300 * time.Second},
{"no PTR record", dns.RcodeSuccess, 3600, 300, 300 * time.Second},
{"limited to an hour", dns.RcodeNameError, 86400, 7200, time.Hour},
{"SOA's own TTL is smaller", dns.RcodeNameError, 100, 300, 100 * time.Second},
}
for _, tt := range tests {
resolver := newFakeResolver(t, soaReply(tt.rcode, tt.soaTTL, tt.minimum))
cache, _ := newTestCache(t, openTestDB(t, t.TempDir()), resolver)
names := lookupOK(t, cache, testAddress)
if len(names) != 0 {
t.Errorf("%s: names %v, want none", tt.name, names)
}
got := cache.entries[testAddress].expires.Sub(fixedTime())
if got != tt.want {
t.Errorf("%s: kept for %s, want %s", tt.name, got, tt.want)
}
lookupOK(t, cache, testAddress)
if count := resolver.requestCount(); count != 1 {
t.Errorf("%s: %d requests, want 1", tt.name, count)
}
}
}
// TestAnswerWithoutNamesOrSOAIsNotCached has nothing to say how long the
// answer may be kept.
func TestAnswerWithoutNamesOrSOAIsNotCached(t *testing.T) {
t.Parallel()
resolver := newFakeResolver(t, rcodeReply(dns.RcodeNameError))
cache, _ := newTestCache(t, openTestDB(t, t.TempDir()), resolver)
lookupOK(t, cache, testAddress)
lookupOK(t, cache, testAddress)
if got := resolver.requestCount(); got != 2 {
t.Errorf("%d requests, want 2", got)
}
}
// TestFailedQueryIsNotCached has the only resolver fail, and then recover.
func TestFailedQueryIsNotCached(t *testing.T) {
t.Parallel()
resolver := newFakeResolver(t, rcodeReply(dns.RcodeServerFailure))
cache, _ := newTestCache(t, openTestDB(t, t.TempDir()), resolver)
_, err := cache.LookupAddr(t.Context(), testAddress)
if !errors.Is(err, errBadResponse) {
t.Fatalf("error %v, want %v", err, errBadResponse)
}
if len(cache.entries)+len(cache.pending) != 0 {
t.Error("a failed query left an entry in the cache")
}
resolver.setReply(ptrReply(300, testName))
names := lookupOK(t, cache, testAddress)
if !slices.Equal(names, []string{testName}) {
t.Errorf("names %v, want %s", names, testName)
}
}
// TestNamesThatAreNotHostNamesAreDropped answers with a name that holds markup
// beside a good one, as a PTR record can.
func TestNamesThatAreNotHostNamesAreDropped(t *testing.T) {
t.Parallel()
resolver := newFakeResolver(t,
ptrReply(300, "bad<script>.example.com.", testName, "."))
cache, _ := newTestCache(t, openTestDB(t, t.TempDir()), resolver)
names := lookupOK(t, cache, testAddress)
if !slices.Equal(names, []string{testName}) {
t.Errorf("names %v, want only %s", names, testName)
}
}
// TestCacheSurvivesARestart stops a cache that holds a positive and a negative
// answer, and starts a new one on the reopened file. It makes no request.
func TestCacheSurvivesARestart(t *testing.T) {
t.Parallel()
dir := t.TempDir()
db := openTestDB(t, dir)
resolver := newFakeResolver(t,
func(w http.ResponseWriter, r *http.Request, query *dns.Msg) {
if query.Question[0].Name == testReverse {
ptrReply(300, testName, otherName)(w, r, query)
return
}
soaReply(dns.RcodeNameError, 3600, 300)(w, r, query)
})
cache, _ := newTestCache(t, db, resolver)
err := cache.Start(t.Context())
if err != nil {
t.Fatalf("Start: %v", err)
}
lookupOK(t, cache, testAddress)
lookupOK(t, cache, otherAddress)
cache.Stop(t.Context())
err = db.Close()
if err != nil {
t.Fatalf("closing the database: %v", err)
}
restarted, _ := newTestCache(t, openTestDB(t, dir), resolver)
err = restarted.Start(t.Context())
if err != nil {
t.Fatalf("Start after the restart: %v", err)
}
defer restarted.Stop(t.Context())
requestsBefore := resolver.requestCount()
names := lookupOK(t, restarted, testAddress)
if !slices.Equal(names, []string{testName, otherName}) {
t.Errorf("names after restart %v", names)
}
if names := lookupOK(t, restarted, otherAddress); len(names) != 0 {
t.Errorf("names of the address with none after restart: %v", names)
}
if got := resolver.requestCount(); got != requestsBefore {
t.Errorf("%d requests after the restart, want none",
got-requestsBefore)
}
}
// TestSweepDeletesExpiredRows also checks that a restart does not load a row
// that has expired since it was written.
func TestSweepDeletesExpiredRows(t *testing.T) {
t.Parallel()
db := openTestDB(t, t.TempDir())
resolver := newFakeResolver(t,
func(w http.ResponseWriter, r *http.Request, query *dns.Msg) {
ttl := uint32(300)
if query.Question[0].Name == testReverse {
ttl = 60
}
ptrReply(ttl, testName)(w, r, query)
})
cache, clock := newTestCache(t, db, resolver)
lookupOK(t, cache, testAddress)
lookupOK(t, cache, otherAddress)
cache.flush(t.Context())
if got := countRows(t, db); got != 2 {
t.Fatalf("%d rows before the sweep, want 2", got)
}
clock.advance(90 * time.Second)
loaded, loadedClock := newTestCache(t, db, resolver)
loadedClock.advance(90 * time.Second)
err := loaded.load(t.Context())
if err != nil {
t.Fatalf("load: %v", err)
}
if len(loaded.entries) != 1 {
t.Errorf("loaded %d entries, want the 1 not expired", len(loaded.entries))
}
cache.sweep(t.Context())
if got := countRows(t, db); got != 1 {
t.Errorf("%d rows after the sweep, want 1", got)
}
if len(cache.entries) != 1 {
t.Errorf("%d entries in memory after the sweep, want 1", len(cache.entries))
}
}
// TestFlushKeepsAnswersItCouldNotWrite makes one write fail by hiding the
// table. The next write, once the table is back, stores the answer.
func TestFlushKeepsAnswersItCouldNotWrite(t *testing.T) {
t.Parallel()
db := openTestDB(t, t.TempDir())
resolver := newFakeResolver(t, ptrReply(300, testName))
cache, _ := newTestCache(t, db, resolver)
lookupOK(t, cache, testAddress)
_, err := db.ExecContext(t.Context(), "ALTER TABLE hostnames RENAME TO hidden")
if err != nil {
t.Fatalf("renaming the table: %v", err)
}
cache.flush(t.Context())
_, err = db.ExecContext(t.Context(), "ALTER TABLE hidden RENAME TO hostnames")
if err != nil {
t.Fatalf("renaming the table back: %v", err)
}
if got := countRows(t, db); got != 0 {
t.Fatalf("%d rows after the failed write, want 0", got)
}
cache.flush(t.Context())
if got := countRows(t, db); got != 1 {
t.Errorf("%d rows after the second write, want 1", got)
}
}
// TestNewAddressIsNotKeptAtTheLimit fills a cache that holds one address. A
// lookup of a second address still returns its answer, and the cache keeps
// nothing for it until the sweep has deleted the first address's expired
// answer.
func TestNewAddressIsNotKeptAtTheLimit(t *testing.T) {
t.Parallel()
db := openTestDB(t, t.TempDir())
resolver := newFakeResolver(t, ptrReply(300, testName))
cache, clock := newTestCache(t, db, resolver)
cache.maxEntries = 1
lookupOK(t, cache, testAddress)
names := lookupOK(t, cache, otherAddress)
if !slices.Equal(names, []string{testName}) {
t.Errorf("names %v at the limit, want %s", names, testName)
}
_, inEntries := cache.entries[otherAddress]
_, inPending := cache.pending[otherAddress]
if inEntries || inPending {
t.Errorf("a new address was kept at the limit: in entries %t, in pending %t",
inEntries, inPending)
}
cache.flush(t.Context())
if got := countRows(t, db); got != 1 {
t.Errorf("%d rows at the limit, want 1", got)
}
clock.advance(300 * time.Second)
cache.sweep(t.Context())
lookupOK(t, cache, otherAddress)
_, inEntries = cache.entries[otherAddress]
if !inEntries {
t.Error("a new address was not kept after the sweep made room")
}
}
// TestHeldAddressIsReplacedAtTheLimit lets the answer of the only address a
// cache holds expire. The new answer replaces it in memory and is queued for
// cache.db.
func TestHeldAddressIsReplacedAtTheLimit(t *testing.T) {
t.Parallel()
resolver := newFakeResolver(t, ptrReply(300, testName))
cache, clock := newTestCache(t, openTestDB(t, t.TempDir()), resolver)
cache.maxEntries = 1
lookupOK(t, cache, testAddress)
clock.advance(300 * time.Second)
resolver.setReply(ptrReply(300, otherName))
lookupOK(t, cache, testAddress)
want := []string{otherName}
if got := cache.entries[testAddress].names; !slices.Equal(got, want) {
t.Errorf("entries holds %v, want %v", got, want)
}
if got := cache.pending[testAddress].names; !slices.Equal(got, want) {
t.Errorf("pending holds %v, want %v", got, want)
}
}
+200
View File
@@ -0,0 +1,200 @@
package ptr
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"math"
"net/http"
"strings"
"time"
"github.com/miekg/dns"
)
const (
// LookupTimeout limits one address's lookup, every resolver included.
LookupTimeout = 2 * time.Second
// minTTL and maxTTL limit how long PTR names are kept.
minTTL = time.Minute
maxTTL = 24 * time.Hour
// maxNegativeTTL limits how long an address with no PTR names is
// remembered as having none.
maxNegativeTTL = time.Hour
// dnsMessageType is the content type of RFC 8484 requests and answers.
dnsMessageType = "application/dns-message"
// maxMessageBytes is the largest possible DNS message.
maxMessageBytes = 65535
)
var (
errBadStatus = errors.New("unexpected HTTP status")
errBadResponse = errors.New("resolver reported an error")
)
// Resolvers returns the URLs of the public DNS-over-HTTPS resolvers, in the
// order they are tried.
func Resolvers() []string {
return []string{
"https://dns.google/dns-query",
"https://cloudflare-dns.com/dns-query",
"https://unfiltered.adguard-dns.com/dns-query",
"https://dns.quad9.net/dns-query",
}
}
// answer is a resolver's answer for one address.
type answer struct {
// names are the address's PTR names, nil when it has none.
names []string
// ttl is how long the answer may be kept. Zero means it is not kept.
ttl time.Duration
}
// lookup asks the resolvers in turn for the PTR names of the reverse name
// until one answers. It fails when none has answered within the time limit.
func (c *Cache) lookup(ctx context.Context, reverseName string) (answer, error) {
ctx, cancel := context.WithTimeout(ctx, c.timeLimit)
defer cancel()
query := new(dns.Msg).SetQuestion(reverseName, dns.TypePTR)
query.Id = 0 // RFC 8484 asks for 0, so that equal queries can be cached.
wire, err := query.Pack()
if err != nil {
return answer{}, fmt.Errorf("packing the query: %w", err)
}
var failures []error
for i, resolver := range c.resolvers {
// Each resolver gets an equal share of the time left, so that one
// that never answers leaves time for the next.
deadline, _ := ctx.Deadline()
share := time.Until(deadline) / time.Duration(len(c.resolvers)-i)
result, err := c.ask(ctx, resolver, wire, share)
if err == nil {
return result, nil
}
failures = append(failures, fmt.Errorf("%s: %w", resolver, err))
}
return answer{}, errors.Join(failures...)
}
// ask sends the query to one resolver and waits at most timeout for its answer.
func (c *Cache) ask(
ctx context.Context, resolver string, query []byte, timeout time.Duration,
) (answer, error) {
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
request, err := http.NewRequestWithContext(
ctx, http.MethodPost, resolver, bytes.NewReader(query))
if err != nil {
return answer{}, err
}
request.Header.Set("Content-Type", dnsMessageType)
request.Header.Set("Accept", dnsMessageType)
response, err := c.client.Do(request)
if err != nil {
return answer{}, err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusOK {
return answer{}, fmt.Errorf("%w: %d", errBadStatus, response.StatusCode)
}
body, err := io.ReadAll(io.LimitReader(response.Body, maxMessageBytes))
if err != nil {
return answer{}, err
}
var reply dns.Msg
err = reply.Unpack(body)
if err != nil {
return answer{}, err
}
return answerFrom(&reply)
}
// answerFrom reads a resolver's reply. A name that does not exist and a name
// with no PTR records are answers. Every other response code is a failure.
func answerFrom(reply *dns.Msg) (answer, error) {
if reply.Rcode != dns.RcodeSuccess && reply.Rcode != dns.RcodeNameError {
return answer{}, fmt.Errorf("%w: %s",
errBadResponse, dns.RcodeToString[reply.Rcode])
}
var names []string
seconds := uint32(math.MaxUint32)
for _, record := range reply.Answer {
// The chain to a PTR record can hold CNAME records, and the answer
// is good only as long as every record in it.
seconds = min(seconds, record.Header().Ttl)
ptr, ok := record.(*dns.PTR)
if ok && isHostName(ptr.Ptr) {
names = append(names, ptr.Ptr)
}
}
if len(names) == 0 {
return answer{ttl: negativeTTL(reply.Ns)}, nil
}
ttl := time.Duration(seconds) * time.Second
return answer{names: names, ttl: min(max(ttl, minTTL), maxTTL)}, nil
}
// negativeTTL returns how long a reply with no PTR names may be kept: the
// smaller of the SOA record's own TTL and its minimum field (RFC 2308), at most
// maxNegativeTTL. Without a SOA record it is zero.
func negativeTTL(authority []dns.RR) time.Duration {
for _, record := range authority {
soa, ok := record.(*dns.SOA)
if ok {
ttl := time.Duration(min(soa.Hdr.Ttl, soa.Minttl)) * time.Second
return min(ttl, maxNegativeTTL)
}
}
return 0
}
// isHostName reports whether name is a host name. A PTR record can hold any
// bytes, and the names go into web pages, so anything else is dropped, as
// net.Resolver.LookupAddr drops it. The root name is not a host name.
func isHostName(name string) bool {
_, ok := dns.IsDomainName(name)
return ok && name != "." && !strings.ContainsFunc(name, notInHostName)
}
// notInHostName reports whether r cannot be part of a host name.
func notInHostName(r rune) bool {
return !strings.ContainsRune(hostNameCharacters, r)
}
// hostNameCharacters are the characters of a host name, in presentation format.
const hostNameCharacters = "abcdefghijklmnopqrstuvwxyz" +
"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_."
+63 -2
View File
@@ -9,17 +9,20 @@ package routewatch
import (
"context"
"database/sql"
"errors"
"fmt"
"path/filepath"
"sync"
"time"
"git.eeqj.de/sneak/routewatch/internal/cachedb"
"git.eeqj.de/sneak/routewatch/internal/config"
"git.eeqj.de/sneak/routewatch/internal/database"
"git.eeqj.de/sneak/routewatch/internal/iplist"
"git.eeqj.de/sneak/routewatch/internal/logger"
"git.eeqj.de/sneak/routewatch/internal/metrics"
"git.eeqj.de/sneak/routewatch/internal/ptr"
"git.eeqj.de/sneak/routewatch/internal/routetable"
"git.eeqj.de/sneak/routewatch/internal/server"
"git.eeqj.de/sneak/routewatch/internal/streamer"
@@ -62,6 +65,8 @@ type RouteWatch struct {
asnFetcher *ASNFetcher
dbMaintainer *DBMaintainer
ipLists *iplist.IPLists
cacheDB *sql.DB
ptrCache *ptr.Cache
}
// ipListsDirName is the directory in the state directory that holds the IP
@@ -150,6 +155,11 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
rw.ipLists = ipLists
rw.server.SetIPLists(ipLists)
err = rw.startPTRCache(ctx)
if err != nil {
return err
}
// Start HTTP server
err = rw.server.Start(ctx)
if err != nil {
@@ -168,8 +178,8 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
// Shutdown stops all services in order: the streamer, the handlers, which
// write what they hold, the WHOIS fetcher, the database maintainer, the
// IP list downloads and the HTTP server, and last the database, which it
// checkpoints and closes. Each stage waits at most until ctx ends, so
// IP list downloads, the HTTP server and the PTR cache, and last the database,
// which it checkpoints and closes. Each stage waits at most until ctx ends, so
// Shutdown returns soon after: a handler logs what it could not write by
// then, and the HTTP server closes the connections of requests still
// running.
@@ -229,6 +239,8 @@ func (rw *RouteWatch) Shutdown(ctx context.Context) {
"error", err)
}
rw.stopPTRCache(ctx)
rw.closeDatabase(ctx)
// Log final metrics
@@ -242,6 +254,55 @@ func (rw *RouteWatch) Shutdown(ctx context.Context) {
)
}
// startPTRCache opens cache.db, loads the PTR cache from it and gives the
// server the cache. The HTTP handlers read it without a lock, so it is set
// before they run.
func (rw *RouteWatch) startPTRCache(ctx context.Context) error {
cacheDB, err := cachedb.Open(ctx, rw.config.StateDir)
if err != nil {
return err
}
ptrCache := ptr.New(ptr.Params{
DB: cacheDB,
Resolvers: ptr.Resolvers(),
Logger: rw.logger.Logger,
})
err = ptrCache.Start(ctx)
if err != nil {
_ = cacheDB.Close()
return err
}
rw.cacheDB = cacheDB
rw.ptrCache = ptrCache
rw.server.SetPTRResolver(ptrCache)
return nil
}
// stopPTRCache stops the PTR cache, which writes the answers cache.db does not
// have yet, and closes cache.db, unless ctx has ended and a write may still be
// running.
func (rw *RouteWatch) stopPTRCache(ctx context.Context) {
if rw.ptrCache == nil {
return
}
rw.ptrCache.Stop(ctx)
if ctx.Err() != nil {
return
}
err := rw.cacheDB.Close()
if err != nil {
rw.logger.Error("Failed to close the cache database", "error", err)
}
}
// closeDatabase checkpoints and closes the database, unless ctx has ended: a
// write may then still be running, and both would wait for it. SQLite rolls
// back a write cut off by the exit when the database next opens.
@@ -8,6 +8,8 @@ import (
"net/http"
"net/http/httptest"
"net/netip"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
@@ -783,3 +785,74 @@ func TestShutdownCheckpointsAndClosesTheDatabaseLast(t *testing.T) {
t.Errorf("Shutdown made the calls %q, want %q", store.calls, want)
}
}
// TestPTRCacheAnswersTheIPLookup starts the PTR cache and the HTTP server as
// Run does, and looks up a reserved address, which asks no resolver. Without
// the PTR cache the server has no PTR resolver, and every IP lookup answers
// 500.
func TestPTRCacheAnswersTheIPLookup(t *testing.T) {
t.Parallel()
store := newMockStore()
log := logger.New()
port := freePort(t)
cfg := &config.Config{Port: port, StateDir: t.TempDir()}
s := streamer.New(log, metrics.New())
// A started server reads its routing table in the background, so it
// needs one.
rw := New(Dependencies{
DB: store,
Streamer: s,
Server: server.New(store, routetable.New(routetable.Params{}), s, log,
cfg),
Logger: log,
Config: cfg,
})
err := rw.startPTRCache(t.Context())
if err != nil {
t.Fatalf("failed to start the PTR cache: %v", err)
}
defer rw.stopPTRCache(t.Context())
err = rw.server.Start(t.Context())
if err != nil {
t.Fatalf("failed to start the HTTP server: %v", err)
}
defer func() { _ = rw.server.Stop(t.Context()) }()
req, err := http.NewRequestWithContext(t.Context(), http.MethodGet,
"http://"+net.JoinHostPort("127.0.0.1", strconv.Itoa(port))+
"/api/v1/ip/10.0.0.1", nil)
if err != nil {
t.Fatalf("failed to build the request: %v", err)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("the IP lookup failed: %v", err)
}
defer func() { _ = resp.Body.Close() }()
body, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatalf("failed to read the answer: %v", err)
}
if resp.StatusCode != http.StatusOK {
t.Fatalf("status %d, want %d; body %s", resp.StatusCode, http.StatusOK, body)
}
if !strings.Contains(string(body), `"ptr":[]`) {
t.Errorf("answer %s, want an empty ptr", body)
}
_, err = os.Stat(filepath.Join(cfg.StateDir, "cache.db"))
if err != nil {
t.Errorf("cache.db is not in the state directory: %v", err)
}
}
+13 -16
View File
@@ -39,12 +39,9 @@ const (
const (
// maxLookupTargets is the most addresses and host names an IP lookup may
// list, and the most addresses it may have once the names are resolved:
// each address costs two database reads and a PTR lookup.
// each address costs two database reads and a PTR lookup. Ten PTR lookups
// that run to ptr.LookupTimeout fit in the request's 30-second limit.
maxLookupTargets = 10
// ptrLookupTimeout is each PTR lookup's own time limit, so that ten
// addresses whose PTR lookups never answer fit in the 30-second limit.
ptrLookupTimeout = 2 * time.Second
)
// The Cache-Control header of an IP lookup answer, such as
@@ -1032,16 +1029,14 @@ func ipLookupFlags(ip string, listings []iplist.Listing) IPLookupFlags {
}
}
// isBogon says whether the address ip is reserved. An IPv4 address written in
// IPv6 form counts as the IPv4 address it holds.
// isBogon says whether the address ip is reserved, as package bogon tells.
func isBogon(ip string) bool {
addr, err := netip.ParseAddr(ip)
if err != nil {
return false
}
addr = addr.Unmap()
_, reserved := bogon.Prefix(netip.PrefixFrom(addr, addr.BitLen()))
_, reserved := bogon.Addr(addr)
return reserved
}
@@ -1220,15 +1215,17 @@ func publicAddresses(addrs []string) []string {
}
// lookupPTRNames returns the PTR records of ip without their trailing dots. It
// returns an empty list, never nil, when there are none, when the lookup fails
// and when it runs past ptrLookupTimeout, so that an answer's ptr is [] rather
// than null.
// returns an empty list, never nil, when there are none and when the lookup
// fails, so that an answer's ptr is [] rather than null.
func (s *Server) lookupPTRNames(ctx context.Context, ip string) []string {
ctx, cancel := context.WithTimeout(ctx, ptrLookupTimeout)
defer cancel()
ptrs, err := s.ptrResolver.LookupAddr(ctx, ip)
if err != nil {
s.logger.Warn("PTR lookup failed", "ip", ip, "error", err)
ptrs, err := s.resolver.LookupAddr(ctx, ip)
if err != nil || ptrs == nil {
return []string{}
}
if ptrs == nil {
return []string{}
}
+71 -24
View File
@@ -25,6 +25,7 @@ import (
"git.eeqj.de/sneak/routewatch/internal/iplist"
"git.eeqj.de/sneak/routewatch/internal/logger"
"git.eeqj.de/sneak/routewatch/internal/metrics"
"git.eeqj.de/sneak/routewatch/internal/ptr"
"git.eeqj.de/sneak/routewatch/internal/ristypes"
"git.eeqj.de/sneak/routewatch/internal/routetable"
"git.eeqj.de/sneak/routewatch/internal/streamer"
@@ -557,8 +558,11 @@ func newServerWithTable(t *testing.T, table *routetable.Table) *Server {
t.Cleanup(func() { _ = db.Close() })
return New(db, table, streamer.New(logger.New(), metrics.New()), logger.New(),
s := New(db, table, streamer.New(logger.New(), metrics.New()), logger.New(),
&config.Config{})
useFakeResolver(s, &fakeResolver{})
return s
}
// TestPrefixDetailHandlersParsePrefix checks that both prefix pages find a
@@ -1252,7 +1256,7 @@ func TestIPLookupAnswers500WhenDatabaseReadFails(t *testing.T) {
s := New(failingIPLookupDB{}, newTestTable(),
streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
s.resolver = &fakeResolver{}
useFakeResolver(s, &fakeResolver{})
tests := []struct {
path string
@@ -1288,16 +1292,19 @@ func TestIPLookupAnswers500WhenDatabaseReadFails(t *testing.T) {
}
}
// errNoSuchHost is what fakeResolver answers a lookup it has no answer for
// with.
// errNoSuchHost is what fakeResolver answers a host name lookup it has no
// addresses for with.
var errNoSuchHost = errors.New("no such host")
// fakeResolver answers a host name lookup with the addresses hosts lists for
// the name, and a PTR lookup with no names or, when ptrNeverAnswers is set,
// not at all: it waits until the lookup's context ends. hostLookups counts the
// host name lookups.
// the name. It answers a PTR lookup with the names ptrs lists for the address,
// and with no names and no error, as the real one does, when ptrs lists none.
// When ptrNeverAnswers is set, a PTR lookup fails after ptr.LookupTimeout, as
// the real one does when no resolver answers. hostLookups counts the host name
// lookups.
type fakeResolver struct {
hosts map[string][]string
ptrs map[string][]string
ptrNeverAnswers bool
hostLookups atomic.Int64
@@ -1314,14 +1321,23 @@ func (r *fakeResolver) LookupHost(_ context.Context, host string) ([]string, err
return addrs, nil
}
func (r *fakeResolver) LookupAddr(ctx context.Context, _ string) ([]string, error) {
func (r *fakeResolver) LookupAddr(ctx context.Context, addr string) ([]string, error) {
if r.ptrNeverAnswers {
ctx, cancel := context.WithTimeout(ctx, ptr.LookupTimeout)
defer cancel()
<-ctx.Done()
return nil, ctx.Err()
}
return nil, errNoSuchHost
return r.ptrs[addr], nil
}
// useFakeResolver makes r answer the server's host name and PTR lookups.
func useFakeResolver(s *Server, r *fakeResolver) {
s.resolver = r
s.ptrResolver = r
}
// countingIPLookupDB embeds database.Store (left nil) and answers the IP
@@ -1392,7 +1408,7 @@ func TestIPLookupRefusesTooManyAddresses(t *testing.T) {
s := New(db, newTestTable(), streamer.New(logger.New(), metrics.New()),
logger.New(), &config.Config{})
resolver := &fakeResolver{hosts: hosts}
s.resolver = resolver
useFakeResolver(s, resolver)
rec := httptest.NewRecorder()
s.router.ServeHTTP(rec, httptest.NewRequestWithContext(
@@ -1427,18 +1443,19 @@ const addrInTestPrefixV4 = "192.0.2.1"
// TestIPLookupKeepsRouteWhenPTRLookupDoesNotAnswer looks up a routed address
// with a resolver that never answers its PTR lookup. Once the PTR lookup's own
// limit passes, the lookup answers 200 with the route. Before the fix the PTR
// lookup ran until the request's time limit, and the lookup answered 504
// without the route (https://git.eeqj.de/sneak/routewatch/issues/110).
// limit passes, the lookup answers 200 with the route and an empty ptr. Before
// the fix the PTR lookup ran until the request's time limit, and the lookup
// answered 504 without the route
// (https://git.eeqj.de/sneak/routewatch/issues/110).
func TestIPLookupKeepsRouteWhenPTRLookupDoesNotAnswer(t *testing.T) {
t.Parallel()
s := newServerWithTestRoutes(t)
s.resolver = &fakeResolver{ptrNeverAnswers: true}
useFakeResolver(s, &fakeResolver{ptrNeverAnswers: true})
// The request's time limit passes well after the PTR lookup's, so that a
// stall on a busy host does not use up the difference.
const requestTimeLimit = ptrLookupTimeout + 15*time.Second
const requestTimeLimit = ptr.LookupTimeout + 15*time.Second
ctx, cancel := context.WithTimeout(t.Context(), requestTimeLimit)
defer cancel()
@@ -1464,6 +1481,10 @@ func TestIPLookupKeepsRouteWhenPTRLookupDoesNotAnswer(t *testing.T) {
*body.Data.Results[0].Netblock != testPrefixV4 {
t.Errorf("answer %s, want one result in %s", rec.Body, testPrefixV4)
}
if !strings.Contains(rec.Body.String(), `"ptr":[]`) {
t.Errorf("answer %s, want an empty ptr list", rec.Body)
}
}
// addrInTestPrefixV6 is an address in testPrefixV6, which
@@ -1541,7 +1562,7 @@ func TestIPLookupFlags(t *testing.T) {
t.Parallel()
s := newServerWithTestRoutes(t)
s.resolver = &fakeResolver{}
useFakeResolver(s, &fakeResolver{})
s.SetIPLists(fakeIPLists{})
tests := []struct {
@@ -1654,7 +1675,7 @@ func TestIPLookupReportsIPLists(t *testing.T) {
t.Parallel()
s := newServerWithTestRoutes(t)
s.resolver = &fakeResolver{}
useFakeResolver(s, &fakeResolver{})
s.SetIPLists(fakeIPLists{})
rec := httptest.NewRecorder()
@@ -1731,7 +1752,7 @@ func TestIPLookupReportsIPListsOfAnAddressWithNoRoute(t *testing.T) {
t.Parallel()
s := newServerWithTestRoutes(t)
s.resolver = &fakeResolver{}
useFakeResolver(s, &fakeResolver{})
s.SetIPLists(fakeIPLists{})
// routeFields are the fields of a result that come from the address's
@@ -1809,9 +1830,9 @@ func TestIPLookupResultsHaveTheSameFields(t *testing.T) {
t.Parallel()
s := newServerWithTestRoutes(t)
s.resolver = &fakeResolver{hosts: map[string][]string{
useFakeResolver(s, &fakeResolver{hosts: map[string][]string{
twoAddressHostName: {addrInTestPrefixV4, listedUnroutedAddr},
}}
}})
s.SetIPLists(fakeIPLists{})
// want is the fields of the first result.
@@ -1844,7 +1865,7 @@ func TestIPLookupOfAnAddressWithoutPTRNamesOrStoredNetblock(t *testing.T) {
t.Parallel()
s := newServerWithTestRoutes(t)
s.resolver = &fakeResolver{}
useFakeResolver(s, &fakeResolver{})
path := "/api/v1/ip/" + addrInTestPrefixV4
@@ -1866,6 +1887,32 @@ func TestIPLookupOfAnAddressWithoutPTRNamesOrStoredNetblock(t *testing.T) {
}
}
// TestIPLookupGivesPTRNamesWithoutTrailingDots looks up an address whose PTR
// name the resolver gives fully qualified. The result's ptr holds the name
// without its trailing dot.
func TestIPLookupGivesPTRNamesWithoutTrailingDots(t *testing.T) {
t.Parallel()
s := newServerWithTestRoutes(t)
useFakeResolver(s, &fakeResolver{ptrs: map[string][]string{
addrInTestPrefixV4: {"host.example.com."},
}})
path := "/api/v1/ip/" + addrInTestPrefixV4
results := resultFields(t, get(t, s, path).Body.Bytes())
if len(results) != 1 {
t.Fatalf("%s: %d results, want 1", path, len(results))
}
got := string(results[0]["ptr"])
want := `["host.example.com"]`
if got != want {
t.Errorf("%s: ptr is %s, want %s", path, got, want)
}
}
// TestIPLookupAnswersMayBeCached checks the Cache-Control header of the IP
// lookup answers. Caches may keep them for the IP lists' shortest interval,
// and only the caller's own browser may keep the answer about the caller's
@@ -1874,7 +1921,7 @@ func TestIPLookupAnswersMayBeCached(t *testing.T) {
t.Parallel()
s := newServerWithTestRoutes(t)
s.resolver = &fakeResolver{}
useFakeResolver(s, &fakeResolver{})
s.SetIPLists(fakeIPLists{})
tests := []struct {
@@ -1925,10 +1972,10 @@ func TestIPLookupHidesPrivateAddresses(t *testing.T) {
t.Parallel()
s := newServerWithTestRoutes(t)
s.resolver = &fakeResolver{hosts: map[string][]string{
useFakeResolver(s, &fakeResolver{hosts: map[string][]string{
internalHostName: {internalHostAddr},
"mixed.test": {internalHostAddr, addrInTestPrefixV4},
}}
}})
notPublic := internalHostName + ": " + hostNotPublicMessage
@@ -85,7 +85,7 @@ func TestIPLookupGivesTheOriginMostPeersCarry(t *testing.T) {
t.Parallel()
s := newServerWithTable(t, newTwoOriginTable())
s.resolver = &fakeResolver{}
useFakeResolver(s, &fakeResolver{})
firstSeen := testRouteTime(-1)
+4 -4
View File
@@ -115,7 +115,7 @@ func TestJSONAnswersPutNoUnitInAString(t *testing.T) {
`^([0-9]+(\.[0-9]+)? ?(ns|µs|ms|s|m|h|B|kB|MB|GB|TB))+$`)
s := newServerWithTestRoutes(t)
s.resolver = &fakeResolver{}
useFakeResolver(s, &fakeResolver{})
s.SetASNFetcher(fixedIntervalASNFetcher{interval: time.Second})
paths := []string{
@@ -179,7 +179,7 @@ func TestJSONAnswersCarryMeta(t *testing.T) {
)
s := newServerWithTestRoutes(t)
s.resolver = &fakeResolver{}
useFakeResolver(s, &fakeResolver{})
s.streamer.GetMetricsTracker().SetConnected(true)
s.router.Get(writesNothingPath, JSONValidationMiddleware(http.HandlerFunc(
func(http.ResponseWriter, *http.Request) {})).ServeHTTP)
@@ -188,7 +188,7 @@ func TestJSONAnswersCarryMeta(t *testing.T) {
failing := New(failingDB{}, newTestTable(),
streamer.New(logger.New(), metrics.New()), logger.New(), &config.Config{})
failing.resolver = &fakeResolver{}
useFakeResolver(failing, &fakeResolver{})
tests := []struct {
method string
@@ -521,7 +521,7 @@ func newServerWithConfig(
s := New(db, newTestTable(), streamer.New(logger.New(), metrics.New()),
&logger.Logger{Logger: slog.New(slog.NewJSONHandler(requestLog, nil))}, cfg)
s.resolver = &fakeResolver{}
useFakeResolver(s, &fakeResolver{})
return s
}
+16 -3
View File
@@ -52,10 +52,14 @@ type IPLists interface {
ShortestInterval() time.Duration
}
// dnsResolver looks up the addresses of host names and the names of addresses
// in DNS, as *net.Resolver does.
// dnsResolver looks up the addresses of host names in DNS, as *net.Resolver
// does.
type dnsResolver interface {
LookupHost(ctx context.Context, host string) ([]string, error)
}
// PTRResolver looks up the PTR names of addresses, as *ptr.Cache does.
type PTRResolver interface {
LookupAddr(ctx context.Context, addr string) ([]string, error)
}
@@ -75,10 +79,13 @@ type Server struct {
// RealIPMiddleware believes
trustedProxies []netip.Prefix
// resolver makes the IP lookup's DNS lookups: net.DefaultResolver, or a
// resolver looks up the IP lookup's host names: net.DefaultResolver, or a
// fake in tests
resolver dnsResolver
// ptrResolver looks up the IP lookup's PTR names. SetPTRResolver sets it.
ptrResolver PTRResolver
// cards is the copy of the home page's cards that the pages serve, nil
// until the server first works them out
cards atomic.Pointer[homeCards]
@@ -196,6 +203,12 @@ func (s *Server) Stop(ctx context.Context) error {
return nil
}
// SetPTRResolver sets what looks up the IP lookup's PTR names. Call it before
// Start: the handlers read it without a lock.
func (s *Server) SetPTRResolver(resolver PTRResolver) {
s.ptrResolver = resolver
}
// SetASNFetcher sets the ASN WHOIS fetcher for on-demand lookups. Call it
// before Start: the handlers read the fetcher without a lock.
func (s *Server) SetASNFetcher(fetcher ASNFetcher) {