Add asn= and prefix= tags from a routewatch instance (closes #204)

SWW_ROUTEWATCH_URL names the instance. The new heuristic routewatch_tags
asks it in the background about each client and tags the request. Answers
are kept in memory for an hour. A request without tags gives the reason in
routewatch_tags_missing, and it is never delayed or refused.

Model: opus-5-5

README.md

@@ -633,8 +633,8 @@ is set aside as `<name>.bad`, and never stops the running process (see
 - `internal/discriminator`: `discriminator`'s server, which runs the checks and
   answers its own endpoints.
 - `internal/ratelimit`, `internal/bans`, `internal/rules`, `internal/waf`,
-  `internal/lookup`, `internal/reputation`, `internal/anomaly` and
+  `internal/lookup`, `internal/reputation`, `internal/routewatch`,
-  `internal/alerts`: one check or source each.
+  `internal/anomaly` and `internal/alerts`: one check or source each.
 - `internal/state`, `internal/requestlog`, `internal/remotelog` and
   `internal/metrics`: the state files, the log lines, the syslog sender and the
   metrics.

SPEC.md

@@ -87,6 +87,8 @@ to them.
       database.
     - `internal/reputation`: fetches the lists, asks the DNS blocklists and
       AbuseIPDB in the background, and keeps their answers.
+    - `internal/routewatch`: asks a routewatch instance about clients in the
+      background, and keeps its answers.
     - `internal/anomaly`: the anomaly counters and their alerts.
     - `internal/alerts`: holds back repeats and alerts past the hourly limit,
       and sends the rest to the webhook, Slack and ntfy, each from a queue of
@@ -156,8 +158,8 @@ request log's `heuristic` field gives that name. `refusal_limit` and
    up to `SWW_LOOKUP_TIMEOUT`, only when one of these needs it:
     - the country list;
     - a biased threshold;
-    - `SWW_TAG_DENY` or `SWW_TAG_LIMIT_PERCENT` naming a `country=` tag, or an
+    - `SWW_TAG_DENY` or `SWW_TAG_LIMIT_PERCENT` naming an `asn=` or `country=`
-      `asn=` tag while `SWW_ROUTEWATCH_URL` is unset;
+      tag;
     - `SWW_ADD_LOOKUP_HEADERS`.
 
     A request that ends before the answer comes is logged without it.
@@ -560,9 +562,11 @@ Each tag and where it comes from:
   (`routewatch_tags`), while `SWW_ROUTEWATCH_URL` is set:
     - The tag `prefix=` holds the netblock the client's address is announced in,
       such as `prefix=192.0.2.0/24`.
-    - An `asn=` tag has the same spelling as the lookup's. A request carries it
+    - An `asn=` tag has the same spelling as the lookup's. When routewatch and
-      once, and carries both when the two sources disagree.
+      the lookup give the same AS number, a request has one `asn=` tag. When
-    - These tags come before `deny_nets`, so a line that check refused has them.
+      they differ, it has both.
+    - They are added before `deny_nets`, so the log line of a request it refused
+      has them.
     - `discriminator` asks routewatch in the background and never waits for it.
       An answer is kept for 1 hour, in memory only.
     - After a query fails, routewatch is not asked for 1 minute.
@@ -586,7 +590,9 @@ Two settings act on tags:
 
 The tag settings are empty by default, for the same reason as the country list.
 The lookup waits for GeoJS when either names an `asn=` or `country=` tag. A
-setting that names one stops the start while `SWW_LOOKUP_SOURCE` is `off`.
+setting that names one stops the start while `SWW_LOOKUP_SOURCE` is `off`. The
+exception is an `asn=` tag while `SWW_ROUTEWATCH_URL` is set, since routewatch
+gives that tag.
 
 ### Biased thresholds
 

docs/settings.md

@@ -145,8 +145,8 @@ cut off; the client is banned.
   the start while a setting that needs a lookup is set:
     - `SWW_EXCLUSIVELY_ALLOWED_COUNTRIES`;
     - `SWW_ADD_LOOKUP_HEADERS`;
-    - `SWW_TAG_DENY` or `SWW_TAG_LIMIT_PERCENT` naming an `asn=` or `country=`
+    - `SWW_TAG_DENY` or `SWW_TAG_LIMIT_PERCENT` naming a `country=` tag, or an
-      tag;
+      `asn=` tag while `SWW_ROUTEWATCH_URL` is unset;
     - the two byte percentage lists and `SWW_ASN_LIMIT_PERCENT_URL`;
     - `SWW_UNKNOWN_LIMIT_PERCENT` below `100`;
     - an anomaly threshold per AS number.
@@ -187,10 +187,11 @@ Every request gets tags from what `discriminator` knows of its client. See
   it.
 
 - `SWW_ROUTEWATCH_URL` (default unset): the `http` or `https` URL of a
-  routewatch instance. `discriminator` asks it about each client, in the
+  routewatch instance. [routewatch](https://git.eeqj.de/sneak/routewatch) is a
-  background, and adds the tags `asn=<number>` and `prefix=<netblock>` from its
+  service that gives an address's announced netblock and origin AS.
-  answer. While it is unset, nothing asks routewatch and no request gets a
+  `discriminator` asks it about each client, in the background, and adds the
-  `prefix=` tag.
+  tags `asn=<number>` and `prefix=<netblock>` from its answer. While it is
+  unset, nothing asks routewatch and no request gets a `prefix=` tag.
 
 `SWW_TAG_DENY` and `SWW_TAG_LIMIT_PERCENT` accept tags in either case. A tag
 that no request can get stops the start. A request can get only these:

internal/config/config.go

@@ -1198,7 +1198,7 @@ func requestCanGetTag(cfg *Config, tag string) bool {
 }
 
 // TagsNeedLookup reports whether SWW_TAG_DENY or SWW_TAG_LIMIT_PERCENT
-// names an asn= or country= tag, which only the lookup can give a request.
+// names an asn= or country= tag.
 func (c *Config) TagsNeedLookup() bool {
 	return slices.ContainsFunc(c.TagDeny, isLookupTag) || tagLimitNeedsLookups(c)
 }

internal/discriminator/routewatchtags_test.go

@@ -117,6 +117,20 @@ func TestRouteWatchTagsAreOnALineDenyNetsRefused(t *testing.T) {
 	wantTags(t, line, "asn=64511", "prefix="+prefixKP)
 }
 
+func TestRouteWithoutAnOriginASGivesOnlyThePrefixTag(t *testing.T) {
+	t.Parallel()
+
+	url := startRouteWatch(t, map[string]standInRoute{fromDE: {prefixDE, 0}})
+	s, _, server := startWithClock(t, "", map[string]string{
+		lookupSource: off, routeWatchURL: url,
+	})
+
+	waitForRouteWatch(t, server, fromDE, 32)
+
+	wantTags(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward),
+		"prefix="+prefixDE)
+}
+
 func TestUnsetRouteWatchChangesNothing(t *testing.T) {
 	t.Parallel()
 

internal/routewatch/routewatch.go

@@ -25,8 +25,9 @@ const (
 	// failureKept is how long routewatch is not asked again after a query
 	// fails, so that a dead routewatch is not asked on every request.
 	failureKept = time.Minute
-	// queryTimeout is how long one query may take.
+	// queryTimeout is how long one query may take. routewatch answers only
-	queryTimeout = 2 * time.Second
+	// after a reverse DNS lookup it allows 2 seconds, so this is longer.
+	queryTimeout = 5 * time.Second
 	// maxAnswers is how many answers are kept. Past it, the one that came
 	// longest ago is dropped.
 	maxAnswers = 100000

internal/routewatch/routewatch_test.go

@@ -21,7 +21,10 @@ const (
 	routedAnswer = `{"status":"ok","data":{"results":[` +
 		`{"ip":"192.0.2.1","netblock":"192.0.2.0/24","asn":64496}]}}`
 	noRouteAnswer = `{"status":"ok","data":{"results":[{"ip":"192.0.2.1"}]}}`
-	waitLimit     = 5 * time.Second
+	// slowAnswer is longer than routewatch allows a reverse DNS lookup.
+	slowAnswer = 2500 * time.Millisecond
+	// waitLimit is longer than the Client allows a query.
+	waitLimit = 10 * time.Second
 )
 
 // clock is a time the tests move by hand.
@@ -49,6 +52,7 @@ func (c *clock) advance(by time.Duration) {
 // the process log.
 type fixture struct {
 	t       *testing.T
+	server  *httptest.Server
 	client  *routewatch.Client
 	clk     *clock
 	queries *atomic.Int32
@@ -87,16 +91,16 @@ func newFixture(t *testing.T, standIn http.HandlerFunc) *fixture {
 		log:     &syncBuffer{},
 	}
 
-	server := httptest.NewServer(http.HandlerFunc(
+	f.server = httptest.NewServer(http.HandlerFunc(
 		func(w http.ResponseWriter, r *http.Request) {
 			f.queries.Add(1)
 			f.path.Store(r.URL.Path)
 			standIn(w, r)
 		}))
-	t.Cleanup(server.Close)
+	t.Cleanup(f.server.Close)
 
 	f.client = routewatch.New(routewatch.Params{
-		URL: server.URL + "/", Now: f.clk.Now,
+		URL: f.server.URL + "/", Now: f.clk.Now,
 		ProcessLog: slog.New(slog.NewTextHandler(f.log, nil)),
 	})
 
@@ -137,6 +141,26 @@ func (f *fixture) waitFor(why routewatch.Missing) routewatch.Route {
 	}
 }
 
+// wantFailureLoggedWithoutTheClient waits for the process log to name a failed
+// query, and checks that it does not name the client.
+func (f *fixture) wantFailureLoggedWithoutTheClient() {
+	f.t.Helper()
+
+	// The failure is noted before it is logged.
+	deadline := time.Now().Add(waitLimit)
+	for !strings.Contains(f.log.String(), "asking routewatch failed") {
+		if time.Now().After(deadline) {
+			f.t.Fatalf("process log %q names no failure", f.log.String())
+		}
+
+		time.Sleep(time.Millisecond)
+	}
+
+	if strings.Contains(f.log.String(), clientAddr) {
+		f.t.Errorf("process log %q names the client", f.log.String())
+	}
+}
+
 func TestRouteGivesRoutewatchsAnswerAndAsksByAddress(t *testing.T) {
 	t.Parallel()
 
@@ -158,6 +182,25 @@ func TestRouteGivesRoutewatchsAnswerAndAsksByAddress(t *testing.T) {
 	}
 }
 
+func TestAnswerAfterASlowReverseDNSLookupIsKept(t *testing.T) {
+	t.Parallel()
+
+	f := newFixture(t, func(w http.ResponseWriter, _ *http.Request) {
+		time.Sleep(slowAnswer)
+
+		_, _ = w.Write([]byte(routedAnswer))
+	})
+
+	f.route()
+
+	route := f.waitFor("")
+
+	want := routewatch.Route{Netblock: netip.MustParsePrefix("192.0.2.0/24"), ASN: 64496}
+	if route != want {
+		t.Errorf("route %+v, want %+v", route, want)
+	}
+}
+
 func TestAddressWithoutARouteIsKeptAsNoRoute(t *testing.T) {
 	t.Parallel()
 
@@ -191,24 +234,23 @@ func TestAnswerRoutewatchCannotBeTakenFromFails(t *testing.T) {
 
 			f.route()
 			f.waitFor(routewatch.RequestFailed)
-
+			f.wantFailureLoggedWithoutTheClient()
-			// The failure is noted before it is logged.
-			deadline := time.Now().Add(waitLimit)
-			for !strings.Contains(f.log.String(), "asking routewatch failed") {
-				if time.Now().After(deadline) {
-					t.Fatalf("process log %q names no failure", f.log.String())
-				}
-
-				time.Sleep(time.Millisecond)
-			}
-
-			if strings.Contains(f.log.String(), clientAddr) {
-				t.Errorf("process log %q names the client", f.log.String())
-			}
 		})
 	}
 }
 
+func TestRefusedConnectionFailsTheQueryAndTheLogOmitsTheClient(t *testing.T) {
+	t.Parallel()
+
+	f := newFixture(t, answering(routedAnswer))
+
+	f.server.Close()
+
+	f.route()
+	f.waitFor(routewatch.RequestFailed)
+	f.wantFailureLoggedWithoutTheClient()
+}
+
 func TestFailureIsKeptForAMinuteSoRoutewatchIsNotAskedPerRequest(t *testing.T) {
 	t.Parallel()
 
@@ -253,6 +295,7 @@ func TestHangingRoutewatchNeverDelaysAndFailsTheQuery(t *testing.T) {
 	}
 
 	f.waitFor(routewatch.RequestFailed)
+	f.wantFailureLoggedWithoutTheClient()
 }
 
 func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {