Add asn= and prefix= tags from a routewatch instance (closes #204)

SWW_ROUTEWATCH_URL names the instance. The new heuristic routewatch_tags
asks it in the background about each client and tags the request. Answers
are kept in memory for an hour. A request without tags gives the reason in
routewatch_tags_missing, and it is never delayed or refused.

Model: opus-5-5

README.md

@@ -633,8 +633,8 @@ is set aside as `<name>.bad`, and never stops the running process (see
 - `internal/discriminator`: `discriminator`'s server, which runs the checks and
   answers its own endpoints.
 - `internal/ratelimit`, `internal/bans`, `internal/rules`, `internal/waf`,
-  `internal/lookup`, `internal/reputation`, `internal/routewatch`,
+  `internal/lookup`, `internal/reputation`, `internal/anomaly` and
-  `internal/anomaly` and `internal/alerts`: one check or source each.
+  `internal/alerts`: one check or source each.
 - `internal/state`, `internal/requestlog`, `internal/remotelog` and
   `internal/metrics`: the state files, the log lines, the syslog sender and the
   metrics.

SPEC.md

@@ -87,8 +87,6 @@ to them.
       database.
     - `internal/reputation`: fetches the lists, asks the DNS blocklists and
       AbuseIPDB in the background, and keeps their answers.
-    - `internal/routewatch`: asks a routewatch instance about clients in the
-      background, and keeps its answers.
     - `internal/anomaly`: the anomaly counters and their alerts.
     - `internal/alerts`: holds back repeats and alerts past the hourly limit,
       and sends the rest to the webhook, Slack and ntfy, each from a queue of
@@ -158,8 +156,8 @@ request log's `heuristic` field gives that name. `refusal_limit` and
    up to `SWW_LOOKUP_TIMEOUT`, only when one of these needs it:
     - the country list;
     - a biased threshold;
-    - `SWW_TAG_DENY` or `SWW_TAG_LIMIT_PERCENT` naming an `asn=` or `country=`
+    - `SWW_TAG_DENY` or `SWW_TAG_LIMIT_PERCENT` naming a `country=` tag, or an
-      tag;
+      `asn=` tag while `SWW_ROUTEWATCH_URL` is unset;
     - `SWW_ADD_LOOKUP_HEADERS`.
 
     A request that ends before the answer comes is logged without it.
@@ -562,11 +560,9 @@ Each tag and where it comes from:
   (`routewatch_tags`), while `SWW_ROUTEWATCH_URL` is set:
     - The tag `prefix=` holds the netblock the client's address is announced in,
       such as `prefix=192.0.2.0/24`.
-    - An `asn=` tag has the same spelling as the lookup's. When routewatch and
+    - An `asn=` tag has the same spelling as the lookup's. A request carries it
-      the lookup give the same AS number, a request has one `asn=` tag. When
+      once, and carries both when the two sources disagree.
-      they differ, it has both.
+    - These tags come before `deny_nets`, so a line that check refused has them.
-    - They are added before `deny_nets`, so the log line of a request it refused
-      has them.
     - `discriminator` asks routewatch in the background and never waits for it.
       An answer is kept for 1 hour, in memory only.
     - After a query fails, routewatch is not asked for 1 minute.
@@ -590,9 +586,7 @@ Two settings act on tags:
 
 The tag settings are empty by default, for the same reason as the country list.
 The lookup waits for GeoJS when either names an `asn=` or `country=` tag. A
-setting that names one stops the start while `SWW_LOOKUP_SOURCE` is `off`. The
+setting that names one stops the start while `SWW_LOOKUP_SOURCE` is `off`.
-exception is an `asn=` tag while `SWW_ROUTEWATCH_URL` is set, since routewatch
-gives that tag.
 
 ### Biased thresholds
 

docs/settings.md

@@ -145,8 +145,8 @@ cut off; the client is banned.
   the start while a setting that needs a lookup is set:
     - `SWW_EXCLUSIVELY_ALLOWED_COUNTRIES`;
     - `SWW_ADD_LOOKUP_HEADERS`;
-    - `SWW_TAG_DENY` or `SWW_TAG_LIMIT_PERCENT` naming a `country=` tag, or an
+    - `SWW_TAG_DENY` or `SWW_TAG_LIMIT_PERCENT` naming an `asn=` or `country=`
-      `asn=` tag while `SWW_ROUTEWATCH_URL` is unset;
+      tag;
     - the two byte percentage lists and `SWW_ASN_LIMIT_PERCENT_URL`;
     - `SWW_UNKNOWN_LIMIT_PERCENT` below `100`;
     - an anomaly threshold per AS number.
@@ -187,11 +187,10 @@ Every request gets tags from what `discriminator` knows of its client. See
   it.
 
 - `SWW_ROUTEWATCH_URL` (default unset): the `http` or `https` URL of a
-  routewatch instance. [routewatch](https://git.eeqj.de/sneak/routewatch) is a
+  routewatch instance. `discriminator` asks it about each client, in the
-  service that gives an address's announced netblock and origin AS.
+  background, and adds the tags `asn=<number>` and `prefix=<netblock>` from its
-  `discriminator` asks it about each client, in the background, and adds the
+  answer. While it is unset, nothing asks routewatch and no request gets a
-  tags `asn=<number>` and `prefix=<netblock>` from its answer. While it is
+  `prefix=` tag.
-  unset, nothing asks routewatch and no request gets a `prefix=` tag.
 
 `SWW_TAG_DENY` and `SWW_TAG_LIMIT_PERCENT` accept tags in either case. A tag
 that no request can get stops the start. A request can get only these:

internal/config/config.go

@@ -1198,7 +1198,7 @@ func requestCanGetTag(cfg *Config, tag string) bool {
 }
 
 // TagsNeedLookup reports whether SWW_TAG_DENY or SWW_TAG_LIMIT_PERCENT
-// names an asn= or country= tag.
+// names an asn= or country= tag, which only the lookup can give a request.
 func (c *Config) TagsNeedLookup() bool {
 	return slices.ContainsFunc(c.TagDeny, isLookupTag) || tagLimitNeedsLookups(c)
 }

internal/discriminator/routewatchtags_test.go

@@ -117,20 +117,6 @@ func TestRouteWatchTagsAreOnALineDenyNetsRefused(t *testing.T) {
 	wantTags(t, line, "asn=64511", "prefix="+prefixKP)
 }
 
-func TestRouteWithoutAnOriginASGivesOnlyThePrefixTag(t *testing.T) {
-	t.Parallel()
-
-	url := startRouteWatch(t, map[string]standInRoute{fromDE: {prefixDE, 0}})
-	s, _, server := startWithClock(t, "", map[string]string{
-		lookupSource: off, routeWatchURL: url,
-	})
-
-	waitForRouteWatch(t, server, fromDE, 32)
-
-	wantTags(t, s.get(fromDE, http.StatusOK, requestlog.ActionForward),
-		"prefix="+prefixDE)
-}
-
 func TestUnsetRouteWatchChangesNothing(t *testing.T) {
 	t.Parallel()
 

internal/routewatch/routewatch.go

@@ -25,9 +25,8 @@ const (
 	// failureKept is how long routewatch is not asked again after a query
 	// fails, so that a dead routewatch is not asked on every request.
 	failureKept = time.Minute
-	// queryTimeout is how long one query may take. routewatch answers only
+	// queryTimeout is how long one query may take.
-	// after a reverse DNS lookup it allows 2 seconds, so this is longer.
+	queryTimeout = 2 * time.Second
-	queryTimeout = 5 * time.Second
 	// maxAnswers is how many answers are kept. Past it, the one that came
 	// longest ago is dropped.
 	maxAnswers = 100000

internal/routewatch/routewatch_test.go

@@ -21,10 +21,7 @@ const (
 	routedAnswer = `{"status":"ok","data":{"results":[` +
 		`{"ip":"192.0.2.1","netblock":"192.0.2.0/24","asn":64496}]}}`
 	noRouteAnswer = `{"status":"ok","data":{"results":[{"ip":"192.0.2.1"}]}}`
-	// slowAnswer is longer than routewatch allows a reverse DNS lookup.
+	waitLimit     = 5 * time.Second
-	slowAnswer = 2500 * time.Millisecond
-	// waitLimit is longer than the Client allows a query.
-	waitLimit = 10 * time.Second
 )
 
 // clock is a time the tests move by hand.
@@ -52,7 +49,6 @@ func (c *clock) advance(by time.Duration) {
 // the process log.
 type fixture struct {
 	t       *testing.T
-	server  *httptest.Server
 	client  *routewatch.Client
 	clk     *clock
 	queries *atomic.Int32
@@ -91,16 +87,16 @@ func newFixture(t *testing.T, standIn http.HandlerFunc) *fixture {
 		log:     &syncBuffer{},
 	}
 
-	f.server = httptest.NewServer(http.HandlerFunc(
+	server := httptest.NewServer(http.HandlerFunc(
 		func(w http.ResponseWriter, r *http.Request) {
 			f.queries.Add(1)
 			f.path.Store(r.URL.Path)
 			standIn(w, r)
 		}))
-	t.Cleanup(f.server.Close)
+	t.Cleanup(server.Close)
 
 	f.client = routewatch.New(routewatch.Params{
-		URL: f.server.URL + "/", Now: f.clk.Now,
+		URL: server.URL + "/", Now: f.clk.Now,
 		ProcessLog: slog.New(slog.NewTextHandler(f.log, nil)),
 	})
 
@@ -141,26 +137,6 @@ func (f *fixture) waitFor(why routewatch.Missing) routewatch.Route {
 	}
 }
 
-// wantFailureLoggedWithoutTheClient waits for the process log to name a failed
-// query, and checks that it does not name the client.
-func (f *fixture) wantFailureLoggedWithoutTheClient() {
-	f.t.Helper()
-
-	// The failure is noted before it is logged.
-	deadline := time.Now().Add(waitLimit)
-	for !strings.Contains(f.log.String(), "asking routewatch failed") {
-		if time.Now().After(deadline) {
-			f.t.Fatalf("process log %q names no failure", f.log.String())
-		}
-
-		time.Sleep(time.Millisecond)
-	}
-
-	if strings.Contains(f.log.String(), clientAddr) {
-		f.t.Errorf("process log %q names the client", f.log.String())
-	}
-}
-
 func TestRouteGivesRoutewatchsAnswerAndAsksByAddress(t *testing.T) {
 	t.Parallel()
 
@@ -182,25 +158,6 @@ func TestRouteGivesRoutewatchsAnswerAndAsksByAddress(t *testing.T) {
 	}
 }
 
-func TestAnswerAfterASlowReverseDNSLookupIsKept(t *testing.T) {
-	t.Parallel()
-
-	f := newFixture(t, func(w http.ResponseWriter, _ *http.Request) {
-		time.Sleep(slowAnswer)
-
-		_, _ = w.Write([]byte(routedAnswer))
-	})
-
-	f.route()
-
-	route := f.waitFor("")
-
-	want := routewatch.Route{Netblock: netip.MustParsePrefix("192.0.2.0/24"), ASN: 64496}
-	if route != want {
-		t.Errorf("route %+v, want %+v", route, want)
-	}
-}
-
 func TestAddressWithoutARouteIsKeptAsNoRoute(t *testing.T) {
 	t.Parallel()
 
@@ -234,23 +191,24 @@ func TestAnswerRoutewatchCannotBeTakenFromFails(t *testing.T) {
 
 			f.route()
 			f.waitFor(routewatch.RequestFailed)
-			f.wantFailureLoggedWithoutTheClient()
+
+			// The failure is noted before it is logged.
+			deadline := time.Now().Add(waitLimit)
+			for !strings.Contains(f.log.String(), "asking routewatch failed") {
+				if time.Now().After(deadline) {
+					t.Fatalf("process log %q names no failure", f.log.String())
+				}
+
+				time.Sleep(time.Millisecond)
+			}
+
+			if strings.Contains(f.log.String(), clientAddr) {
+				t.Errorf("process log %q names the client", f.log.String())
+			}
 		})
 	}
 }
 
-func TestRefusedConnectionFailsTheQueryAndTheLogOmitsTheClient(t *testing.T) {
-	t.Parallel()
-
-	f := newFixture(t, answering(routedAnswer))
-
-	f.server.Close()
-
-	f.route()
-	f.waitFor(routewatch.RequestFailed)
-	f.wantFailureLoggedWithoutTheClient()
-}
-
 func TestFailureIsKeptForAMinuteSoRoutewatchIsNotAskedPerRequest(t *testing.T) {
 	t.Parallel()
 
@@ -295,7 +253,6 @@ func TestHangingRoutewatchNeverDelaysAndFailsTheQuery(t *testing.T) {
 	}
 
 	f.waitFor(routewatch.RequestFailed)
-	f.wantFailureLoggedWithoutTheClient()
 }
 
 func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {