Caches of outside APIs live in /var/cache/sww (closes #314)

The answers of GeoJS, the lists fetched by URL, the DNS blocklists,
AbuseIPDB and routewatch each go in a cache file in SWW_CACHE_DIR
(default /var/cache/sww): lookups.json, lists.json, dnsbl.json,
abuseipdb-scores.json and routewatch.json. reputation.json is gone. The
AbuseIPDB checks spent today are a count, not a cache, and stay in
SWW_STATE_DIR as abuseipdb-checks-spent.json.

A cache file never stops the start: one that does not parse is set
aside as an edit is, one that cannot be read is written over, and its
cache starts empty. An unwritable SWW_CACHE_DIR stops the start, as an
unwritable SWW_STATE_DIR does. Each directory is watched on its own.
The run script makes the cache directory and gives it to sww.

Model: opus-5-5

internal/routewatch/routewatch.go

@@ -1,6 +1,8 @@
 // Package routewatch asks a routewatch instance about client addresses, in the
-// background, and keeps its answers in memory. A routewatch instance knows, for
-// an address, the netblock it is announced in and the AS that originates it.
+// background, and keeps its answers. A routewatch instance knows, for an
+// address, the netblock it is announced in and the AS that originates it. The
+// answers are kept in memory, for at most 100,000 clients and for an hour each,
+// and are written to routewatch.json and read from it by the state package.
 package routewatch
 
 import (
@@ -13,6 +15,7 @@ import (
 	"net/http"
 	"net/netip"
 	"net/url"
+	"slices"
 	"sync"
 	"time"
 
@@ -65,6 +68,17 @@ type Route struct {
 	ASN      uint32
 }
 
+// Answer is what routewatch answered about a client, as routewatch.json holds
+// it: the netblock the client's address is announced in, none for an address
+// without a live route, the AS that originates it, and when routewatch
+// answered.
+type Answer struct {
+	Client   netip.Prefix `json:"client"`
+	Netblock netip.Prefix `json:"netblock"`
+	ASN      uint32       `json:"asn"`
+	Answered time.Time    `json:"answered"`
+}
+
 // Params are what New needs.
 type Params struct {
 	// URL is where routewatch is (SWW_ROUTEWATCH_URL).
@@ -84,23 +98,16 @@ type Client struct {
 	mu sync.Mutex
 	// answers are by client. Each is added as it comes and never moved up,
 	// so that the one that came longest ago is the first dropped.
-	answers *simplelru.LRU[netip.Prefix, answer]
+	answers *simplelru.LRU[netip.Prefix, Answer]
 	// asking are the clients whose query is under way.
 	asking map[netip.Prefix]bool
 	// retryAt is when routewatch may be asked again after a query failed.
 	retryAt time.Time
 }
 
-// answer is one kept answer: the route, or why there is none, and when it came.
-type answer struct {
-	route   Route
-	missing Missing
-	came    time.Time
-}
-
 // New returns a Client with no answer yet.
 func New(params Params) *Client {
-	answers, err := simplelru.NewLRU[netip.Prefix, answer](maxAnswers, nil)
+	answers, err := simplelru.NewLRU[netip.Prefix, Answer](maxAnswers, nil)
 	if err != nil {
 		panic(err) // NewLRU fails only for a size below one
 	}
@@ -128,8 +135,12 @@ func (c *Client) Route(
 	now := c.params.Now()
 
 	kept, found := c.answers.Peek(client)
-	if found && now.Sub(kept.came) < answerKept {
-		return kept.route, kept.missing
+	if found && now.Sub(kept.Answered) < answerKept {
+		if !kept.Netblock.IsValid() {
+			return Route{}, NoRoute
+		}
+
+		return Route{Netblock: kept.Netblock, ASN: kept.ASN}, ""
 	}
 
 	if now.Before(c.retryAt) {
@@ -145,12 +156,58 @@ func (c *Client) Route(
 	return Route{}, NoAnswerYet
 }
 
+// Snapshot returns every answer still in use, sorted by client, as
+// routewatch.json lists them.
+func (c *Client) Snapshot() []Answer {
+	c.mu.Lock()
+
+	now := c.params.Now()
+	answers := make([]Answer, 0, c.answers.Len())
+
+	for _, kept := range c.answers.Values() {
+		if now.Sub(kept.Answered) < answerKept {
+			answers = append(answers, kept)
+		}
+	}
+
+	c.mu.Unlock()
+
+	slices.SortFunc(answers, func(a, b Answer) int {
+		return a.Client.Compare(b.Client)
+	})
+
+	return answers
+}
+
+// Load keeps answers, read from routewatch.json, in place of those it keeps,
+// but for those that came answerKept ago or more and, past maxAnswers, those
+// that came longest ago.
+func (c *Client) Load(answers []Answer) {
+	answers = slices.Clone(answers)
+	slices.SortStableFunc(answers, func(a, b Answer) int {
+		return a.Answered.Compare(b.Answered)
+	})
+
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	c.answers.Purge()
+
+	now := c.params.Now()
+
+	for _, kept := range answers {
+		if now.Sub(kept.Answered) < answerKept {
+			c.answers.Add(kept.Client, kept)
+		}
+	}
+}
+
 // ask asks routewatch about addr, keeps the answer as client's, and notes the
 // query as no longer under way. A query that fails gives no answer, and
 // routewatch is not asked again for failureKept. Only the first failure of
 // those under way is logged.
 func (c *Client) ask(ctx context.Context, client netip.Prefix, addr netip.Addr) {
-	route, missing, err := c.query(ctx, addr)
+	route, err := c.query(ctx, addr)
 	now := c.params.Now()
 
 	c.mu.Lock()
@@ -160,7 +217,9 @@ func (c *Client) ask(ctx context.Context, client netip.Prefix, addr netip.Addr)
 	firstFailure := err != nil && !now.Before(c.retryAt)
 
 	if err == nil {
-		c.answers.Add(client, answer{route: route, missing: missing, came: now})
+		c.answers.Add(client, Answer{
+			Client: client, Netblock: route.Netblock, ASN: route.ASN, Answered: now,
+		})
 	} else {
 		c.retryAt = now.Add(failureKept)
 	}
@@ -174,19 +233,19 @@ func (c *Client) ask(ctx context.Context, client netip.Prefix, addr netip.Addr)
 
 // query asks routewatch's IP info API about addr. An answer other than 200, one
 // that cannot be read, and none within queryTimeout, fail. A result with no
-// netblock is an address without a live route.
-func (c *Client) query(ctx context.Context, addr netip.Addr) (Route, Missing, error) {
+// netblock is an address without a live route, and gives a Route without one.
+func (c *Client) query(ctx context.Context, addr netip.Addr) (Route, error) {
 	ctx, cancel := context.WithTimeout(ctx, queryTimeout)
 	defer cancel()
 
 	endpoint, err := url.JoinPath(c.params.URL, "api/v1/ip", addr.String())
 	if err != nil {
-		return Route{}, "", fmt.Errorf("make the URL: %w", err)
+		return Route{}, fmt.Errorf("make the URL: %w", err)
 	}
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, http.NoBody)
 	if err != nil {
-		return Route{}, "", fmt.Errorf("make the request: %w", err)
+		return Route{}, fmt.Errorf("make the request: %w", err)
 	}
 
 	req.Header.Set("Accept", "application/json")
@@ -195,7 +254,7 @@ func (c *Client) query(ctx context.Context, addr netip.Addr) (Route, Missing, er
 	if err != nil {
 		// Do's error names the URL, which holds the client's address, which
 		// is not to be logged: only what went wrong is kept.
-		return Route{}, "", fmt.Errorf("ask routewatch: %w", errors.Unwrap(err))
+		return Route{}, fmt.Errorf("ask routewatch: %w", errors.Unwrap(err))
 	}
 
 	defer func() {
@@ -203,14 +262,14 @@ func (c *Client) query(ctx context.Context, addr netip.Addr) (Route, Missing, er
 	}()
 
 	if res.StatusCode != http.StatusOK {
-		return Route{}, "", fmt.Errorf("%w %s", errStatus, res.Status)
+		return Route{}, fmt.Errorf("%w %s", errStatus, res.Status)
 	}
 
 	return readAnswer(io.LimitReader(res.Body, maxAnswerBytes))
 }
 
 // readAnswer reads routewatch's answer about one address.
-func readAnswer(body io.Reader) (Route, Missing, error) {
+func readAnswer(body io.Reader) (Route, error) {
 	var decoded struct {
 		Data struct {
 			Results []struct {
@@ -222,22 +281,22 @@ func readAnswer(body io.Reader) (Route, Missing, error) {
 
 	err := json.NewDecoder(body).Decode(&decoded)
 	if err != nil {
-		return Route{}, "", fmt.Errorf("read the answer: %w", err)
+		return Route{}, fmt.Errorf("read the answer: %w", err)
 	}
 
 	if len(decoded.Data.Results) == 0 {
-		return Route{}, "", errNoResult
+		return Route{}, errNoResult
 	}
 
 	result := decoded.Data.Results[0]
 	if result.Netblock == "" {
-		return Route{}, NoRoute, nil
+		return Route{}, nil
 	}
 
 	netblock, err := netip.ParsePrefix(result.Netblock)
 	if err != nil {
-		return Route{}, "", fmt.Errorf("%w: %q", errNotNetblock, result.Netblock)
+		return Route{}, fmt.Errorf("%w: %q", errNotNetblock, result.Netblock)
 	}
 
-	return Route{Netblock: netblock.Masked(), ASN: result.ASN}, "", nil
+	return Route{Netblock: netblock.Masked(), ASN: result.ASN}, nil
 }

internal/routewatch/routewatch_test.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/netip"
+	"slices"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -336,3 +337,96 @@ func TestRepeatedRequestsMakeOneQueryAndTheAnswerExpires(t *testing.T) {
 		t.Errorf("%d queries after the answer expired, want 2", got)
 	}
 }
+
+func TestSnapshotHoldsTheAnswerUntilItExpires(t *testing.T) {
+	t.Parallel()
+
+	f := newFixture(t, answering(routedAnswer))
+
+	f.route()
+	f.waitFor("")
+
+	want := []routewatch.Answer{{
+		Client:   netip.MustParsePrefix(clientAddr + "/32"),
+		Netblock: netip.MustParsePrefix("192.0.2.0/24"), ASN: 64496,
+		Answered: f.clk.Now(),
+	}}
+	if got := f.client.Snapshot(); !slices.Equal(got, want) {
+		t.Errorf("snapshot %+v, want %+v", got, want)
+	}
+
+	f.clk.advance(time.Hour)
+
+	if got := f.client.Snapshot(); len(got) != 0 {
+		t.Errorf("snapshot %+v an hour after the answer, want none", got)
+	}
+}
+
+func TestLoadedAnswersAreUsedForAnHourFromWhenTheyCame(t *testing.T) {
+	t.Parallel()
+
+	f := newFixture(t, answering(routedAnswer))
+	now := f.clk.Now()
+	kept := routewatch.Answer{
+		Client:   netip.MustParsePrefix(clientAddr + "/32"),
+		Netblock: netip.MustParsePrefix("198.51.100.0/24"), ASN: 64497,
+		Answered: now.Add(-time.Hour + time.Second),
+	}
+	stale := routewatch.Answer{
+		Client:   netip.MustParsePrefix("192.0.2.2/32"),
+		Netblock: netip.MustParsePrefix("192.0.2.0/24"), ASN: 64496,
+		Answered: now.Add(-time.Hour),
+	}
+
+	f.client.Load([]routewatch.Answer{kept, stale})
+
+	if got := f.client.Snapshot(); !slices.Equal(got, []routewatch.Answer{kept}) {
+		t.Errorf("kept %+v, want only the answer that came less than an hour ago", got)
+	}
+
+	route, missing := f.route()
+
+	want := routewatch.Route{Netblock: kept.Netblock, ASN: kept.ASN}
+	if route != want || missing != "" {
+		t.Errorf("route %+v (%q), want %+v", route, missing, want)
+	}
+
+	if got := f.queries.Load(); got != 0 {
+		t.Errorf("%d queries for a client with an answer loaded, want none", got)
+	}
+}
+
+func TestLoadDropsTheAnswerThatCameLongestAgoFirst(t *testing.T) {
+	t.Parallel()
+
+	const maxAnswers = 100000
+
+	f := newFixture(t, answering(routedAnswer))
+	now := f.clk.Now()
+
+	// routewatch.json lists the answers by client. Here each came a
+	// millisecond before the one listed before it, so the last listed is
+	// the one that came longest ago, and the one dropped.
+	answers := make([]routewatch.Answer, maxAnswers+1)
+	addr := netip.MustParseAddr("10.0.0.0")
+
+	for i := range answers {
+		answers[i] = routewatch.Answer{
+			Client:   netip.PrefixFrom(addr, addr.BitLen()),
+			Answered: now.Add(-time.Duration(i) * time.Millisecond),
+		}
+		addr = addr.Next()
+	}
+
+	f.client.Load(answers)
+
+	got := f.client.Snapshot()
+	if len(got) != maxAnswers {
+		t.Fatalf("%d answers kept, want %d", len(got), maxAnswers)
+	}
+
+	if got[0] != answers[0] || got[maxAnswers-1] != answers[maxAnswers-1] {
+		t.Errorf("answers kept from %s to %s, want from %s to %s", got[0].Client,
+			got[maxAnswers-1].Client, answers[0].Client, answers[maxAnswers-1].Client)
+	}
+}