Read every route from the routing table in memory (part of #122)

The IP lookups, the AS and prefix pages and their API answers, the prefix-length pages and the stats read routes only from the routing table the prefix handler keeps. fx reads the table at startup and gives it to both. AS names, WHOIS details and first-seen times still come from the database.

The IP lookup names the origin AS most peers carry, with the time of the newest kept route. The prefix answers list peers whose routes are not kept without a path or time. The stats gain origins_without_kept_route. They no longer read route times from the per-peer tables, so the two helpers that did are gone.

Model: opus-5-5

internal/database/counts_internal_test.go

@@ -657,60 +657,6 @@ func checkOneLiveRoutePerIPVersion(t *testing.T, db *Database) {
 	assertDistribution(t, "IPv6 distribution query", queryV6, wantV6)
 }
 
-// TestStatsRouteTimestamps checks the oldest/newest route timestamps are read
-// from the right rows across both tables and parse into time.Time. The old
-// MIN/MAX union query read its result into *time.Time, which the driver could
-// not parse, so it logged a warning every call and left both timestamps nil.
-func TestStatsRouteTimestamps(t *testing.T) {
-	t.Parallel()
-
-	cfg := &config.Config{StateDir: t.TempDir()}
-
-	db, err := New(cfg, logger.New())
-	if err != nil {
-		t.Fatalf("failed to create database: %v", err)
-	}
-	defer func() { _ = db.Close() }()
-
-	ctx := context.Background()
-
-	empty, err := db.GetStatsContext(ctx)
-	if err != nil {
-		t.Fatalf("GetStatsContext on empty database: %v", err)
-	}
-
-	if empty.OldestRoute != nil || empty.NewestRoute != nil {
-		t.Fatalf("empty database timestamps = (%v, %v), want (nil, nil)",
-			empty.OldestRoute, empty.NewestRoute)
-	}
-
-	base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
-	oldest := base
-	newest := base.Add(2 * time.Minute)
-
-	err = db.UpsertLiveRouteBatch([]*LiveRoute{
-		mkV4Route(t, "198.51.100.0/24", 64500, base.Add(time.Minute)),
-		mkV4Route(t, "203.0.113.0/24", 64501, oldest),
-		mkV6Route(testV6Prefix, 64502, newest),
-	})
-	if err != nil {
-		t.Fatalf("UpsertLiveRouteBatch: %v", err)
-	}
-
-	stats, err := db.GetStatsContext(ctx)
-	if err != nil {
-		t.Fatalf("GetStatsContext: %v", err)
-	}
-
-	if stats.OldestRoute == nil || !stats.OldestRoute.Equal(oldest) {
-		t.Errorf("OldestRoute = %v, want %v", stats.OldestRoute, oldest)
-	}
-
-	if stats.NewestRoute == nil || !stats.NewestRoute.Equal(newest) {
-		t.Errorf("NewestRoute = %v, want %v", stats.NewestRoute, newest)
-	}
-}
-
 // TestLiveCountsConcurrentReadWrite runs writers and stats readers at once so
 // the race detector proves the counters are safe under concurrent use.
 func TestLiveCountsConcurrentReadWrite(t *testing.T) {

internal/database/database.go

@@ -1452,10 +1452,9 @@ func (d *Database) GetStats() (Stats, error) {
 //
 // The row counts (ASNs, prefixes, peerings, peers, live routes) and the prefix
 // distribution come from the in-memory counters, seeded at startup and kept
-// current on every write. The oldest/newest route timestamps are read from the
-// ends of the last_updated index, and the file size from a stat(). No part of
-// the read scans a table or a whole index.
-func (d *Database) GetStatsContext(ctx context.Context) (Stats, error) {
+// current on every write, and the file size from a stat(). The read touches
+// no table.
+func (d *Database) GetStatsContext(_ context.Context) (Stats, error) {
 	var stats Stats
 
 	// Row counts and prefix distribution from memory, as a single consistent
@@ -1470,20 +1469,6 @@ func (d *Database) GetStatsContext(ctx context.Context) (Stats, error) {
 		stats.FileSizeBytes = fileInfo.Size()
 	}
 
-	// Oldest and newest route timestamps read one row from each end of the
-	// last_updated index (a log-time lookup, not a scan). Selecting the column
-	// directly lets the driver parse the DATETIME into time.Time; the old
-	// MIN/MAX union scan returned an untyped string that failed to scan and
-	// logged a warning on every call.
-	oldest, newest, err := d.routeTimestampRange(ctx)
-	if err != nil {
-		// Display-only fields; log but keep the rest of the stats.
-		d.logger.Warn("Failed to get route timestamps", "error", err)
-	} else {
-		stats.OldestRoute = oldest
-		stats.NewestRoute = newest
-	}
-
 	return stats, nil
 }
 
@@ -1778,7 +1763,35 @@ func (d *Database) GetASDetailsContext(
 		return nil, nil, fmt.Errorf("%w: AS%d", ErrNoRoute, asn)
 	}
 
-	// Get AS information
+	asnInfo, known, err := d.GetAS(ctx, asn)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	allPrefixes, err := d.originPrefixes(ctx, asn, ipVersionV4)
+	if err != nil {
+		return asnInfo, nil, err
+	}
+
+	prefixesV6, err := d.originPrefixes(ctx, asn, ipVersionV6)
+	if err != nil {
+		return asnInfo, allPrefixes, err
+	}
+
+	allPrefixes = append(allPrefixes, prefixesV6...)
+
+	if !known && len(allPrefixes) == 0 {
+		return nil, nil, fmt.Errorf("%w: AS%d", ErrNoRoute, asn)
+	}
+
+	return asnInfo, allPrefixes, nil
+}
+
+// GetAS returns the AS's handle, description and first and last seen times
+// from the asns table. When the table has no record of the AS, its handle and
+// description come from the AS info lookup, and it has no seen times. The bool
+// says whether the asns table or the AS info lookup holds the AS.
+func (d *Database) GetAS(ctx context.Context, asn int) (*ASN, bool, error) {
 	var (
 		asnInfo             ASN
 		handle, description sql.NullString
@@ -1791,9 +1804,6 @@ func (d *Database) GetASDetailsContext(
 	).Scan(&asnInfo.ASN, &handle, &description, &asnInfo.FirstSeen,
 		&asnInfo.LastSeen)
 
-	// known is whether the asns table or the AS info lookup holds the AS.
-	known := true
-
 	switch {
 	case errors.Is(err, sql.ErrNoRows):
 		asnInfo.ASN = asn
@@ -1804,31 +1814,96 @@ func (d *Database) GetASDetailsContext(
 			asnInfo.Description = info.Description
 		}
 
-		known = ok
+		return &asnInfo, ok, nil
 	case err != nil:
-		return nil, nil, fmt.Errorf("failed to query AS: %w", err)
-	default:
-		asnInfo.Handle = handle.String
-		asnInfo.Description = description.String
+		return nil, false, fmt.Errorf("failed to query AS: %w", err)
+	}
+
+	asnInfo.Handle = handle.String
+	asnInfo.Description = description.String
+
+	return &asnInfo, true, nil
+}
+
+// asWHOISQuery reads the handle, description and WHOIS details of one AS, and
+// when the WHOIS details were fetched.
+const asWHOISQuery = `
+	SELECT handle, description, as_name, org_name, org_id, address,
+		country_code, abuse_email, rir, whois_updated_at
+	FROM asns WHERE asn = ?`
+
+// GetASWHOIS returns the handle, description and WHOIS details that the asns
+// table holds for the AS, and when the WHOIS details were fetched. When the
+// table has no record of the AS, it returns an ASN with only its number.
+func (d *Database) GetASWHOIS(ctx context.Context, asn int) (*ASN, error) {
+	var (
+		handle, description, asName, orgName, orgID sql.NullString
+		address, countryCode, abuseEmail, rir       sql.NullString
+		whoisUpdatedAt                              sql.NullTime
+	)
+
+	err := d.db.QueryRowContext(ctx, asWHOISQuery, asn).Scan(
+		&handle, &description, &asName, &orgName, &orgID, &address,
+		&countryCode, &abuseEmail, &rir, &whoisUpdatedAt,
+	)
+	if errors.Is(err, sql.ErrNoRows) {
+		return &ASN{ASN: asn}, nil
 	}
 
-	allPrefixes, err := d.originPrefixes(ctx, asn, ipVersionV4)
 	if err != nil {
-		return &asnInfo, nil, err
+		return nil, fmt.Errorf("failed to query AS: %w", err)
+	}
+
+	info := &ASN{
+		ASN:         asn,
+		Handle:      handle.String,
+		Description: description.String,
+		ASName:      asName.String,
+		OrgName:     orgName.String,
+		OrgID:       orgID.String,
+		Address:     address.String,
+		CountryCode: countryCode.String,
+		AbuseEmail:  abuseEmail.String,
+		RIR:         rir.String,
+	}
+
+	if whoisUpdatedAt.Valid {
+		info.WHOISUpdatedAt = &whoisUpdatedAt.Time
+	}
+
+	return info, nil
+}
+
+// The queries GetPrefixFirstSeen reads an IPv4 and an IPv6 prefix's first-seen
+// time with.
+const (
+	prefixFirstSeenV4SQL = `SELECT first_seen FROM prefixes_v4 WHERE prefix = ?`
+	prefixFirstSeenV6SQL = `SELECT first_seen FROM prefixes_v6 WHERE prefix = ?`
+)
+
+// GetPrefixFirstSeen returns when the prefix tables first saw prefix, or the
+// zero time when they have no record of it.
+func (d *Database) GetPrefixFirstSeen(
+	ctx context.Context, prefix netip.Prefix,
+) (time.Time, error) {
+	query := prefixFirstSeenV6SQL
+	if prefix.Addr().Is4() {
+		query = prefixFirstSeenV4SQL
+	}
+
+	var firstSeen time.Time
+
+	err := d.db.QueryRowContext(ctx, query, prefix.Masked().String()).
+		Scan(&firstSeen)
+	if errors.Is(err, sql.ErrNoRows) {
+		return time.Time{}, nil
 	}
 
-	prefixesV6, err := d.originPrefixes(ctx, asn, ipVersionV6)
 	if err != nil {
-		return &asnInfo, allPrefixes, err
+		return time.Time{}, fmt.Errorf("failed to query prefix: %w", err)
 	}
 
-	allPrefixes = append(allPrefixes, prefixesV6...)
-
-	if !known && len(allPrefixes) == 0 {
-		return nil, nil, fmt.Errorf("%w: AS%d", ErrNoRoute, asn)
-	}
-
-	return &asnInfo, allPrefixes, nil
+	return firstSeen, nil
 }
 
 // ASPeer represents a peering relationship with another AS including handle,
@@ -2564,76 +2639,6 @@ func (d *Database) deletePeerRouteChunk(peerIP string) (int, int, error) {
 	return int(deletedV4), int(deletedV6), nil
 }
 
-// routeTimestampRange returns the earliest and latest last_updated across both
-// live route tables, or nil values when both tables are empty. Each query reads
-// one row from an end of the last_updated index rather than scanning the tables.
-func (d *Database) routeTimestampRange(
-	ctx context.Context,
-) (*time.Time, *time.Time, error) {
-	var oldest, newest *time.Time
-
-	oldestV4, ok, err := d.scanRouteTimestamp(ctx, "SELECT last_updated "+
-		"FROM live_routes_v4 ORDER BY last_updated ASC LIMIT 1")
-	if err != nil {
-		return nil, nil, err
-	}
-
-	if ok {
-		oldest = &oldestV4
-	}
-
-	oldestV6, ok, err := d.scanRouteTimestamp(ctx, "SELECT last_updated "+
-		"FROM live_routes_v6 ORDER BY last_updated ASC LIMIT 1")
-	if err != nil {
-		return nil, nil, err
-	}
-
-	if ok && (oldest == nil || oldestV6.Before(*oldest)) {
-		oldest = &oldestV6
-	}
-
-	newestV4, ok, err := d.scanRouteTimestamp(ctx, "SELECT last_updated "+
-		"FROM live_routes_v4 ORDER BY last_updated DESC LIMIT 1")
-	if err != nil {
-		return nil, nil, err
-	}
-
-	if ok {
-		newest = &newestV4
-	}
-
-	newestV6, ok, err := d.scanRouteTimestamp(ctx, "SELECT last_updated "+
-		"FROM live_routes_v6 ORDER BY last_updated DESC LIMIT 1")
-	if err != nil {
-		return nil, nil, err
-	}
-
-	if ok && (newest == nil || newestV6.After(*newest)) {
-		newest = &newestV6
-	}
-
-	return oldest, newest, nil
-}
-
-// scanRouteTimestamp runs a single-row timestamp query. The bool is false when
-// the table is empty. The query selects the last_updated column directly so the
-// driver parses the DATETIME value into a time.Time.
-func (d *Database) scanRouteTimestamp(
-	ctx context.Context, query string,
-) (time.Time, bool, error) {
-	var ts time.Time
-
-	err := d.db.QueryRowContext(ctx, query).Scan(&ts)
-	switch {
-	case errors.Is(err, sql.ErrNoRows):
-		return time.Time{}, false, nil
-	case err != nil:
-		return time.Time{}, false, err
-	default:
-		return ts, true, nil
-	}
-}
-
 // The AS page lists the prefixes an AS originates one at a time, each in two
 // index searches that read no table row: one of the origin_asn and prefix
 // index for the AS's first prefix after the previous one, and one of the

internal/database/database_internal_test.go

@@ -1780,6 +1780,165 @@ func TestGetASDetailsOfASNotInDatabase(t *testing.T) {
 	}
 }
 
+// TestGetASReadsTheRecordOrTheBuiltInList asks GetAS for an AS the asns table
+// holds, one only the AS info lookup holds, and one neither holds. The first
+// gets its record's seen times, the second its handle and description from the
+// lookup, and only the third is not known.
+func TestGetASReadsTheRecordOrTheBuiltInList(t *testing.T) {
+	t.Parallel()
+
+	db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+	if err != nil {
+		t.Fatalf("failed to create database: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+
+	const (
+		storedASN  = 64500      // a documentation AS, not in the lookup
+		listedASN  = 15169      // in the lookup, not stored
+		unknownASN = 4200000001 // in neither
+	)
+
+	ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+
+	_, err = db.GetOrCreateASN(storedASN, ts)
+	if err != nil {
+		t.Fatalf("GetOrCreateASN: %v", err)
+	}
+
+	listed, ok := asinfo.Get(listedASN)
+	if !ok {
+		t.Fatalf("AS%d is not in the AS info data", listedASN)
+	}
+
+	tests := []struct {
+		asn       int
+		want      ASN
+		wantKnown bool
+	}{
+		{storedASN, ASN{ASN: storedASN, FirstSeen: ts, LastSeen: ts}, true},
+		{listedASN, ASN{
+			ASN: listedASN, Handle: listed.Handle, Description: listed.Description,
+		}, true},
+		{unknownASN, ASN{ASN: unknownASN}, false},
+	}
+
+	for _, tt := range tests {
+		got, known, err := db.GetAS(t.Context(), tt.asn)
+		if err != nil {
+			t.Errorf("GetAS(%d): %v", tt.asn, err)
+
+			continue
+		}
+
+		if !got.FirstSeen.Equal(tt.want.FirstSeen) ||
+			!got.LastSeen.Equal(tt.want.LastSeen) {
+			t.Errorf("GetAS(%d) seen %v to %v, want %v to %v", tt.asn,
+				got.FirstSeen, got.LastSeen, tt.want.FirstSeen, tt.want.LastSeen)
+		}
+
+		got.FirstSeen, got.LastSeen = tt.want.FirstSeen, tt.want.LastSeen
+		if *got != tt.want || known != tt.wantKnown {
+			t.Errorf("GetAS(%d) = %+v, known %t; want %+v, known %t",
+				tt.asn, *got, known, tt.want, tt.wantKnown)
+		}
+	}
+}
+
+// TestGetASWHOISReadsTheStoredDetails stores an AS with WHOIS details and asks
+// GetASWHOIS for it and for an AS the asns table does not hold. The first gets
+// its stored handle, description and details, and the time they were fetched.
+// The second gets only its number, although the AS info lookup holds it.
+func TestGetASWHOISReadsTheStoredDetails(t *testing.T) {
+	t.Parallel()
+
+	db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+	if err != nil {
+		t.Fatalf("failed to create database: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+
+	const storedASN, unstoredASN = 15169, 13335
+
+	_, err = db.GetOrCreateASN(storedASN, time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC))
+	if err != nil {
+		t.Fatalf("GetOrCreateASN: %v", err)
+	}
+
+	update := &ASNWHOISUpdate{
+		ASN: storedASN, ASName: "TEST-AS", OrgName: "Test Org", OrgID: "ORG-1",
+		Address: "1 Test Street", CountryCode: "DE", AbuseEmail: "abuse@test",
+		RIR: "RIPE",
+	}
+
+	err = db.UpdateASNWHOIS(t.Context(), update)
+	if err != nil {
+		t.Fatalf("UpdateASNWHOIS: %v", err)
+	}
+
+	got, err := db.GetASWHOIS(t.Context(), storedASN)
+	if err != nil {
+		t.Fatalf("GetASWHOIS(%d): %v", storedASN, err)
+	}
+
+	want := ASN{
+		ASN: storedASN, Handle: asinfo.GetHandle(storedASN),
+		Description: asinfo.GetDescription(storedASN), ASName: update.ASName,
+		OrgName: update.OrgName, OrgID: update.OrgID, Address: update.Address,
+		CountryCode: update.CountryCode, AbuseEmail: update.AbuseEmail,
+		RIR: update.RIR,
+	}
+
+	if got.WHOISUpdatedAt == nil {
+		t.Errorf("GetASWHOIS(%d) has no time the details were fetched", storedASN)
+	}
+
+	got.WHOISUpdatedAt = nil
+	if *got != want {
+		t.Errorf("GetASWHOIS(%d) = %+v, want %+v", storedASN, *got, want)
+	}
+
+	got, err = db.GetASWHOIS(t.Context(), unstoredASN)
+	if err != nil || *got != (ASN{ASN: unstoredASN}) {
+		t.Errorf("GetASWHOIS(%d) = %+v, %v; want only the number",
+			unstoredASN, got, err)
+	}
+}
+
+// TestGetPrefixFirstSeenReadsThePrefixTables stores an IPv4 and an IPv6
+// prefix and expects GetPrefixFirstSeen to give each its first-seen time, and
+// the zero time for a prefix that is not stored.
+func TestGetPrefixFirstSeenReadsThePrefixTables(t *testing.T) {
+	t.Parallel()
+
+	db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
+	if err != nil {
+		t.Fatalf("failed to create database: %v", err)
+	}
+	defer func() { _ = db.Close() }()
+
+	ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+	stored := map[string]time.Time{
+		"192.0.2.0/24": ts,
+		testV6Prefix:   ts.Add(time.Hour),
+	}
+
+	err = db.UpdatePrefixesBatch(stored)
+	if err != nil {
+		t.Fatalf("UpdatePrefixesBatch: %v", err)
+	}
+
+	stored["198.51.100.0/24"] = time.Time{}
+
+	for prefix, want := range stored {
+		got, err := db.GetPrefixFirstSeen(t.Context(), netip.MustParsePrefix(prefix))
+		if err != nil || !got.Equal(want) {
+			t.Errorf("GetPrefixFirstSeen(%s) = %v, %v; want %v",
+				prefix, got, err, want)
+		}
+	}
+}
+
 // TestASPrefixQueriesUseCoveringIndexes checks the plan of each query that
 // lists the prefixes an AS originates: a search of a covering index and no
 // temporary B-tree, so the list reads one index entry per prefix, not one

internal/database/interface.go

@@ -6,6 +6,7 @@ package database
 
 import (
 	"context"
+	"net/netip"
 	"time"
 
 	"git.eeqj.de/sneak/routewatch/internal/routetable"
@@ -24,8 +25,6 @@ type Stats struct {
 	LiveRoutes             int
 	IPv4Routes             int
 	IPv6Routes             int
-	OldestRoute            *time.Time
-	NewestRoute            *time.Time
 	IPv4PrefixDistribution []PrefixDistribution
 	IPv6PrefixDistribution []PrefixDistribution
 }
@@ -85,6 +84,9 @@ type Store interface {
 	GetWHOISStats(ctx context.Context, staleThreshold time.Duration) (*WHOISStats, error)
 
 	// AS and prefix detail operations
+	GetAS(ctx context.Context, asn int) (*ASN, bool, error)
+	GetASWHOIS(ctx context.Context, asn int) (*ASN, error)
+	GetPrefixFirstSeen(ctx context.Context, prefix netip.Prefix) (time.Time, error)
 	GetASDetails(asn int) (*ASN, []LiveRoute, error)
 	GetASDetailsContext(ctx context.Context, asn int) (*ASN, []LiveRoute, error)
 	GetASPeers(asn int) ([]ASPeer, error)

internal/database/models.go

@@ -66,7 +66,9 @@ type ASNPeering struct {
 	LastSeen  time.Time
 }
 
-// LiveRoute represents a route in the live routing table
+// LiveRoute represents a route in the live routing table. JSON leaves out an
+// empty AS path, next hop or time: the routing table in memory keeps them only
+// for the two newest routes of each prefix and origin AS.
 //
 //nolint:tagliatelle // field names are the existing JSON API
 type LiveRoute struct {
@@ -75,9 +77,9 @@ type LiveRoute struct {
 	IPVersion   int       `json:"ip_version"`
 	OriginASN   int       `json:"origin_asn"`
 	PeerIP      string    `json:"peer_ip"`
-	ASPath      []int     `json:"as_path"`
-	NextHop     string    `json:"next_hop"`
-	LastUpdated time.Time `json:"last_updated"`
+	ASPath      []int     `json:"as_path,omitempty"`
+	NextHop     string    `json:"next_hop,omitempty"`
+	LastUpdated time.Time `json:"last_updated,omitzero"`
 }
 
 // PrefixDistribution represents the distribution of prefixes by mask length
@@ -108,9 +110,10 @@ type IPInfo struct {
 	CountryCode string `json:"country_code,omitempty"`
 	AbuseEmail  string `json:"abuse_email,omitempty"`
 	RIR         string `json:"rir,omitempty"`
-	// Timestamps
+	// Timestamps. LastSeen is zero, and left out of JSON, when the prefix
+	// keeps no route.
 	FirstSeen time.Time `json:"first_seen"`
-	LastSeen  time.Time `json:"last_seen"`
+	LastSeen  time.Time `json:"last_seen,omitzero"`
 	// Indicates if WHOIS data needs refresh (not serialized)
 	NeedsWHOISRefresh bool `json:"-"`
 }