sneak/routewatch
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Look up IP addresses by prefix at each mask length (closes #48)
check / check (push) Successful in 3m13s
Details

Browse Source 
Looking up an address on /ip/ and /api/v1/ip/ read every IPv6 live route and about half of the IPv4 range index, so on a day-sized database it passed the 30-second request timeout. The lookup is now one function for both families: from the longest mask length down, it looks up the address's network prefix on the existing prefix index, and the first live route wins.

The feed sends IPv6 withdrawals uncompressed, so they never matched the stored compressed prefix and never removed a route. Prefixes are now stored in the text form net/netip prints, so IPv6 withdrawals take effect.

ip_start, ip_end and GetASInfoForIP are removed; nothing read them. A database from before this change must be deleted.

Model: opus-5-5
This commit was merged in pull request #51.
This commit is contained in:
clawbot
2026-10-03 16:04:38 +02:00
parent 7322f936e4
commit 32704c601e
11 changed files with 247 additions and 705 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/database/database.go
+66 -331
View File
@@ -9,6 +9,7 @@ import (
"errors"
"fmt"
"net"
"net/netip"
"os"
"path/filepath"
"runtime"
@@ -32,10 +33,6 @@ const (
dirPermissions = 0750 // rwxr-x---
ipVersionV4 = 4
ipVersionV6 = 6
ipv6Length = 16
ipv4Offset = 12
ipv4Bits = 32
maxIPv4 = 0xFFFFFFFF
)
// SQLite memory tuning. cache_size and busy_timeout go in the DSN so every
@@ -225,10 +222,10 @@ func (d *Database) beginTx() (*loggingTx, error) {
// upsert did.
const (
updateLiveRouteV4SQL = `UPDATE live_routes_v4 SET mask_length = ?, as_path = ?, next_hop = ?,
last_updated = ?, ip_start = ?, ip_end = ? WHERE prefix = ? AND origin_asn = ? AND peer_ip = ?`
last_updated = ? WHERE prefix = ? AND origin_asn = ? AND peer_ip = ?`
insertLiveRouteV4SQL = `INSERT INTO live_routes_v4 (id, prefix, mask_length, origin_asn, peer_ip,
as_path, next_hop, last_updated, ip_start, ip_end) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
as_path, next_hop, last_updated) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
updateLiveRouteV6SQL = `UPDATE live_routes_v6 SET mask_length = ?, as_path = ?, next_hop = ?,
last_updated = ? WHERE prefix = ? AND origin_asn = ? AND peer_ip = ?`
@@ -240,12 +237,8 @@ const (
// upsertRouteRowV4 updates an IPv4 live route, inserting it when no row matched,
// and reports whether a new row was inserted.
func upsertRouteRowV4(upd, ins *sql.Stmt, route *LiveRoute, pathJSON string) (inserted bool, err error) {
if route.V4IPStart == nil || route.V4IPEnd == nil {
return false, fmt.Errorf("IPv4 route %s missing range values", route.Prefix)
}
res, err := upd.Exec(route.MaskLength, pathJSON, route.NextHop, route.LastUpdated,
*route.V4IPStart, *route.V4IPEnd, route.Prefix, route.OriginASN, route.PeerIP)
route.Prefix, route.OriginASN, route.PeerIP)
if err != nil {
return false, err
}
@@ -258,7 +251,7 @@ func upsertRouteRowV4(upd, ins *sql.Stmt, route *LiveRoute, pathJSON string) (in
}
_, err = ins.Exec(route.ID.String(), route.Prefix, route.MaskLength, route.OriginASN,
route.PeerIP, pathJSON, route.NextHop, route.LastUpdated, *route.V4IPStart, *route.V4IPEnd)
route.PeerIP, pathJSON, route.NextHop, route.LastUpdated)
if err != nil {
return false, err
}
@@ -1322,176 +1315,6 @@ func (d *Database) GetLiveRouteCountsContext(ctx context.Context) (ipv4Count, ip
return ipv4Count, ipv6Count, nil
}
// GetASInfoForIP returns AS information for the given IP address
func (d *Database) GetASInfoForIP(ip string) (*ASInfo, error) {
return d.GetASInfoForIPContext(context.Background(), ip)
}
// GetASInfoForIPContext returns AS information for the given IP address with context support
func (d *Database) GetASInfoForIPContext(ctx context.Context, ip string) (*ASInfo, error) {
// Parse the IP to validate it
parsedIP := net.ParseIP(ip)
if parsedIP == nil {
return nil, fmt.Errorf("%w: %s", ErrInvalidIP, ip)
}
// Determine IP version
ipVersion := ipVersionV4
ipv4 := parsedIP.To4()
if ipv4 == nil {
ipVersion = ipVersionV6
}
// For IPv4, use optimized range query
if ipVersion == ipVersionV4 {
// Convert IP to 32-bit unsigned integer
ipUint := ipToUint32(ipv4)
query := `
SELECT DISTINCT lr.prefix, lr.mask_length, lr.origin_asn, lr.last_updated, a.handle, a.description
FROM live_routes_v4 lr
LEFT JOIN asns a ON a.asn = lr.origin_asn
WHERE lr.ip_start <= ? AND lr.ip_end >= ?
ORDER BY lr.mask_length DESC
LIMIT 1
`
var prefix string
var maskLength, originASN int
var lastUpdated time.Time
var handle, description sql.NullString
err := d.db.QueryRowContext(ctx, query, ipUint, ipUint).Scan(
&prefix, &maskLength, &originASN, &lastUpdated, &handle, &description)
if err != nil {
if err == sql.ErrNoRows {
return nil, fmt.Errorf("%w for IP %s", ErrNoRoute, ip)
}
return nil, fmt.Errorf("failed to query routes: %w", err)
}
age := time.Since(lastUpdated).Round(time.Second).String()
return &ASInfo{
ASN: originASN,
Handle: handle.String,
Description: description.String,
Prefix: prefix,
LastUpdated: lastUpdated,
Age: age,
}, nil
}
// For IPv6, use the original method since we don't have range optimization
query := `
SELECT DISTINCT lr.prefix, lr.mask_length, lr.origin_asn, lr.last_updated, a.handle, a.description
FROM live_routes_v6 lr
LEFT JOIN asns a ON a.asn = lr.origin_asn
ORDER BY lr.mask_length DESC
`
rows, err := d.db.QueryContext(ctx, query)
if err != nil {
return nil, fmt.Errorf("failed to query routes: %w", err)
}
defer func() { _ = rows.Close() }()
// Find the most specific matching prefix
var bestMatch struct {
prefix string
maskLength int
originASN int
lastUpdated time.Time
handle sql.NullString
description sql.NullString
}
bestMaskLength := -1
for rows.Next() {
var prefix string
var maskLength, originASN int
var lastUpdated time.Time
var handle, description sql.NullString
if err := rows.Scan(&prefix, &maskLength, &originASN, &lastUpdated, &handle, &description); err != nil {
continue
}
// Parse the prefix CIDR
_, ipNet, err := net.ParseCIDR(prefix)
if err != nil {
continue
}
// Check if the IP is in this prefix
if ipNet.Contains(parsedIP) && maskLength > bestMaskLength {
bestMatch.prefix = prefix
bestMatch.maskLength = maskLength
bestMatch.originASN = originASN
bestMatch.lastUpdated = lastUpdated
bestMatch.handle = handle
bestMatch.description = description
bestMaskLength = maskLength
}
}
if bestMaskLength == -1 {
return nil, fmt.Errorf("%w for IP %s", ErrNoRoute, ip)
}
age := time.Since(bestMatch.lastUpdated).Round(time.Second).String()
return &ASInfo{
ASN: bestMatch.originASN,
Handle: bestMatch.handle.String,
Description: bestMatch.description.String,
Prefix: bestMatch.prefix,
LastUpdated: bestMatch.lastUpdated,
Age: age,
}, nil
}
// ipToUint32 converts an IPv4 address to a 32-bit unsigned integer
func ipToUint32(ip net.IP) uint32 {
if len(ip) == ipv6Length {
// Convert to 4-byte representation
ip = ip[ipv4Offset:ipv6Length]
}
return uint32(ip[0])<<24 | uint32(ip[1])<<16 | uint32(ip[2])<<8 | uint32(ip[3])
}
// CalculateIPv4Range calculates the start and end IP addresses for an IPv4 CIDR block
func CalculateIPv4Range(cidr string) (start, end uint32, err error) {
_, ipNet, err := net.ParseCIDR(cidr)
if err != nil {
return 0, 0, err
}
// Get the network address (start of range)
ip := ipNet.IP.To4()
if ip == nil {
return 0, 0, fmt.Errorf("not an IPv4 address")
}
start = ipToUint32(ip)
// Calculate the end of the range
ones, bits := ipNet.Mask.Size()
hostBits := bits - ones
if hostBits >= ipv4Bits {
// Special case for /0 - entire IPv4 space
end = maxIPv4
} else {
// Safe to convert since we checked hostBits < 32
//nolint:gosec // hostBits is guaranteed to be < 32 from the check above
end = start | ((1 << uint(hostBits)) - 1)
}
return start, end, nil
}
// GetASDetails returns detailed information about an AS including prefixes
func (d *Database) GetASDetails(asn int) (*ASN, []LiveRoute, error) {
return d.GetASDetailsContext(context.Background(), asn)
@@ -1913,39 +1736,37 @@ func (d *Database) GetIPInfo(ip string) (*IPInfo, error) {
}
// GetIPInfoContext returns comprehensive IP information with context support.
//
// It finds the most specific live route for the address with point lookups on
// the prefix index: for each mask length from the longest (32 or 128) down to 0
// it looks up the address's network prefix at that length, and the first length
// with a live route wins. That is at most 33 or 129 indexed lookups, however
// many routes are live. The prefix is looked up in the text form net/netip
// prints, which is the form the prefix handler stores.
func (d *Database) GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, error) {
// Parse the IP to validate it
parsedIP := net.ParseIP(ip)
if parsedIP == nil {
addr, err := netip.ParseAddr(ip)
if err != nil {
return nil, fmt.Errorf("%w: %s", ErrInvalidIP, ip)
}
// An IPv4 address written in IPv6 form (::ffff:192.0.2.1) is an IPv4 lookup.
addr = addr.Unmap()
// Determine IP version
ipv4 := parsedIP.To4()
if ipv4 != nil {
return d.getIPv4Info(ctx, ip, ipv4)
info := &IPInfo{IP: ip, IPVersion: ipVersionV6}
routesTable, prefixesTable := "live_routes_v6", "prefixes_v6"
if addr.Is4() {
info.IPVersion = ipVersionV4
routesTable, prefixesTable = "live_routes_v4", "prefixes_v4"
}
return d.getIPv6Info(ctx, ip, parsedIP)
}
// getIPv4Info returns comprehensive IP information for an IPv4 address.
func (d *Database) getIPv4Info(ctx context.Context, ip string, ipv4 net.IP) (*IPInfo, error) {
info := &IPInfo{
IP: ip,
IPVersion: ipVersionV4,
}
ipUint := ipToUint32(ipv4)
// Get route info with peer count and prefix first_seen
query := `
// Route info with peer count and prefix first_seen
//nolint:gosec // Table names are hardcoded based on IP version
query := fmt.Sprintf(`
SELECT
lr.prefix,
lr.mask_length,
lr.origin_asn,
lr.last_updated,
(SELECT COUNT(DISTINCT peer_ip) FROM live_routes_v4 WHERE prefix = lr.prefix) as num_peers,
(SELECT COUNT(DISTINCT peer_ip) FROM %[1]s WHERE prefix = lr.prefix) as num_peers,
p.first_seen,
a.handle,
a.description,
@@ -1957,42 +1778,56 @@ func (d *Database) getIPv4Info(ctx context.Context, ip string, ipv4 net.IP) (*IP
a.abuse_email,
a.rir,
a.whois_updated_at
FROM live_routes_v4 lr
LEFT JOIN prefixes_v4 p ON p.prefix = lr.prefix
FROM %[1]s lr
LEFT JOIN %[2]s p ON p.prefix = lr.prefix
LEFT JOIN asns a ON a.asn = lr.origin_asn
WHERE lr.ip_start <= ? AND lr.ip_end >= ?
ORDER BY lr.mask_length DESC
WHERE lr.prefix = ?
LIMIT 1
`
`, routesTable, prefixesTable)
stmt, err := d.db.PrepareContext(ctx, query)
if err != nil {
return nil, fmt.Errorf("failed to prepare route query: %w", err)
}
defer func() { _ = stmt.Close() }()
var handle, description, asName, orgName, orgID, address, countryCode, abuseEmail, rir sql.NullString
var prefixFirstSeen sql.NullTime
var whoisUpdatedAt sql.NullTime
err := d.db.QueryRowContext(ctx, query, ipUint, ipUint).Scan(
&info.Netblock,
&info.MaskLength,
&info.ASN,
&info.LastSeen,
&info.NumPeers,
&prefixFirstSeen,
&handle,
&description,
&asName,
&orgName,
&orgID,
&address,
&countryCode,
&abuseEmail,
&rir,
&whoisUpdatedAt,
)
if err != nil {
if err == sql.ErrNoRows {
return nil, fmt.Errorf("%w for IP %s", ErrNoRoute, ip)
found := false
for bits := addr.BitLen(); bits >= 0; bits-- {
prefix := netip.PrefixFrom(addr, bits).Masked().String()
err := stmt.QueryRowContext(ctx, prefix).Scan(
&info.Netblock,
&info.MaskLength,
&info.ASN,
&info.LastSeen,
&info.NumPeers,
&prefixFirstSeen,
&handle,
&description,
&asName,
&orgName,
&orgID,
&address,
&countryCode,
&abuseEmail,
&rir,
&whoisUpdatedAt,
)
if errors.Is(err, sql.ErrNoRows) {
continue
}
if err != nil {
return nil, fmt.Errorf("failed to query routes: %w", err)
}
found = true
return nil, fmt.Errorf("failed to query routes: %w", err)
break
}
if !found {
return nil, fmt.Errorf("%w for IP %s", ErrNoRoute, ip)
}
info.Handle = handle.String
@@ -2016,106 +1851,6 @@ func (d *Database) getIPv4Info(ctx context.Context, ip string, ipv4 net.IP) (*IP
return info, nil
}
// getIPv6Info returns comprehensive IP information for an IPv6 address.
func (d *Database) getIPv6Info(ctx context.Context, ip string, parsedIP net.IP) (*IPInfo, error) {
info := &IPInfo{
IP: ip,
IPVersion: ipVersionV6,
}
// For IPv6, scan all routes and find best match
query := `
SELECT DISTINCT
lr.prefix,
lr.mask_length,
lr.origin_asn,
lr.last_updated,
a.handle,
a.description,
a.as_name,
a.org_name,
a.org_id,
a.address,
a.country_code,
a.abuse_email,
a.rir,
a.whois_updated_at
FROM live_routes_v6 lr
LEFT JOIN asns a ON a.asn = lr.origin_asn
ORDER BY lr.mask_length DESC
`
rows, err := d.db.QueryContext(ctx, query)
if err != nil {
return nil, fmt.Errorf("failed to query routes: %w", err)
}
defer func() { _ = rows.Close() }()
bestMaskLength := -1
for rows.Next() {
var prefix string
var maskLength, originASN int
var lastUpdated time.Time
var handle, description, asName, orgName, orgID, address, countryCode, abuseEmail, rir sql.NullString
var whoisUpdatedAt sql.NullTime
if err := rows.Scan(
&prefix, &maskLength, &originASN, &lastUpdated,
&handle, &description, &asName, &orgName, &orgID,
&address, &countryCode, &abuseEmail, &rir, &whoisUpdatedAt,
); err != nil {
continue
}
_, ipNet, err := net.ParseCIDR(prefix)
if err != nil {
continue
}
if ipNet.Contains(parsedIP) && maskLength > bestMaskLength {
info.Netblock = prefix
info.MaskLength = maskLength
info.ASN = originASN
info.LastSeen = lastUpdated
info.Handle = handle.String
info.Description = description.String
info.ASName = asName.String
info.OrgName = orgName.String
info.OrgID = orgID.String
info.Address = address.String
info.CountryCode = countryCode.String
info.AbuseEmail = abuseEmail.String
info.RIR = rir.String
bestMaskLength = maskLength
if !whoisUpdatedAt.Valid {
info.NeedsWHOISRefresh = true
} else {
const staleThreshold = 30 * 24 * time.Hour
info.NeedsWHOISRefresh = time.Since(whoisUpdatedAt.Time) > staleThreshold
}
}
}
if bestMaskLength == -1 {
return nil, fmt.Errorf("%w for IP %s", ErrNoRoute, ip)
}
// Get peer count and first_seen for IPv6
countQuery := `SELECT COUNT(DISTINCT peer_ip) FROM live_routes_v6 WHERE prefix = ?`
_ = d.db.QueryRowContext(ctx, countQuery, info.Netblock).Scan(&info.NumPeers)
firstSeenQuery := `SELECT first_seen FROM prefixes_v6 WHERE prefix = ?`
var prefixFirstSeen sql.NullTime
err = d.db.QueryRowContext(ctx, firstSeenQuery, info.Netblock).Scan(&prefixFirstSeen)
if err == nil && prefixFirstSeen.Valid {
info.FirstSeen = prefixFirstSeen.Time
}
return info, nil
}
// Vacuum runs incremental vacuum to reclaim unused pages without blocking writes.
// It frees up to the specified number of pages per call (0 = all freeable pages).
func (d *Database) Vacuum(ctx context.Context) error {