Add IP information API with background WHOIS fetcher
- Add /ip and /ip/{addr} JSON endpoints returning comprehensive IP info
- Include ASN, netblock, country code, org name, abuse contact, RIR data
- Extend ASN schema with WHOIS fields (country, org, abuse contact, etc)
- Create background WHOIS fetcher for rate-limited ASN info updates
- Store raw WHOIS responses for debugging and data preservation
- Queue on-demand WHOIS lookups when stale data is requested
- Refactor handleIPInfo to serve all IP endpoints consistently
This commit is contained in:
@@ -44,6 +44,8 @@ var (
|
||||
ErrInvalidIP = errors.New("invalid IP address")
|
||||
// ErrNoRoute is returned when no route is found for an IP
|
||||
ErrNoRoute = errors.New("no route found")
|
||||
// ErrNoStaleASN is returned when no ASN needs WHOIS refresh
|
||||
ErrNoStaleASN = errors.New("no stale ASN found")
|
||||
)
|
||||
|
||||
// Database manages the SQLite database connection and operations.
|
||||
@@ -1630,3 +1632,288 @@ func (d *Database) GetRandomPrefixesByLengthContext(
|
||||
|
||||
return routes, nil
|
||||
}
|
||||
|
||||
// GetNextStaleASN returns an ASN that needs WHOIS data refresh.
|
||||
// Priority: ASNs with no whois_updated_at, then oldest whois_updated_at.
|
||||
func (d *Database) GetNextStaleASN(ctx context.Context, staleThreshold time.Duration) (int, error) {
|
||||
cutoff := time.Now().Add(-staleThreshold)
|
||||
|
||||
query := `
|
||||
SELECT asn FROM asns
|
||||
WHERE whois_updated_at IS NULL
|
||||
OR whois_updated_at < ?
|
||||
ORDER BY
|
||||
CASE WHEN whois_updated_at IS NULL THEN 0 ELSE 1 END,
|
||||
whois_updated_at ASC
|
||||
LIMIT 1
|
||||
`
|
||||
|
||||
var asn int
|
||||
err := d.db.QueryRowContext(ctx, query, cutoff).Scan(&asn)
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return 0, ErrNoStaleASN
|
||||
}
|
||||
|
||||
return 0, fmt.Errorf("failed to get stale ASN: %w", err)
|
||||
}
|
||||
|
||||
return asn, nil
|
||||
}
|
||||
|
||||
// UpdateASNWHOIS updates an ASN record with WHOIS data.
|
||||
func (d *Database) UpdateASNWHOIS(ctx context.Context, update *ASNWHOISUpdate) error {
|
||||
d.lock("UpdateASNWHOIS")
|
||||
defer d.unlock()
|
||||
|
||||
query := `
|
||||
UPDATE asns SET
|
||||
as_name = ?,
|
||||
org_name = ?,
|
||||
org_id = ?,
|
||||
address = ?,
|
||||
country_code = ?,
|
||||
abuse_email = ?,
|
||||
abuse_phone = ?,
|
||||
tech_email = ?,
|
||||
tech_phone = ?,
|
||||
rir = ?,
|
||||
rir_registration_date = ?,
|
||||
rir_last_modified = ?,
|
||||
whois_raw = ?,
|
||||
whois_updated_at = ?
|
||||
WHERE asn = ?
|
||||
`
|
||||
|
||||
_, err := d.db.ExecContext(ctx, query,
|
||||
update.ASName,
|
||||
update.OrgName,
|
||||
update.OrgID,
|
||||
update.Address,
|
||||
update.CountryCode,
|
||||
update.AbuseEmail,
|
||||
update.AbusePhone,
|
||||
update.TechEmail,
|
||||
update.TechPhone,
|
||||
update.RIR,
|
||||
update.RIRRegDate,
|
||||
update.RIRLastMod,
|
||||
update.WHOISRaw,
|
||||
time.Now(),
|
||||
update.ASN,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to update ASN WHOIS: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetIPInfo returns comprehensive IP information for the /ip endpoint.
|
||||
func (d *Database) GetIPInfo(ip string) (*IPInfo, error) {
|
||||
return d.GetIPInfoContext(context.Background(), ip)
|
||||
}
|
||||
|
||||
// GetIPInfoContext returns comprehensive IP information with context support.
|
||||
func (d *Database) GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, error) {
|
||||
// Parse the IP to validate it
|
||||
parsedIP := net.ParseIP(ip)
|
||||
if parsedIP == nil {
|
||||
return nil, fmt.Errorf("%w: %s", ErrInvalidIP, ip)
|
||||
}
|
||||
|
||||
// Determine IP version
|
||||
ipv4 := parsedIP.To4()
|
||||
if ipv4 != nil {
|
||||
return d.getIPv4Info(ctx, ip, ipv4)
|
||||
}
|
||||
|
||||
return d.getIPv6Info(ctx, ip, parsedIP)
|
||||
}
|
||||
|
||||
// getIPv4Info returns comprehensive IP information for an IPv4 address.
|
||||
func (d *Database) getIPv4Info(ctx context.Context, ip string, ipv4 net.IP) (*IPInfo, error) {
|
||||
info := &IPInfo{
|
||||
IP: ip,
|
||||
IPVersion: ipVersionV4,
|
||||
}
|
||||
|
||||
ipUint := ipToUint32(ipv4)
|
||||
|
||||
// Get route info with peer count and prefix first_seen
|
||||
query := `
|
||||
SELECT
|
||||
lr.prefix,
|
||||
lr.mask_length,
|
||||
lr.origin_asn,
|
||||
lr.last_updated,
|
||||
(SELECT COUNT(DISTINCT peer_ip) FROM live_routes_v4 WHERE prefix = lr.prefix) as num_peers,
|
||||
p.first_seen,
|
||||
a.handle,
|
||||
a.description,
|
||||
a.as_name,
|
||||
a.org_name,
|
||||
a.org_id,
|
||||
a.address,
|
||||
a.country_code,
|
||||
a.abuse_email,
|
||||
a.rir,
|
||||
a.whois_updated_at
|
||||
FROM live_routes_v4 lr
|
||||
LEFT JOIN prefixes_v4 p ON p.prefix = lr.prefix
|
||||
LEFT JOIN asns a ON a.asn = lr.origin_asn
|
||||
WHERE lr.ip_start <= ? AND lr.ip_end >= ?
|
||||
ORDER BY lr.mask_length DESC
|
||||
LIMIT 1
|
||||
`
|
||||
|
||||
var handle, description, asName, orgName, orgID, address, countryCode, abuseEmail, rir sql.NullString
|
||||
var prefixFirstSeen sql.NullTime
|
||||
var whoisUpdatedAt sql.NullTime
|
||||
|
||||
err := d.db.QueryRowContext(ctx, query, ipUint, ipUint).Scan(
|
||||
&info.Netblock,
|
||||
&info.MaskLength,
|
||||
&info.ASN,
|
||||
&info.LastSeen,
|
||||
&info.NumPeers,
|
||||
&prefixFirstSeen,
|
||||
&handle,
|
||||
&description,
|
||||
&asName,
|
||||
&orgName,
|
||||
&orgID,
|
||||
&address,
|
||||
&countryCode,
|
||||
&abuseEmail,
|
||||
&rir,
|
||||
&whoisUpdatedAt,
|
||||
)
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, fmt.Errorf("%w for IP %s", ErrNoRoute, ip)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("failed to query routes: %w", err)
|
||||
}
|
||||
|
||||
info.Handle = handle.String
|
||||
info.Description = description.String
|
||||
info.ASName = asName.String
|
||||
info.OrgName = orgName.String
|
||||
info.OrgID = orgID.String
|
||||
info.Address = address.String
|
||||
info.CountryCode = countryCode.String
|
||||
info.AbuseEmail = abuseEmail.String
|
||||
info.RIR = rir.String
|
||||
|
||||
if prefixFirstSeen.Valid {
|
||||
info.FirstSeen = prefixFirstSeen.Time
|
||||
}
|
||||
|
||||
// Check if WHOIS data needs refresh (never fetched or older than 30 days)
|
||||
const staleThreshold = 30 * 24 * time.Hour
|
||||
info.NeedsWHOISRefresh = !whoisUpdatedAt.Valid || time.Since(whoisUpdatedAt.Time) > staleThreshold
|
||||
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// getIPv6Info returns comprehensive IP information for an IPv6 address.
|
||||
func (d *Database) getIPv6Info(ctx context.Context, ip string, parsedIP net.IP) (*IPInfo, error) {
|
||||
info := &IPInfo{
|
||||
IP: ip,
|
||||
IPVersion: ipVersionV6,
|
||||
}
|
||||
|
||||
// For IPv6, scan all routes and find best match
|
||||
query := `
|
||||
SELECT DISTINCT
|
||||
lr.prefix,
|
||||
lr.mask_length,
|
||||
lr.origin_asn,
|
||||
lr.last_updated,
|
||||
a.handle,
|
||||
a.description,
|
||||
a.as_name,
|
||||
a.org_name,
|
||||
a.org_id,
|
||||
a.address,
|
||||
a.country_code,
|
||||
a.abuse_email,
|
||||
a.rir,
|
||||
a.whois_updated_at
|
||||
FROM live_routes_v6 lr
|
||||
LEFT JOIN asns a ON a.asn = lr.origin_asn
|
||||
ORDER BY lr.mask_length DESC
|
||||
`
|
||||
|
||||
rows, err := d.db.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query routes: %w", err)
|
||||
}
|
||||
defer func() { _ = rows.Close() }()
|
||||
|
||||
bestMaskLength := -1
|
||||
|
||||
for rows.Next() {
|
||||
var prefix string
|
||||
var maskLength, originASN int
|
||||
var lastUpdated time.Time
|
||||
var handle, description, asName, orgName, orgID, address, countryCode, abuseEmail, rir sql.NullString
|
||||
var whoisUpdatedAt sql.NullTime
|
||||
|
||||
if err := rows.Scan(
|
||||
&prefix, &maskLength, &originASN, &lastUpdated,
|
||||
&handle, &description, &asName, &orgName, &orgID,
|
||||
&address, &countryCode, &abuseEmail, &rir, &whoisUpdatedAt,
|
||||
); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
_, ipNet, err := net.ParseCIDR(prefix)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if ipNet.Contains(parsedIP) && maskLength > bestMaskLength {
|
||||
info.Netblock = prefix
|
||||
info.MaskLength = maskLength
|
||||
info.ASN = originASN
|
||||
info.LastSeen = lastUpdated
|
||||
info.Handle = handle.String
|
||||
info.Description = description.String
|
||||
info.ASName = asName.String
|
||||
info.OrgName = orgName.String
|
||||
info.OrgID = orgID.String
|
||||
info.Address = address.String
|
||||
info.CountryCode = countryCode.String
|
||||
info.AbuseEmail = abuseEmail.String
|
||||
info.RIR = rir.String
|
||||
bestMaskLength = maskLength
|
||||
|
||||
if !whoisUpdatedAt.Valid {
|
||||
info.NeedsWHOISRefresh = true
|
||||
} else {
|
||||
const staleThreshold = 30 * 24 * time.Hour
|
||||
info.NeedsWHOISRefresh = time.Since(whoisUpdatedAt.Time) > staleThreshold
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if bestMaskLength == -1 {
|
||||
return nil, fmt.Errorf("%w for IP %s", ErrNoRoute, ip)
|
||||
}
|
||||
|
||||
// Get peer count and first_seen for IPv6
|
||||
countQuery := `SELECT COUNT(DISTINCT peer_ip) FROM live_routes_v6 WHERE prefix = ?`
|
||||
_ = d.db.QueryRowContext(ctx, countQuery, info.Netblock).Scan(&info.NumPeers)
|
||||
|
||||
firstSeenQuery := `SELECT first_seen FROM prefixes_v6 WHERE prefix = ?`
|
||||
var prefixFirstSeen sql.NullTime
|
||||
err = d.db.QueryRowContext(ctx, firstSeenQuery, info.Netblock).Scan(&prefixFirstSeen)
|
||||
if err == nil && prefixFirstSeen.Valid {
|
||||
info.FirstSeen = prefixFirstSeen.Time
|
||||
}
|
||||
|
||||
return info, nil
|
||||
}
|
||||
|
||||
@@ -61,6 +61,12 @@ type Store interface {
|
||||
// IP lookup operations
|
||||
GetASInfoForIP(ip string) (*ASInfo, error)
|
||||
GetASInfoForIPContext(ctx context.Context, ip string) (*ASInfo, error)
|
||||
GetIPInfo(ip string) (*IPInfo, error)
|
||||
GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, error)
|
||||
|
||||
// ASN WHOIS operations
|
||||
GetNextStaleASN(ctx context.Context, staleThreshold time.Duration) (int, error)
|
||||
UpdateASNWHOIS(ctx context.Context, update *ASNWHOISUpdate) error
|
||||
|
||||
// AS and prefix detail operations
|
||||
GetASDetails(asn int) (*ASN, []LiveRoute, error)
|
||||
|
||||
@@ -8,13 +8,29 @@ import (
|
||||
)
|
||||
|
||||
// ASN represents an Autonomous System Number with its metadata including
|
||||
// handle, description, and first/last seen timestamps.
|
||||
// handle, description, WHOIS data, and first/last seen timestamps.
|
||||
type ASN struct {
|
||||
ASN int `json:"asn"`
|
||||
Handle string `json:"handle"`
|
||||
Description string `json:"description"`
|
||||
FirstSeen time.Time `json:"first_seen"`
|
||||
LastSeen time.Time `json:"last_seen"`
|
||||
ASN int `json:"asn"`
|
||||
Handle string `json:"handle"`
|
||||
Description string `json:"description"`
|
||||
// WHOIS parsed fields
|
||||
ASName string `json:"as_name,omitempty"`
|
||||
OrgName string `json:"org_name,omitempty"`
|
||||
OrgID string `json:"org_id,omitempty"`
|
||||
Address string `json:"address,omitempty"`
|
||||
CountryCode string `json:"country_code,omitempty"`
|
||||
AbuseEmail string `json:"abuse_email,omitempty"`
|
||||
AbusePhone string `json:"abuse_phone,omitempty"`
|
||||
TechEmail string `json:"tech_email,omitempty"`
|
||||
TechPhone string `json:"tech_phone,omitempty"`
|
||||
RIR string `json:"rir,omitempty"` // ARIN, RIPE, APNIC, LACNIC, AFRINIC
|
||||
RIRRegDate *time.Time `json:"rir_registration_date,omitempty"`
|
||||
RIRLastMod *time.Time `json:"rir_last_modified,omitempty"`
|
||||
WHOISRaw string `json:"whois_raw,omitempty"`
|
||||
// Timestamps
|
||||
FirstSeen time.Time `json:"first_seen"`
|
||||
LastSeen time.Time `json:"last_seen"`
|
||||
WHOISUpdatedAt *time.Time `json:"whois_updated_at,omitempty"`
|
||||
}
|
||||
|
||||
// Prefix represents an IP prefix (CIDR block) with its IP version (4 or 6)
|
||||
@@ -72,7 +88,7 @@ type PrefixDistribution struct {
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
// ASInfo represents AS information for an IP lookup
|
||||
// ASInfo represents AS information for an IP lookup (legacy format)
|
||||
type ASInfo struct {
|
||||
ASN int `json:"asn"`
|
||||
Handle string `json:"handle"`
|
||||
@@ -82,6 +98,31 @@ type ASInfo struct {
|
||||
Age string `json:"age"`
|
||||
}
|
||||
|
||||
// IPInfo represents comprehensive IP information for the /ip endpoint
|
||||
type IPInfo struct {
|
||||
IP string `json:"ip"`
|
||||
Netblock string `json:"netblock"`
|
||||
MaskLength int `json:"mask_length"`
|
||||
IPVersion int `json:"ip_version"`
|
||||
NumPeers int `json:"num_peers"`
|
||||
// AS information
|
||||
ASN int `json:"asn"`
|
||||
ASName string `json:"as_name,omitempty"`
|
||||
Handle string `json:"handle,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
OrgName string `json:"org_name,omitempty"`
|
||||
OrgID string `json:"org_id,omitempty"`
|
||||
Address string `json:"address,omitempty"`
|
||||
CountryCode string `json:"country_code,omitempty"`
|
||||
AbuseEmail string `json:"abuse_email,omitempty"`
|
||||
RIR string `json:"rir,omitempty"`
|
||||
// Timestamps
|
||||
FirstSeen time.Time `json:"first_seen"`
|
||||
LastSeen time.Time `json:"last_seen"`
|
||||
// Indicates if WHOIS data needs refresh (not serialized)
|
||||
NeedsWHOISRefresh bool `json:"-"`
|
||||
}
|
||||
|
||||
// LiveRouteDeletion represents parameters for deleting a live route
|
||||
type LiveRouteDeletion struct {
|
||||
Prefix string
|
||||
@@ -97,3 +138,21 @@ type PeerUpdate struct {
|
||||
MessageType string
|
||||
Timestamp time.Time
|
||||
}
|
||||
|
||||
// ASNWHOISUpdate contains WHOIS data for updating an ASN record.
|
||||
type ASNWHOISUpdate struct {
|
||||
ASN int
|
||||
ASName string
|
||||
OrgName string
|
||||
OrgID string
|
||||
Address string
|
||||
CountryCode string
|
||||
AbuseEmail string
|
||||
AbusePhone string
|
||||
TechEmail string
|
||||
TechPhone string
|
||||
RIR string
|
||||
RIRRegDate *time.Time
|
||||
RIRLastMod *time.Time
|
||||
WHOISRaw string
|
||||
}
|
||||
|
||||
@@ -6,8 +6,25 @@ CREATE TABLE IF NOT EXISTS asns (
|
||||
asn INTEGER PRIMARY KEY,
|
||||
handle TEXT,
|
||||
description TEXT,
|
||||
-- WHOIS parsed fields
|
||||
as_name TEXT,
|
||||
org_name TEXT,
|
||||
org_id TEXT,
|
||||
address TEXT, -- full address (may be multi-line)
|
||||
country_code TEXT,
|
||||
abuse_email TEXT,
|
||||
abuse_phone TEXT,
|
||||
tech_email TEXT,
|
||||
tech_phone TEXT,
|
||||
rir TEXT, -- ARIN, RIPE, APNIC, LACNIC, AFRINIC
|
||||
rir_registration_date DATETIME,
|
||||
rir_last_modified DATETIME,
|
||||
-- Raw WHOIS response
|
||||
whois_raw TEXT, -- complete WHOIS response text
|
||||
-- Timestamps
|
||||
first_seen DATETIME NOT NULL,
|
||||
last_seen DATETIME NOT NULL
|
||||
last_seen DATETIME NOT NULL,
|
||||
whois_updated_at DATETIME -- when we last fetched WHOIS data
|
||||
);
|
||||
|
||||
-- IPv4 prefixes table
|
||||
|
||||
Reference in New Issue
Block a user