Add IP information API with background WHOIS fetcher
- Add /ip and /ip/{addr} JSON endpoints returning comprehensive IP info
- Include ASN, netblock, country code, org name, abuse contact, RIR data
- Extend ASN schema with WHOIS fields (country, org, abuse contact, etc)
- Create background WHOIS fetcher for rate-limited ASN info updates
- Store raw WHOIS responses for debugging and data preservation
- Queue on-demand WHOIS lookups when stale data is requested
- Refactor handleIPInfo to serve all IP endpoints consistently
This commit is contained in:
@@ -0,0 +1,155 @@
|
||||
// Package routewatch contains the ASN WHOIS fetcher for background updates.
|
||||
package routewatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
||||
"git.eeqj.de/sneak/routewatch/internal/whois"
|
||||
)
|
||||
|
||||
// ASN fetcher configuration constants.
|
||||
const (
|
||||
// backgroundFetchInterval is how often the background fetcher runs.
|
||||
backgroundFetchInterval = time.Minute
|
||||
|
||||
// whoisStaleThreshold is how old WHOIS data can be before refresh.
|
||||
whoisStaleThreshold = 30 * 24 * time.Hour // 30 days
|
||||
|
||||
// immediateQueueSize is the buffer size for immediate fetch requests.
|
||||
immediateQueueSize = 100
|
||||
)
|
||||
|
||||
// ASNFetcher handles background WHOIS lookups for ASNs.
|
||||
type ASNFetcher struct {
|
||||
db database.Store
|
||||
whoisClient *whois.Client
|
||||
logger *slog.Logger
|
||||
immediateQueue chan int
|
||||
stopCh chan struct{}
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// NewASNFetcher creates a new ASN fetcher.
|
||||
func NewASNFetcher(db database.Store, logger *slog.Logger) *ASNFetcher {
|
||||
return &ASNFetcher{
|
||||
db: db,
|
||||
whoisClient: whois.NewClient(),
|
||||
logger: logger.With("component", "asn_fetcher"),
|
||||
immediateQueue: make(chan int, immediateQueueSize),
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// Start begins the background ASN fetcher goroutine.
|
||||
func (f *ASNFetcher) Start() {
|
||||
f.wg.Add(1)
|
||||
go f.run()
|
||||
f.logger.Info("ASN fetcher started", "interval", backgroundFetchInterval)
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the fetcher.
|
||||
func (f *ASNFetcher) Stop() {
|
||||
close(f.stopCh)
|
||||
f.wg.Wait()
|
||||
f.logger.Info("ASN fetcher stopped")
|
||||
}
|
||||
|
||||
// QueueImmediate queues an ASN for immediate WHOIS lookup.
|
||||
// Non-blocking - if queue is full, the request is dropped.
|
||||
func (f *ASNFetcher) QueueImmediate(asn int) {
|
||||
select {
|
||||
case f.immediateQueue <- asn:
|
||||
f.logger.Debug("Queued immediate WHOIS lookup", "asn", asn)
|
||||
default:
|
||||
f.logger.Debug("Immediate queue full, dropping request", "asn", asn)
|
||||
}
|
||||
}
|
||||
|
||||
// run is the main background loop.
|
||||
func (f *ASNFetcher) run() {
|
||||
defer f.wg.Done()
|
||||
|
||||
ticker := time.NewTicker(backgroundFetchInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-f.stopCh:
|
||||
return
|
||||
|
||||
case asn := <-f.immediateQueue:
|
||||
// Process immediate request
|
||||
f.fetchAndUpdate(asn)
|
||||
|
||||
case <-ticker.C:
|
||||
// Background fetch of stale/missing ASN
|
||||
f.fetchNextStale()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// fetchNextStale finds and fetches the next ASN needing WHOIS data.
|
||||
func (f *ASNFetcher) fetchNextStale() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
|
||||
asn, err := f.db.GetNextStaleASN(ctx, whoisStaleThreshold)
|
||||
if err != nil {
|
||||
if err != database.ErrNoStaleASN {
|
||||
f.logger.Error("Failed to get stale ASN", "error", err)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
f.fetchAndUpdate(asn)
|
||||
}
|
||||
|
||||
// fetchAndUpdate performs a WHOIS lookup and updates the database.
|
||||
func (f *ASNFetcher) fetchAndUpdate(asn int) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
|
||||
f.logger.Info("Fetching WHOIS data", "asn", asn)
|
||||
|
||||
info, err := f.whoisClient.LookupASN(ctx, asn)
|
||||
if err != nil {
|
||||
f.logger.Error("WHOIS lookup failed", "asn", asn, "error", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Update database with WHOIS data
|
||||
err = f.db.UpdateASNWHOIS(ctx, &database.ASNWHOISUpdate{
|
||||
ASN: asn,
|
||||
ASName: info.ASName,
|
||||
OrgName: info.OrgName,
|
||||
OrgID: info.OrgID,
|
||||
Address: info.Address,
|
||||
CountryCode: info.CountryCode,
|
||||
AbuseEmail: info.AbuseEmail,
|
||||
AbusePhone: info.AbusePhone,
|
||||
TechEmail: info.TechEmail,
|
||||
TechPhone: info.TechPhone,
|
||||
RIR: info.RIR,
|
||||
RIRRegDate: info.RegDate,
|
||||
RIRLastMod: info.LastMod,
|
||||
WHOISRaw: info.RawResponse,
|
||||
})
|
||||
if err != nil {
|
||||
f.logger.Error("Failed to update ASN WHOIS data", "asn", asn, "error", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
f.logger.Info("Updated ASN WHOIS data",
|
||||
"asn", asn,
|
||||
"org_name", info.OrgName,
|
||||
"country", info.CountryCode,
|
||||
"rir", info.RIR,
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user