Compare commits
49 Commits
optimize-s
...
9043cf9bc0
| Author | SHA1 | Date | |
|---|---|---|---|
| 9043cf9bc0 | |||
| 3a9ec98d5c | |||
| 0ae89c33db | |||
| 8e79b8c074 | |||
| 5d7358fce6 | |||
| d7e6f46320 | |||
| da6d605e4d | |||
| d2041a5a55 | |||
| f8b7d3b773 | |||
| cb75409647 | |||
| 8eaf4e5f4b | |||
| 3b159454eb | |||
| 7e4dc528bd | |||
| ab392d874c | |||
| 95bbb655ab | |||
| 23dcdd800b | |||
| c292fef0ac | |||
| e1d0ab5ea6 | |||
| 8323a95be9 | |||
| 2f96141e48 | |||
| 1ec0b3e7ca | |||
| 037bbfb813 | |||
| 1fded42651 | |||
| 3338e92785 | |||
| 7aec01c499 | |||
| deeedae180 | |||
| d3966f2320 | |||
| 23127b86e9 | |||
| 2cfca78464 | |||
| c9da20e630 | |||
| a165ecf759 | |||
| 725d04ffa8 | |||
| fc32090483 | |||
| 3673264552 | |||
| 8e12c07396 | |||
| b6ad50f23f | |||
| c35b76deb8 | |||
| 6d46bbad5b | |||
| 9518519208 | |||
| 7d39bd18bc | |||
| e0a4c8642e | |||
| 0196251906 | |||
| 62ed5e08aa | |||
| 5fb3fc0381 | |||
| 9a63553f8d | |||
| ba13c76c53 | |||
| 1dcde74a90 | |||
| 81267431f7 | |||
| dc3ceb8d94 |
3
.gitignore
vendored
3
.gitignore
vendored
@@ -6,6 +6,9 @@
|
|||||||
*.dylib
|
*.dylib
|
||||||
/bin/
|
/bin/
|
||||||
|
|
||||||
|
.DS_Store
|
||||||
|
log.txt
|
||||||
|
|
||||||
# Test binary, built with `go test -c`
|
# Test binary, built with `go test -c`
|
||||||
*.test
|
*.test
|
||||||
|
|
||||||
|
|||||||
66
Dockerfile
Normal file
66
Dockerfile
Normal file
@@ -0,0 +1,66 @@
|
|||||||
|
# Build stage
|
||||||
|
FROM golang:1.24-bookworm AS builder
|
||||||
|
|
||||||
|
# Install build dependencies (zstd for archive, gcc for CGO/sqlite3)
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
zstd \
|
||||||
|
gcc \
|
||||||
|
libc6-dev \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
WORKDIR /src
|
||||||
|
|
||||||
|
# Copy everything
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# Vendor dependencies (must be after copying source)
|
||||||
|
RUN go mod download && go mod vendor
|
||||||
|
|
||||||
|
# Build the binary with CGO enabled (required for sqlite3)
|
||||||
|
RUN CGO_ENABLED=1 GOOS=linux go build -o /routewatch ./cmd/routewatch
|
||||||
|
|
||||||
|
# Create source archive with vendored dependencies
|
||||||
|
RUN tar --zstd -cf /routewatch-source.tar.zst \
|
||||||
|
--exclude='.git' \
|
||||||
|
--exclude='*.tar.zst' \
|
||||||
|
.
|
||||||
|
|
||||||
|
# Runtime stage
|
||||||
|
FROM debian:bookworm-slim
|
||||||
|
|
||||||
|
# Install runtime dependencies
|
||||||
|
# - ca-certificates: for HTTPS connections
|
||||||
|
# - curl: for health checks
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
ca-certificates \
|
||||||
|
curl \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Create non-root user
|
||||||
|
RUN useradd -r -u 1000 -m routewatch
|
||||||
|
|
||||||
|
# Create state directory
|
||||||
|
RUN mkdir -p /var/lib/routewatch && chown routewatch:routewatch /var/lib/routewatch
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Copy binary and source archive from builder
|
||||||
|
COPY --from=builder /routewatch /app/routewatch
|
||||||
|
COPY --from=builder /routewatch-source.tar.zst /app/source/routewatch-source.tar.zst
|
||||||
|
|
||||||
|
# Set ownership
|
||||||
|
RUN chown -R routewatch:routewatch /app
|
||||||
|
|
||||||
|
USER routewatch
|
||||||
|
|
||||||
|
# Default state directory
|
||||||
|
ENV ROUTEWATCH_STATE_DIR=/var/lib/routewatch
|
||||||
|
|
||||||
|
# Expose HTTP port
|
||||||
|
EXPOSE 8080
|
||||||
|
|
||||||
|
# Health check using the health endpoint
|
||||||
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
|
||||||
|
CMD curl -sf http://localhost:8080/.well-known/healthcheck.json || exit 1
|
||||||
|
|
||||||
|
ENTRYPOINT ["/app/routewatch"]
|
||||||
2
Makefile
2
Makefile
@@ -21,7 +21,7 @@ clean:
|
|||||||
rm -rf bin/
|
rm -rf bin/
|
||||||
|
|
||||||
run: build
|
run: build
|
||||||
./bin/routewatch
|
DEBUG=routewatch ./bin/routewatch 2>&1 | tee log.txt
|
||||||
|
|
||||||
asupdate:
|
asupdate:
|
||||||
@echo "Updating AS info data..."
|
@echo "Updating AS info data..."
|
||||||
|
|||||||
193
README.md
Normal file
193
README.md
Normal file
@@ -0,0 +1,193 @@
|
|||||||
|
# RouteWatch
|
||||||
|
|
||||||
|
RouteWatch is a real-time BGP routing table monitor that streams BGP UPDATE messages from the RIPE RIS Live service, maintains a live routing table in SQLite, and provides HTTP APIs for querying routing information.
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
- Real-time streaming of BGP updates from RIPE RIS Live
|
||||||
|
- Maintains live IPv4 and IPv6 routing tables
|
||||||
|
- Tracks AS peering relationships
|
||||||
|
- HTTP API for IP-to-AS lookups, prefix details, and AS information
|
||||||
|
- Automatic reconnection with exponential backoff
|
||||||
|
- Batched database writes for high performance
|
||||||
|
- Backpressure handling to prevent memory exhaustion
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go build -o routewatch ./cmd/routewatch
|
||||||
|
```
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Run the daemon (listens on port 8080 by default)
|
||||||
|
./routewatch
|
||||||
|
|
||||||
|
# Set custom port
|
||||||
|
PORT=3000 ./routewatch
|
||||||
|
|
||||||
|
# Enable debug logging
|
||||||
|
DEBUG=routewatch ./routewatch
|
||||||
|
```
|
||||||
|
|
||||||
|
## HTTP Endpoints
|
||||||
|
|
||||||
|
### Web Interface
|
||||||
|
- `GET /` - Redirects to /status
|
||||||
|
- `GET /status` - HTML status dashboard
|
||||||
|
- `GET /status.json` - JSON statistics
|
||||||
|
- `GET /as/{asn}` - AS detail page (HTML)
|
||||||
|
- `GET /prefix/{prefix}` - Prefix detail page (HTML)
|
||||||
|
- `GET /prefixlength/{length}` - IPv4 prefixes by mask length
|
||||||
|
- `GET /prefixlength6/{length}` - IPv6 prefixes by mask length
|
||||||
|
- `GET /ip/{ip}` - Redirects to prefix containing the IP
|
||||||
|
|
||||||
|
### API v1
|
||||||
|
- `GET /api/v1/stats` - Detailed statistics with handler metrics
|
||||||
|
- `GET /api/v1/ip/{ip}` - Look up AS information for an IP address
|
||||||
|
- `GET /api/v1/as/{asn}` - Get prefixes announced by an AS
|
||||||
|
- `GET /api/v1/prefix/{prefix}` - Get routes for a specific prefix
|
||||||
|
|
||||||
|
## Code Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
routewatch/
|
||||||
|
├── cmd/
|
||||||
|
│ ├── routewatch/ # Main daemon entry point
|
||||||
|
│ ├── asinfo-gen/ # Utility to generate AS info data
|
||||||
|
│ └── streamdumper/ # Debug utility for raw stream output
|
||||||
|
├── internal/
|
||||||
|
│ ├── routewatch/ # Core application logic
|
||||||
|
│ ├── server/ # HTTP server and handlers
|
||||||
|
│ ├── database/ # SQLite storage layer
|
||||||
|
│ ├── streamer/ # RIPE RIS Live client
|
||||||
|
│ ├── ristypes/ # BGP message data structures
|
||||||
|
│ ├── logger/ # Structured logging wrapper
|
||||||
|
│ ├── metrics/ # Performance metrics tracking
|
||||||
|
│ ├── config/ # Configuration management
|
||||||
|
│ └── templates/ # HTML templates
|
||||||
|
└── pkg/
|
||||||
|
└── asinfo/ # AS information lookup (public API)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Architecture Overview
|
||||||
|
|
||||||
|
### Component Relationships
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────────────────────────────────────────────────────┐
|
||||||
|
│ RouteWatch │
|
||||||
|
│ (internal/routewatch/app.go - main orchestrator) │
|
||||||
|
├─────────────────────────────────────────────────────────────────┤
|
||||||
|
│ │
|
||||||
|
│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │
|
||||||
|
│ │ Streamer │───▶│ Handlers │───▶│ Database │ │
|
||||||
|
│ │ │ │ │ │ │ │
|
||||||
|
│ │ RIS Live │ │ - ASHandler │ │ SQLite with │ │
|
||||||
|
│ │ WebSocket │ │ - PeerHandler│ │ WAL mode │ │
|
||||||
|
│ │ client │ │ - PrefixHdlr │ │ │ │
|
||||||
|
│ │ │ │ - PeeringHdlr│ │ Tables: │ │
|
||||||
|
│ └──────────────┘ └──────────────┘ │ - asns │ │
|
||||||
|
│ │ - prefixes │ │
|
||||||
|
│ ┌──────────────┐ ┌──────────────┐ │ - live_routes│ │
|
||||||
|
│ │ Server │───▶│ Handlers │───▶│ - peerings │ │
|
||||||
|
│ │ │ │ │ │ - bgp_peers │ │
|
||||||
|
│ │ Chi router │ │ Status, API │ └──────────────┘ │
|
||||||
|
│ │ port 8080 │ │ AS, Prefix │ │
|
||||||
|
│ └──────────────┘ └──────────────┘ │
|
||||||
|
│ │
|
||||||
|
└─────────────────────────────────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
### Execution Flow
|
||||||
|
|
||||||
|
1. **Startup** (`cmd/routewatch/main.go` → `internal/routewatch/cli.go`)
|
||||||
|
- Uber fx dependency injection initializes all components
|
||||||
|
- Signal handlers registered for graceful shutdown
|
||||||
|
|
||||||
|
2. **Initialization** (`internal/routewatch/app.go`)
|
||||||
|
- Database created with SQLite schema (WAL mode, 3GB cache)
|
||||||
|
- Message handlers registered with the streamer
|
||||||
|
- HTTP server started on configured port
|
||||||
|
|
||||||
|
3. **Message Processing Pipeline**
|
||||||
|
```
|
||||||
|
RIS Live Stream → JSON Parser → Message Dispatcher → Handler Queues → Batch Writers → SQLite
|
||||||
|
```
|
||||||
|
- Streamer connects to `ris-live.ripe.net` via HTTP
|
||||||
|
- Parses BGP UPDATE messages from JSON stream
|
||||||
|
- Dispatches to registered handlers based on message type
|
||||||
|
- Each handler has its own queue with backpressure handling
|
||||||
|
- Handlers batch writes for efficiency (25K-30K ops, 1-2s timeout)
|
||||||
|
|
||||||
|
4. **Handler Details**
|
||||||
|
- **ASHandler**: Tracks all ASNs seen in AS paths
|
||||||
|
- **PeerHandler**: Records BGP peer information
|
||||||
|
- **PrefixHandler**: Maintains live routing table (upserts on announcement, deletes on withdrawal)
|
||||||
|
- **PeeringHandler**: Extracts AS peering relationships from AS paths
|
||||||
|
|
||||||
|
5. **HTTP Request Flow**
|
||||||
|
```
|
||||||
|
Request → Chi Router → Middleware (timeout, logging) → Handler → Database Query → Response
|
||||||
|
```
|
||||||
|
|
||||||
|
### Key Design Patterns
|
||||||
|
|
||||||
|
- **Batched Writes**: All database operations are batched for performance
|
||||||
|
- **Backpressure**: Probabilistic message dropping when queues exceed 50% capacity
|
||||||
|
- **Graceful Shutdown**: 60-second timeout, flushes all pending batches
|
||||||
|
- **Reconnection**: Exponential backoff (5s-320s) with reset after 30s of stable connection
|
||||||
|
- **IPv4 Optimization**: IP ranges stored as uint32 for O(1) lookups
|
||||||
|
|
||||||
|
### Database Schema
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- Core tables
|
||||||
|
asns(id, number, handle, description, first_seen, last_seen)
|
||||||
|
prefixes_v4(id, prefix, mask_length, first_seen, last_seen)
|
||||||
|
prefixes_v6(id, prefix, mask_length, first_seen, last_seen)
|
||||||
|
|
||||||
|
-- Live routing tables (one per IP version)
|
||||||
|
live_routes_v4(id, prefix, mask_length, origin_asn, peer_ip, as_path,
|
||||||
|
next_hop, last_updated, v4_ip_start, v4_ip_end)
|
||||||
|
live_routes_v6(id, prefix, mask_length, origin_asn, peer_ip, as_path,
|
||||||
|
next_hop, last_updated)
|
||||||
|
|
||||||
|
-- Relationship tracking
|
||||||
|
peerings(id, as_a, as_b, first_seen, last_seen)
|
||||||
|
bgp_peers(id, peer_ip, peer_asn, last_message_type, last_seen)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
Configuration is handled via environment variables and OS-specific paths:
|
||||||
|
|
||||||
|
| Variable | Default | Description |
|
||||||
|
|----------|---------|-------------|
|
||||||
|
| `PORT` | `8080` | HTTP server port |
|
||||||
|
| `DEBUG` | (empty) | Set to `routewatch` for debug logging |
|
||||||
|
|
||||||
|
State directory (database location):
|
||||||
|
- macOS: `~/Library/Application Support/routewatch/`
|
||||||
|
- Linux: `/var/lib/routewatch/` or `~/.local/share/routewatch/`
|
||||||
|
|
||||||
|
## Development
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Run tests
|
||||||
|
make test
|
||||||
|
|
||||||
|
# Format code
|
||||||
|
make fmt
|
||||||
|
|
||||||
|
# Run linter
|
||||||
|
make lint
|
||||||
|
|
||||||
|
# Build
|
||||||
|
make
|
||||||
|
```
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
See LICENSE file.
|
||||||
4
go.mod
4
go.mod
@@ -3,18 +3,18 @@ module git.eeqj.de/sneak/routewatch
|
|||||||
go 1.24.4
|
go 1.24.4
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/dustin/go-humanize v1.0.1
|
||||||
github.com/go-chi/chi/v5 v5.2.2
|
github.com/go-chi/chi/v5 v5.2.2
|
||||||
github.com/google/uuid v1.6.0
|
github.com/google/uuid v1.6.0
|
||||||
github.com/mattn/go-sqlite3 v1.14.29
|
github.com/mattn/go-sqlite3 v1.14.29
|
||||||
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9
|
github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9
|
||||||
go.uber.org/fx v1.24.0
|
go.uber.org/fx v1.24.0
|
||||||
|
golang.org/x/term v0.33.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
|
||||||
go.uber.org/dig v1.19.0 // indirect
|
go.uber.org/dig v1.19.0 // indirect
|
||||||
go.uber.org/multierr v1.10.0 // indirect
|
go.uber.org/multierr v1.10.0 // indirect
|
||||||
go.uber.org/zap v1.26.0 // indirect
|
go.uber.org/zap v1.26.0 // indirect
|
||||||
golang.org/x/sys v0.34.0 // indirect
|
golang.org/x/sys v0.34.0 // indirect
|
||||||
golang.org/x/term v0.33.0 // indirect
|
|
||||||
)
|
)
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -1,10 +1,14 @@
|
|||||||
|
// Package database provides SQLite storage for BGP routing data including ASNs,
|
||||||
|
// prefixes, announcements, peerings, and live route tables.
|
||||||
package database
|
package database
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Stats contains database statistics
|
// Stats contains database statistics including counts for ASNs, prefixes,
|
||||||
|
// peerings, peers, and live routes, as well as file size and prefix distribution data.
|
||||||
type Stats struct {
|
type Stats struct {
|
||||||
ASNs int
|
ASNs int
|
||||||
Prefixes int
|
Prefixes int
|
||||||
@@ -18,7 +22,9 @@ type Stats struct {
|
|||||||
IPv6PrefixDistribution []PrefixDistribution
|
IPv6PrefixDistribution []PrefixDistribution
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store defines the interface for database operations
|
// Store defines the interface for database operations. It provides methods for
|
||||||
|
// managing ASNs, prefixes, announcements, peerings, BGP peers, and live routes.
|
||||||
|
// Implementations must be safe for concurrent use.
|
||||||
type Store interface {
|
type Store interface {
|
||||||
// ASN operations
|
// ASN operations
|
||||||
GetOrCreateASN(number int, timestamp time.Time) (*ASN, error)
|
GetOrCreateASN(number int, timestamp time.Time) (*ASN, error)
|
||||||
@@ -26,6 +32,7 @@ type Store interface {
|
|||||||
|
|
||||||
// Prefix operations
|
// Prefix operations
|
||||||
GetOrCreatePrefix(prefix string, timestamp time.Time) (*Prefix, error)
|
GetOrCreatePrefix(prefix string, timestamp time.Time) (*Prefix, error)
|
||||||
|
UpdatePrefixesBatch(prefixes map[string]time.Time) error
|
||||||
|
|
||||||
// Announcement operations
|
// Announcement operations
|
||||||
RecordAnnouncement(announcement *Announcement) error
|
RecordAnnouncement(announcement *Announcement) error
|
||||||
@@ -35,6 +42,7 @@ type Store interface {
|
|||||||
|
|
||||||
// Statistics
|
// Statistics
|
||||||
GetStats() (Stats, error)
|
GetStats() (Stats, error)
|
||||||
|
GetStatsContext(ctx context.Context) (Stats, error)
|
||||||
|
|
||||||
// Peer operations
|
// Peer operations
|
||||||
UpdatePeer(peerIP string, peerASN int, messageType string, timestamp time.Time) error
|
UpdatePeer(peerIP string, peerASN int, messageType string, timestamp time.Time) error
|
||||||
@@ -46,17 +54,37 @@ type Store interface {
|
|||||||
DeleteLiveRoute(prefix string, originASN int, peerIP string) error
|
DeleteLiveRoute(prefix string, originASN int, peerIP string) error
|
||||||
DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error
|
DeleteLiveRouteBatch(deletions []LiveRouteDeletion) error
|
||||||
GetPrefixDistribution() (ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error)
|
GetPrefixDistribution() (ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error)
|
||||||
|
GetPrefixDistributionContext(ctx context.Context) (ipv4 []PrefixDistribution, ipv6 []PrefixDistribution, err error)
|
||||||
GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error)
|
GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error)
|
||||||
|
GetLiveRouteCountsContext(ctx context.Context) (ipv4Count, ipv6Count int, err error)
|
||||||
|
|
||||||
// IP lookup operations
|
// IP lookup operations
|
||||||
GetASInfoForIP(ip string) (*ASInfo, error)
|
GetASInfoForIP(ip string) (*ASInfo, error)
|
||||||
|
GetASInfoForIPContext(ctx context.Context, ip string) (*ASInfo, error)
|
||||||
|
GetIPInfo(ip string) (*IPInfo, error)
|
||||||
|
GetIPInfoContext(ctx context.Context, ip string) (*IPInfo, error)
|
||||||
|
|
||||||
|
// ASN WHOIS operations
|
||||||
|
GetNextStaleASN(ctx context.Context, staleThreshold time.Duration) (int, error)
|
||||||
|
UpdateASNWHOIS(ctx context.Context, update *ASNWHOISUpdate) error
|
||||||
|
GetWHOISStats(ctx context.Context, staleThreshold time.Duration) (*WHOISStats, error)
|
||||||
|
|
||||||
// AS and prefix detail operations
|
// AS and prefix detail operations
|
||||||
GetASDetails(asn int) (*ASN, []LiveRoute, error)
|
GetASDetails(asn int) (*ASN, []LiveRoute, error)
|
||||||
|
GetASDetailsContext(ctx context.Context, asn int) (*ASN, []LiveRoute, error)
|
||||||
|
GetASPeers(asn int) ([]ASPeer, error)
|
||||||
|
GetASPeersContext(ctx context.Context, asn int) ([]ASPeer, error)
|
||||||
GetPrefixDetails(prefix string) ([]LiveRoute, error)
|
GetPrefixDetails(prefix string) ([]LiveRoute, error)
|
||||||
|
GetPrefixDetailsContext(ctx context.Context, prefix string) ([]LiveRoute, error)
|
||||||
|
GetRandomPrefixesByLength(maskLength, ipVersion, limit int) ([]LiveRoute, error)
|
||||||
|
GetRandomPrefixesByLengthContext(ctx context.Context, maskLength, ipVersion, limit int) ([]LiveRoute, error)
|
||||||
|
|
||||||
// Lifecycle
|
// Lifecycle
|
||||||
Close() error
|
Close() error
|
||||||
|
|
||||||
|
// Maintenance operations
|
||||||
|
Vacuum(ctx context.Context) error
|
||||||
|
Analyze(ctx context.Context) error
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ensure Database implements Store
|
// Ensure Database implements Store
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
// Package database provides SQLite storage for BGP routing data.
|
||||||
package database
|
package database
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -6,17 +7,34 @@ import (
|
|||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ASN represents an Autonomous System Number
|
// ASN represents an Autonomous System Number with its metadata including
|
||||||
|
// handle, description, WHOIS data, and first/last seen timestamps.
|
||||||
type ASN struct {
|
type ASN struct {
|
||||||
ID uuid.UUID `json:"id"`
|
ASN int `json:"asn"`
|
||||||
Number int `json:"number"`
|
|
||||||
Handle string `json:"handle"`
|
Handle string `json:"handle"`
|
||||||
Description string `json:"description"`
|
Description string `json:"description"`
|
||||||
|
// WHOIS parsed fields
|
||||||
|
ASName string `json:"as_name,omitempty"`
|
||||||
|
OrgName string `json:"org_name,omitempty"`
|
||||||
|
OrgID string `json:"org_id,omitempty"`
|
||||||
|
Address string `json:"address,omitempty"`
|
||||||
|
CountryCode string `json:"country_code,omitempty"`
|
||||||
|
AbuseEmail string `json:"abuse_email,omitempty"`
|
||||||
|
AbusePhone string `json:"abuse_phone,omitempty"`
|
||||||
|
TechEmail string `json:"tech_email,omitempty"`
|
||||||
|
TechPhone string `json:"tech_phone,omitempty"`
|
||||||
|
RIR string `json:"rir,omitempty"` // ARIN, RIPE, APNIC, LACNIC, AFRINIC
|
||||||
|
RIRRegDate *time.Time `json:"rir_registration_date,omitempty"`
|
||||||
|
RIRLastMod *time.Time `json:"rir_last_modified,omitempty"`
|
||||||
|
WHOISRaw string `json:"whois_raw,omitempty"`
|
||||||
|
// Timestamps
|
||||||
FirstSeen time.Time `json:"first_seen"`
|
FirstSeen time.Time `json:"first_seen"`
|
||||||
LastSeen time.Time `json:"last_seen"`
|
LastSeen time.Time `json:"last_seen"`
|
||||||
|
WHOISUpdatedAt *time.Time `json:"whois_updated_at,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prefix represents an IP prefix (CIDR block)
|
// Prefix represents an IP prefix (CIDR block) with its IP version (4 or 6)
|
||||||
|
// and first/last seen timestamps.
|
||||||
type Prefix struct {
|
type Prefix struct {
|
||||||
ID uuid.UUID `json:"id"`
|
ID uuid.UUID `json:"id"`
|
||||||
Prefix string `json:"prefix"`
|
Prefix string `json:"prefix"`
|
||||||
@@ -25,23 +43,25 @@ type Prefix struct {
|
|||||||
LastSeen time.Time `json:"last_seen"`
|
LastSeen time.Time `json:"last_seen"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Announcement represents a BGP announcement
|
// Announcement represents a BGP announcement or withdrawal event,
|
||||||
|
// containing the prefix, AS path, origin ASN, peer ASN, next hop, and timestamp.
|
||||||
type Announcement struct {
|
type Announcement struct {
|
||||||
ID uuid.UUID `json:"id"`
|
ID uuid.UUID `json:"id"`
|
||||||
PrefixID uuid.UUID `json:"prefix_id"`
|
PrefixID uuid.UUID `json:"prefix_id"`
|
||||||
ASNID uuid.UUID `json:"asn_id"`
|
PeerASN int `json:"peer_asn"`
|
||||||
OriginASNID uuid.UUID `json:"origin_asn_id"`
|
OriginASN int `json:"origin_asn"`
|
||||||
Path string `json:"path"` // JSON-encoded AS path
|
Path string `json:"path"` // JSON-encoded AS path
|
||||||
NextHop string `json:"next_hop"`
|
NextHop string `json:"next_hop"`
|
||||||
Timestamp time.Time `json:"timestamp"`
|
Timestamp time.Time `json:"timestamp"`
|
||||||
IsWithdrawal bool `json:"is_withdrawal"`
|
IsWithdrawal bool `json:"is_withdrawal"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ASNPeering represents a peering relationship between two ASNs
|
// ASNPeering represents a peering relationship between two ASNs,
|
||||||
|
// stored with the lower ASN as ASA and the higher as ASB.
|
||||||
type ASNPeering struct {
|
type ASNPeering struct {
|
||||||
ID uuid.UUID `json:"id"`
|
ID uuid.UUID `json:"id"`
|
||||||
FromASNID uuid.UUID `json:"from_asn_id"`
|
ASA int `json:"as_a"`
|
||||||
ToASNID uuid.UUID `json:"to_asn_id"`
|
ASB int `json:"as_b"`
|
||||||
FirstSeen time.Time `json:"first_seen"`
|
FirstSeen time.Time `json:"first_seen"`
|
||||||
LastSeen time.Time `json:"last_seen"`
|
LastSeen time.Time `json:"last_seen"`
|
||||||
}
|
}
|
||||||
@@ -68,7 +88,7 @@ type PrefixDistribution struct {
|
|||||||
Count int `json:"count"`
|
Count int `json:"count"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ASInfo represents AS information for an IP lookup
|
// ASInfo represents AS information for an IP lookup (legacy format)
|
||||||
type ASInfo struct {
|
type ASInfo struct {
|
||||||
ASN int `json:"asn"`
|
ASN int `json:"asn"`
|
||||||
Handle string `json:"handle"`
|
Handle string `json:"handle"`
|
||||||
@@ -78,11 +98,38 @@ type ASInfo struct {
|
|||||||
Age string `json:"age"`
|
Age string `json:"age"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IPInfo represents comprehensive IP information for the /ip endpoint
|
||||||
|
type IPInfo struct {
|
||||||
|
IP string `json:"ip"`
|
||||||
|
PTR []string `json:"ptr,omitempty"`
|
||||||
|
Netblock string `json:"netblock"`
|
||||||
|
MaskLength int `json:"mask_length"`
|
||||||
|
IPVersion int `json:"ip_version"`
|
||||||
|
NumPeers int `json:"num_peers"`
|
||||||
|
// AS information
|
||||||
|
ASN int `json:"asn"`
|
||||||
|
ASName string `json:"as_name,omitempty"`
|
||||||
|
Handle string `json:"handle,omitempty"`
|
||||||
|
Description string `json:"description,omitempty"`
|
||||||
|
OrgName string `json:"org_name,omitempty"`
|
||||||
|
OrgID string `json:"org_id,omitempty"`
|
||||||
|
Address string `json:"address,omitempty"`
|
||||||
|
CountryCode string `json:"country_code,omitempty"`
|
||||||
|
AbuseEmail string `json:"abuse_email,omitempty"`
|
||||||
|
RIR string `json:"rir,omitempty"`
|
||||||
|
// Timestamps
|
||||||
|
FirstSeen time.Time `json:"first_seen"`
|
||||||
|
LastSeen time.Time `json:"last_seen"`
|
||||||
|
// Indicates if WHOIS data needs refresh (not serialized)
|
||||||
|
NeedsWHOISRefresh bool `json:"-"`
|
||||||
|
}
|
||||||
|
|
||||||
// LiveRouteDeletion represents parameters for deleting a live route
|
// LiveRouteDeletion represents parameters for deleting a live route
|
||||||
type LiveRouteDeletion struct {
|
type LiveRouteDeletion struct {
|
||||||
Prefix string
|
Prefix string
|
||||||
OriginASN int
|
OriginASN int
|
||||||
PeerIP string
|
PeerIP string
|
||||||
|
IPVersion int
|
||||||
}
|
}
|
||||||
|
|
||||||
// PeerUpdate represents parameters for updating a peer
|
// PeerUpdate represents parameters for updating a peer
|
||||||
@@ -92,3 +139,21 @@ type PeerUpdate struct {
|
|||||||
MessageType string
|
MessageType string
|
||||||
Timestamp time.Time
|
Timestamp time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ASNWHOISUpdate contains WHOIS data for updating an ASN record.
|
||||||
|
type ASNWHOISUpdate struct {
|
||||||
|
ASN int
|
||||||
|
ASName string
|
||||||
|
OrgName string
|
||||||
|
OrgID string
|
||||||
|
Address string
|
||||||
|
CountryCode string
|
||||||
|
AbuseEmail string
|
||||||
|
AbusePhone string
|
||||||
|
TechEmail string
|
||||||
|
TechPhone string
|
||||||
|
RIR string
|
||||||
|
RIRRegDate *time.Time
|
||||||
|
RIRLastMod *time.Time
|
||||||
|
WHOISRaw string
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,16 +1,44 @@
|
|||||||
|
-- IMPORTANT: This is the ONLY place where schema changes should be made.
|
||||||
|
-- We do NOT support migrations. All schema changes MUST be in this file.
|
||||||
|
-- DO NOT make schema changes anywhere else in the codebase.
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS asns (
|
CREATE TABLE IF NOT EXISTS asns (
|
||||||
id TEXT PRIMARY KEY,
|
asn INTEGER PRIMARY KEY,
|
||||||
number INTEGER UNIQUE NOT NULL,
|
|
||||||
handle TEXT,
|
handle TEXT,
|
||||||
description TEXT,
|
description TEXT,
|
||||||
|
-- WHOIS parsed fields
|
||||||
|
as_name TEXT,
|
||||||
|
org_name TEXT,
|
||||||
|
org_id TEXT,
|
||||||
|
address TEXT, -- full address (may be multi-line)
|
||||||
|
country_code TEXT,
|
||||||
|
abuse_email TEXT,
|
||||||
|
abuse_phone TEXT,
|
||||||
|
tech_email TEXT,
|
||||||
|
tech_phone TEXT,
|
||||||
|
rir TEXT, -- ARIN, RIPE, APNIC, LACNIC, AFRINIC
|
||||||
|
rir_registration_date DATETIME,
|
||||||
|
rir_last_modified DATETIME,
|
||||||
|
-- Raw WHOIS response
|
||||||
|
whois_raw TEXT, -- complete WHOIS response text
|
||||||
|
-- Timestamps
|
||||||
|
first_seen DATETIME NOT NULL,
|
||||||
|
last_seen DATETIME NOT NULL,
|
||||||
|
whois_updated_at DATETIME -- when we last fetched WHOIS data
|
||||||
|
);
|
||||||
|
|
||||||
|
-- IPv4 prefixes table
|
||||||
|
CREATE TABLE IF NOT EXISTS prefixes_v4 (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
prefix TEXT UNIQUE NOT NULL,
|
||||||
first_seen DATETIME NOT NULL,
|
first_seen DATETIME NOT NULL,
|
||||||
last_seen DATETIME NOT NULL
|
last_seen DATETIME NOT NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS prefixes (
|
-- IPv6 prefixes table
|
||||||
|
CREATE TABLE IF NOT EXISTS prefixes_v6 (
|
||||||
id TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
prefix TEXT UNIQUE NOT NULL,
|
prefix TEXT UNIQUE NOT NULL,
|
||||||
ip_version INTEGER NOT NULL, -- 4 for IPv4, 6 for IPv6
|
|
||||||
first_seen DATETIME NOT NULL,
|
first_seen DATETIME NOT NULL,
|
||||||
last_seen DATETIME NOT NULL
|
last_seen DATETIME NOT NULL
|
||||||
);
|
);
|
||||||
@@ -18,15 +46,14 @@ CREATE TABLE IF NOT EXISTS prefixes (
|
|||||||
CREATE TABLE IF NOT EXISTS announcements (
|
CREATE TABLE IF NOT EXISTS announcements (
|
||||||
id TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
prefix_id TEXT NOT NULL,
|
prefix_id TEXT NOT NULL,
|
||||||
asn_id TEXT NOT NULL,
|
peer_asn INTEGER NOT NULL,
|
||||||
origin_asn_id TEXT NOT NULL,
|
origin_asn INTEGER NOT NULL,
|
||||||
path TEXT NOT NULL,
|
path TEXT NOT NULL,
|
||||||
next_hop TEXT,
|
next_hop TEXT,
|
||||||
timestamp DATETIME NOT NULL,
|
timestamp DATETIME NOT NULL,
|
||||||
is_withdrawal BOOLEAN NOT NULL DEFAULT 0,
|
is_withdrawal BOOLEAN NOT NULL DEFAULT 0,
|
||||||
FOREIGN KEY (prefix_id) REFERENCES prefixes(id),
|
FOREIGN KEY (peer_asn) REFERENCES asns(asn),
|
||||||
FOREIGN KEY (asn_id) REFERENCES asns(id),
|
FOREIGN KEY (origin_asn) REFERENCES asns(asn)
|
||||||
FOREIGN KEY (origin_asn_id) REFERENCES asns(id)
|
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS peerings (
|
CREATE TABLE IF NOT EXISTS peerings (
|
||||||
@@ -48,47 +75,72 @@ CREATE TABLE IF NOT EXISTS bgp_peers (
|
|||||||
last_message_type TEXT
|
last_message_type TEXT
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_prefixes_ip_version ON prefixes(ip_version);
|
-- Indexes for prefixes_v4 table
|
||||||
CREATE INDEX IF NOT EXISTS idx_prefixes_version_prefix ON prefixes(ip_version, prefix);
|
CREATE INDEX IF NOT EXISTS idx_prefixes_v4_prefix ON prefixes_v4(prefix);
|
||||||
|
|
||||||
|
-- Indexes for prefixes_v6 table
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_prefixes_v6_prefix ON prefixes_v6(prefix);
|
||||||
CREATE INDEX IF NOT EXISTS idx_announcements_timestamp ON announcements(timestamp);
|
CREATE INDEX IF NOT EXISTS idx_announcements_timestamp ON announcements(timestamp);
|
||||||
CREATE INDEX IF NOT EXISTS idx_announcements_prefix_id ON announcements(prefix_id);
|
CREATE INDEX IF NOT EXISTS idx_announcements_prefix_id ON announcements(prefix_id);
|
||||||
CREATE INDEX IF NOT EXISTS idx_announcements_asn_id ON announcements(asn_id);
|
CREATE INDEX IF NOT EXISTS idx_announcements_peer_asn ON announcements(peer_asn);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_announcements_origin_asn ON announcements(origin_asn);
|
||||||
CREATE INDEX IF NOT EXISTS idx_peerings_as_a ON peerings(as_a);
|
CREATE INDEX IF NOT EXISTS idx_peerings_as_a ON peerings(as_a);
|
||||||
CREATE INDEX IF NOT EXISTS idx_peerings_as_b ON peerings(as_b);
|
CREATE INDEX IF NOT EXISTS idx_peerings_as_b ON peerings(as_b);
|
||||||
CREATE INDEX IF NOT EXISTS idx_peerings_lookup ON peerings(as_a, as_b);
|
CREATE INDEX IF NOT EXISTS idx_peerings_lookup ON peerings(as_a, as_b);
|
||||||
|
|
||||||
-- Additional indexes for prefixes table
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_prefixes_prefix ON prefixes(prefix);
|
|
||||||
|
|
||||||
-- Indexes for asns table
|
-- Indexes for asns table
|
||||||
CREATE INDEX IF NOT EXISTS idx_asns_number ON asns(number);
|
CREATE INDEX IF NOT EXISTS idx_asns_asn ON asns(asn);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_asns_whois_updated_at ON asns(whois_updated_at);
|
||||||
|
|
||||||
-- Indexes for bgp_peers table
|
-- Indexes for bgp_peers table
|
||||||
CREATE INDEX IF NOT EXISTS idx_bgp_peers_asn ON bgp_peers(peer_asn);
|
CREATE INDEX IF NOT EXISTS idx_bgp_peers_asn ON bgp_peers(peer_asn);
|
||||||
CREATE INDEX IF NOT EXISTS idx_bgp_peers_last_seen ON bgp_peers(last_seen);
|
CREATE INDEX IF NOT EXISTS idx_bgp_peers_last_seen ON bgp_peers(last_seen);
|
||||||
CREATE INDEX IF NOT EXISTS idx_bgp_peers_ip ON bgp_peers(peer_ip);
|
CREATE INDEX IF NOT EXISTS idx_bgp_peers_ip ON bgp_peers(peer_ip);
|
||||||
|
|
||||||
-- Live routing table maintained by PrefixHandler
|
-- IPv4 routing table maintained by PrefixHandler
|
||||||
CREATE TABLE IF NOT EXISTS live_routes (
|
CREATE TABLE IF NOT EXISTS live_routes_v4 (
|
||||||
id TEXT PRIMARY KEY,
|
id TEXT PRIMARY KEY,
|
||||||
prefix TEXT NOT NULL,
|
prefix TEXT NOT NULL,
|
||||||
mask_length INTEGER NOT NULL, -- CIDR mask length (0-32 for IPv4, 0-128 for IPv6)
|
mask_length INTEGER NOT NULL, -- CIDR mask length (0-32)
|
||||||
ip_version INTEGER NOT NULL, -- 4 or 6
|
|
||||||
origin_asn INTEGER NOT NULL,
|
origin_asn INTEGER NOT NULL,
|
||||||
peer_ip TEXT NOT NULL,
|
peer_ip TEXT NOT NULL,
|
||||||
as_path TEXT NOT NULL, -- JSON array
|
as_path TEXT NOT NULL, -- JSON array
|
||||||
next_hop TEXT NOT NULL,
|
next_hop TEXT NOT NULL,
|
||||||
last_updated DATETIME NOT NULL,
|
last_updated DATETIME NOT NULL,
|
||||||
-- IPv4 range columns for fast lookups (NULL for IPv6)
|
-- IPv4 range columns for fast lookups
|
||||||
v4_ip_start INTEGER, -- Start of IPv4 range as 32-bit unsigned int
|
ip_start INTEGER NOT NULL, -- Start of IPv4 range as 32-bit unsigned int
|
||||||
v4_ip_end INTEGER, -- End of IPv4 range as 32-bit unsigned int
|
ip_end INTEGER NOT NULL, -- End of IPv4 range as 32-bit unsigned int
|
||||||
UNIQUE(prefix, origin_asn, peer_ip)
|
UNIQUE(prefix, origin_asn, peer_ip)
|
||||||
);
|
);
|
||||||
|
|
||||||
-- Indexes for live_routes table
|
-- IPv6 routing table maintained by PrefixHandler
|
||||||
CREATE INDEX IF NOT EXISTS idx_live_routes_prefix ON live_routes(prefix);
|
CREATE TABLE IF NOT EXISTS live_routes_v6 (
|
||||||
CREATE INDEX IF NOT EXISTS idx_live_routes_mask_length ON live_routes(mask_length);
|
id TEXT PRIMARY KEY,
|
||||||
CREATE INDEX IF NOT EXISTS idx_live_routes_ip_version_mask ON live_routes(ip_version, mask_length);
|
prefix TEXT NOT NULL,
|
||||||
CREATE INDEX IF NOT EXISTS idx_live_routes_last_updated ON live_routes(last_updated);
|
mask_length INTEGER NOT NULL, -- CIDR mask length (0-128)
|
||||||
|
origin_asn INTEGER NOT NULL,
|
||||||
|
peer_ip TEXT NOT NULL,
|
||||||
|
as_path TEXT NOT NULL, -- JSON array
|
||||||
|
next_hop TEXT NOT NULL,
|
||||||
|
last_updated DATETIME NOT NULL,
|
||||||
|
-- Note: IPv6 doesn't use integer range columns
|
||||||
|
UNIQUE(prefix, origin_asn, peer_ip)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Indexes for live_routes_v4 table
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_prefix ON live_routes_v4(prefix);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_length ON live_routes_v4(mask_length);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_origin_asn ON live_routes_v4(origin_asn);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_last_updated ON live_routes_v4(last_updated);
|
||||||
-- Indexes for IPv4 range queries
|
-- Indexes for IPv4 range queries
|
||||||
CREATE INDEX IF NOT EXISTS idx_live_routes_ipv4_range ON live_routes(v4_ip_start, v4_ip_end) WHERE ip_version = 4;
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_ip_range ON live_routes_v4(ip_start, ip_end);
|
||||||
|
-- Index to optimize prefix distribution queries
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v4_mask_prefix ON live_routes_v4(mask_length, prefix);
|
||||||
|
|
||||||
|
-- Indexes for live_routes_v6 table
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_prefix ON live_routes_v6(prefix);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_length ON live_routes_v6(mask_length);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_origin_asn ON live_routes_v6(origin_asn);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_last_updated ON live_routes_v6(last_updated);
|
||||||
|
-- Index to optimize prefix distribution queries
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_live_routes_v6_mask_prefix ON live_routes_v6(mask_length, prefix);
|
||||||
@@ -8,7 +8,7 @@ import (
|
|||||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
const slowQueryThreshold = 50 * time.Millisecond
|
const slowQueryThreshold = 25 * time.Millisecond
|
||||||
|
|
||||||
// logSlowQuery logs queries that take longer than slowQueryThreshold
|
// logSlowQuery logs queries that take longer than slowQueryThreshold
|
||||||
func logSlowQuery(logger *logger.Logger, query string, start time.Time) {
|
func logSlowQuery(logger *logger.Logger, query string, start time.Time) {
|
||||||
@@ -19,6 +19,7 @@ func logSlowQuery(logger *logger.Logger, query string, start time.Time) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// queryRow wraps QueryRow with slow query logging
|
// queryRow wraps QueryRow with slow query logging
|
||||||
|
// nolint:unused // kept for consistency with other query wrappers
|
||||||
func (d *Database) queryRow(query string, args ...interface{}) *sql.Row {
|
func (d *Database) queryRow(query string, args ...interface{}) *sql.Row {
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
defer logSlowQuery(d.logger, query, start)
|
defer logSlowQuery(d.logger, query, start)
|
||||||
|
|||||||
@@ -1,4 +1,8 @@
|
|||||||
// Package logger provides a structured logger with source location tracking
|
// Package logger provides a structured logger with source location tracking.
|
||||||
|
// It wraps the standard library's log/slog package and automatically enriches
|
||||||
|
// log messages with the file name, line number, and function name of the caller.
|
||||||
|
// The output format is automatically selected based on the runtime environment:
|
||||||
|
// human-readable text for terminals, JSON for non-terminal output.
|
||||||
package logger
|
package logger
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -12,17 +16,25 @@ import (
|
|||||||
"golang.org/x/term"
|
"golang.org/x/term"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Logger wraps slog.Logger to add source location information
|
// Logger wraps slog.Logger to add automatic source location information
|
||||||
|
// to all log messages. It embeds slog.Logger and provides the same logging
|
||||||
|
// methods (Debug, Info, Warn, Error) but enriches each message with the
|
||||||
|
// file name, line number, and function name of the caller.
|
||||||
type Logger struct {
|
type Logger struct {
|
||||||
*slog.Logger
|
*slog.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
// AsSlog returns the underlying slog.Logger
|
// AsSlog returns the underlying slog.Logger for use with APIs that require
|
||||||
|
// a standard slog.Logger instance rather than the custom Logger type.
|
||||||
func (l *Logger) AsSlog() *slog.Logger {
|
func (l *Logger) AsSlog() *slog.Logger {
|
||||||
return l.Logger
|
return l.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new logger with appropriate handler based on environment
|
// New creates a new Logger with an appropriate handler based on the runtime
|
||||||
|
// environment. If stdout is a terminal, it uses a human-readable text format;
|
||||||
|
// otherwise, it outputs JSON for structured log aggregation. The log level
|
||||||
|
// defaults to Info, but can be set to Debug by including "routewatch" in the
|
||||||
|
// DEBUG environment variable.
|
||||||
func New() *Logger {
|
func New() *Logger {
|
||||||
level := slog.LevelInfo
|
level := slog.LevelInfo
|
||||||
if debug := os.Getenv("DEBUG"); strings.Contains(debug, "routewatch") {
|
if debug := os.Getenv("DEBUG"); strings.Contains(debug, "routewatch") {
|
||||||
@@ -45,7 +57,10 @@ func New() *Logger {
|
|||||||
return &Logger{Logger: slog.New(handler)}
|
return &Logger{Logger: slog.New(handler)}
|
||||||
}
|
}
|
||||||
|
|
||||||
const sourceSkipLevel = 2 // Skip levels for source location tracking
|
// sourceSkipLevel defines the number of call stack frames to skip when
|
||||||
|
// determining the caller's source location. This accounts for the logger
|
||||||
|
// method itself and the getSourceAttrs helper function.
|
||||||
|
const sourceSkipLevel = 2
|
||||||
|
|
||||||
// getSourceAttrs returns attributes for the calling source location
|
// getSourceAttrs returns attributes for the calling source location
|
||||||
func getSourceAttrs() []slog.Attr {
|
func getSourceAttrs() []slog.Attr {
|
||||||
@@ -75,7 +90,10 @@ func getSourceAttrs() []slog.Attr {
|
|||||||
return attrs
|
return attrs
|
||||||
}
|
}
|
||||||
|
|
||||||
// Debug logs at debug level with source location
|
// Debug logs a message at debug level with automatic source location tracking.
|
||||||
|
// Additional structured attributes can be passed as key-value pairs in args.
|
||||||
|
// Debug messages are only output when the DEBUG environment variable contains
|
||||||
|
// "routewatch".
|
||||||
func (l *Logger) Debug(msg string, args ...any) {
|
func (l *Logger) Debug(msg string, args ...any) {
|
||||||
sourceAttrs := getSourceAttrs()
|
sourceAttrs := getSourceAttrs()
|
||||||
allArgs := make([]any, 0, len(args)+len(sourceAttrs)*2)
|
allArgs := make([]any, 0, len(args)+len(sourceAttrs)*2)
|
||||||
@@ -91,7 +109,8 @@ func (l *Logger) Debug(msg string, args ...any) {
|
|||||||
l.Logger.Debug(msg, allArgs...)
|
l.Logger.Debug(msg, allArgs...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Info logs at info level with source location
|
// Info logs a message at info level with automatic source location tracking.
|
||||||
|
// Additional structured attributes can be passed as key-value pairs in args.
|
||||||
func (l *Logger) Info(msg string, args ...any) {
|
func (l *Logger) Info(msg string, args ...any) {
|
||||||
sourceAttrs := getSourceAttrs()
|
sourceAttrs := getSourceAttrs()
|
||||||
allArgs := make([]any, 0, len(args)+len(sourceAttrs)*2)
|
allArgs := make([]any, 0, len(args)+len(sourceAttrs)*2)
|
||||||
@@ -107,7 +126,8 @@ func (l *Logger) Info(msg string, args ...any) {
|
|||||||
l.Logger.Info(msg, allArgs...)
|
l.Logger.Info(msg, allArgs...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Warn logs at warn level with source location
|
// Warn logs a message at warn level with automatic source location tracking.
|
||||||
|
// Additional structured attributes can be passed as key-value pairs in args.
|
||||||
func (l *Logger) Warn(msg string, args ...any) {
|
func (l *Logger) Warn(msg string, args ...any) {
|
||||||
sourceAttrs := getSourceAttrs()
|
sourceAttrs := getSourceAttrs()
|
||||||
allArgs := make([]any, 0, len(args)+len(sourceAttrs)*2)
|
allArgs := make([]any, 0, len(args)+len(sourceAttrs)*2)
|
||||||
@@ -123,7 +143,8 @@ func (l *Logger) Warn(msg string, args ...any) {
|
|||||||
l.Logger.Warn(msg, allArgs...)
|
l.Logger.Warn(msg, allArgs...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Error logs at error level with source location
|
// Error logs a message at error level with automatic source location tracking.
|
||||||
|
// Additional structured attributes can be passed as key-value pairs in args.
|
||||||
func (l *Logger) Error(msg string, args ...any) {
|
func (l *Logger) Error(msg string, args ...any) {
|
||||||
sourceAttrs := getSourceAttrs()
|
sourceAttrs := getSourceAttrs()
|
||||||
allArgs := make([]any, 0, len(args)+len(sourceAttrs)*2)
|
allArgs := make([]any, 0, len(args)+len(sourceAttrs)*2)
|
||||||
@@ -139,12 +160,16 @@ func (l *Logger) Error(msg string, args ...any) {
|
|||||||
l.Logger.Error(msg, allArgs...)
|
l.Logger.Error(msg, allArgs...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// With returns a new logger with additional attributes
|
// With returns a new Logger with additional structured attributes that will
|
||||||
|
// be included in all subsequent log messages. The args parameter accepts
|
||||||
|
// key-value pairs in the same format as the logging methods.
|
||||||
func (l *Logger) With(args ...any) *Logger {
|
func (l *Logger) With(args ...any) *Logger {
|
||||||
return &Logger{Logger: l.Logger.With(args...)}
|
return &Logger{Logger: l.Logger.With(args...)}
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithGroup returns a new logger with a group prefix
|
// WithGroup returns a new Logger that adds the specified group name as a
|
||||||
|
// prefix to all attribute keys in subsequent log messages. This is useful
|
||||||
|
// for organizing related attributes under a common namespace.
|
||||||
func (l *Logger) WithGroup(name string) *Logger {
|
func (l *Logger) WithGroup(name string) *Logger {
|
||||||
return &Logger{Logger: l.Logger.WithGroup(name)}
|
return &Logger{Logger: l.Logger.WithGroup(name)}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,13 +15,18 @@ type Tracker struct {
|
|||||||
registry metrics.Registry
|
registry metrics.Registry
|
||||||
connectedSince time.Time
|
connectedSince time.Time
|
||||||
isConnected atomic.Bool
|
isConnected atomic.Bool
|
||||||
|
reconnectCount atomic.Uint64
|
||||||
|
|
||||||
// Stream metrics
|
// Stream metrics (decompressed data)
|
||||||
messageCounter metrics.Counter
|
messageCounter metrics.Counter
|
||||||
byteCounter metrics.Counter
|
byteCounter metrics.Counter
|
||||||
messageRate metrics.Meter
|
messageRate metrics.Meter
|
||||||
byteRate metrics.Meter
|
byteRate metrics.Meter
|
||||||
|
|
||||||
|
// Wire bytes metrics (actual bytes on the wire, before decompression)
|
||||||
|
wireByteCounter metrics.Counter
|
||||||
|
wireByteRate metrics.Meter
|
||||||
|
|
||||||
// Route update metrics
|
// Route update metrics
|
||||||
ipv4UpdateRate metrics.Meter
|
ipv4UpdateRate metrics.Meter
|
||||||
ipv6UpdateRate metrics.Meter
|
ipv6UpdateRate metrics.Meter
|
||||||
@@ -37,6 +42,8 @@ func New() *Tracker {
|
|||||||
byteCounter: metrics.NewCounter(),
|
byteCounter: metrics.NewCounter(),
|
||||||
messageRate: metrics.NewMeter(),
|
messageRate: metrics.NewMeter(),
|
||||||
byteRate: metrics.NewMeter(),
|
byteRate: metrics.NewMeter(),
|
||||||
|
wireByteCounter: metrics.NewCounter(),
|
||||||
|
wireByteRate: metrics.NewMeter(),
|
||||||
ipv4UpdateRate: metrics.NewMeter(),
|
ipv4UpdateRate: metrics.NewMeter(),
|
||||||
ipv6UpdateRate: metrics.NewMeter(),
|
ipv6UpdateRate: metrics.NewMeter(),
|
||||||
}
|
}
|
||||||
@@ -44,12 +51,21 @@ func New() *Tracker {
|
|||||||
|
|
||||||
// SetConnected updates the connection status
|
// SetConnected updates the connection status
|
||||||
func (t *Tracker) SetConnected(connected bool) {
|
func (t *Tracker) SetConnected(connected bool) {
|
||||||
t.isConnected.Store(connected)
|
wasConnected := t.isConnected.Swap(connected)
|
||||||
if connected {
|
if connected {
|
||||||
t.mu.Lock()
|
t.mu.Lock()
|
||||||
t.connectedSince = time.Now()
|
t.connectedSince = time.Now()
|
||||||
t.mu.Unlock()
|
t.mu.Unlock()
|
||||||
|
// Increment reconnect count (but not for the initial connection)
|
||||||
|
if wasConnected || t.reconnectCount.Load() > 0 {
|
||||||
|
t.reconnectCount.Add(1)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetReconnectCount returns the number of reconnections since startup
|
||||||
|
func (t *Tracker) GetReconnectCount() uint64 {
|
||||||
|
return t.reconnectCount.Load()
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsConnected returns the current connection status
|
// IsConnected returns the current connection status
|
||||||
@@ -57,7 +73,7 @@ func (t *Tracker) IsConnected() bool {
|
|||||||
return t.isConnected.Load()
|
return t.isConnected.Load()
|
||||||
}
|
}
|
||||||
|
|
||||||
// RecordMessage records a received message and its size
|
// RecordMessage records a received message and its decompressed size
|
||||||
func (t *Tracker) RecordMessage(bytes int64) {
|
func (t *Tracker) RecordMessage(bytes int64) {
|
||||||
t.messageCounter.Inc(1)
|
t.messageCounter.Inc(1)
|
||||||
t.byteCounter.Inc(bytes)
|
t.byteCounter.Inc(bytes)
|
||||||
@@ -65,6 +81,12 @@ func (t *Tracker) RecordMessage(bytes int64) {
|
|||||||
t.byteRate.Mark(bytes)
|
t.byteRate.Mark(bytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RecordWireBytes records actual bytes received on the wire (before decompression)
|
||||||
|
func (t *Tracker) RecordWireBytes(bytes int64) {
|
||||||
|
t.wireByteCounter.Inc(bytes)
|
||||||
|
t.wireByteRate.Mark(bytes)
|
||||||
|
}
|
||||||
|
|
||||||
// GetStreamMetrics returns current streaming metrics
|
// GetStreamMetrics returns current streaming metrics
|
||||||
func (t *Tracker) GetStreamMetrics() StreamMetrics {
|
func (t *Tracker) GetStreamMetrics() StreamMetrics {
|
||||||
t.mu.RLock()
|
t.mu.RLock()
|
||||||
@@ -76,22 +98,29 @@ func (t *Tracker) GetStreamMetrics() StreamMetrics {
|
|||||||
// Safely convert counters to uint64
|
// Safely convert counters to uint64
|
||||||
msgCount := t.messageCounter.Count()
|
msgCount := t.messageCounter.Count()
|
||||||
byteCount := t.byteCounter.Count()
|
byteCount := t.byteCounter.Count()
|
||||||
|
wireByteCount := t.wireByteCounter.Count()
|
||||||
|
|
||||||
var totalMessages, totalBytes uint64
|
var totalMessages, totalBytes, totalWireBytes uint64
|
||||||
if msgCount >= 0 {
|
if msgCount >= 0 {
|
||||||
totalMessages = uint64(msgCount)
|
totalMessages = uint64(msgCount)
|
||||||
}
|
}
|
||||||
if byteCount >= 0 {
|
if byteCount >= 0 {
|
||||||
totalBytes = uint64(byteCount)
|
totalBytes = uint64(byteCount)
|
||||||
}
|
}
|
||||||
|
if wireByteCount >= 0 {
|
||||||
|
totalWireBytes = uint64(wireByteCount)
|
||||||
|
}
|
||||||
|
|
||||||
return StreamMetrics{
|
return StreamMetrics{
|
||||||
TotalMessages: totalMessages,
|
TotalMessages: totalMessages,
|
||||||
TotalBytes: totalBytes,
|
TotalBytes: totalBytes,
|
||||||
|
TotalWireBytes: totalWireBytes,
|
||||||
ConnectedSince: connectedSince,
|
ConnectedSince: connectedSince,
|
||||||
Connected: t.isConnected.Load(),
|
Connected: t.isConnected.Load(),
|
||||||
MessagesPerSec: t.messageRate.Rate1(),
|
MessagesPerSec: t.messageRate.Rate1(),
|
||||||
BitsPerSec: t.byteRate.Rate1() * bitsPerByte,
|
BitsPerSec: t.byteRate.Rate1() * bitsPerByte,
|
||||||
|
WireBitsPerSec: t.wireByteRate.Rate1() * bitsPerByte,
|
||||||
|
ReconnectCount: t.reconnectCount.Load(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -115,16 +144,30 @@ func (t *Tracker) GetRouteMetrics() RouteMetrics {
|
|||||||
|
|
||||||
// StreamMetrics contains streaming statistics
|
// StreamMetrics contains streaming statistics
|
||||||
type StreamMetrics struct {
|
type StreamMetrics struct {
|
||||||
|
// TotalMessages is the total number of messages received since startup
|
||||||
TotalMessages uint64
|
TotalMessages uint64
|
||||||
|
// TotalBytes is the total number of decompressed bytes received since startup
|
||||||
TotalBytes uint64
|
TotalBytes uint64
|
||||||
|
// TotalWireBytes is the total number of bytes received on the wire (before decompression)
|
||||||
|
TotalWireBytes uint64
|
||||||
|
// ConnectedSince is the time when the current connection was established
|
||||||
ConnectedSince time.Time
|
ConnectedSince time.Time
|
||||||
|
// Connected indicates whether the stream is currently connected
|
||||||
Connected bool
|
Connected bool
|
||||||
|
// MessagesPerSec is the rate of messages received per second (1-minute average)
|
||||||
MessagesPerSec float64
|
MessagesPerSec float64
|
||||||
|
// BitsPerSec is the rate of decompressed bits received per second (1-minute average)
|
||||||
BitsPerSec float64
|
BitsPerSec float64
|
||||||
|
// WireBitsPerSec is the rate of bits received on the wire per second (1-minute average)
|
||||||
|
WireBitsPerSec float64
|
||||||
|
// ReconnectCount is the number of reconnections since startup
|
||||||
|
ReconnectCount uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
// RouteMetrics contains route update statistics
|
// RouteMetrics contains route update statistics
|
||||||
type RouteMetrics struct {
|
type RouteMetrics struct {
|
||||||
|
// IPv4UpdatesPerSec is the rate of IPv4 route updates per second (1-minute average)
|
||||||
IPv4UpdatesPerSec float64
|
IPv4UpdatesPerSec float64
|
||||||
|
// IPv6UpdatesPerSec is the rate of IPv6 route updates per second (1-minute average)
|
||||||
IPv6UpdatesPerSec float64
|
IPv6UpdatesPerSec float64
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,10 +6,14 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ASPath represents an AS path that may contain nested AS sets
|
// ASPath represents a BGP AS path as a slice of AS numbers.
|
||||||
|
// It handles JSON unmarshaling of both simple arrays and nested AS sets,
|
||||||
|
// flattening any nested structures into a single sequence of AS numbers.
|
||||||
type ASPath []int
|
type ASPath []int
|
||||||
|
|
||||||
// UnmarshalJSON implements custom JSON unmarshaling to flatten nested arrays
|
// UnmarshalJSON implements the json.Unmarshaler interface for ASPath.
|
||||||
|
// It handles both simple integer arrays [1, 2, 3] and nested AS sets
|
||||||
|
// like [1, [2, 3], 4], flattening them into a single slice of integers.
|
||||||
func (p *ASPath) UnmarshalJSON(data []byte) error {
|
func (p *ASPath) UnmarshalJSON(data []byte) error {
|
||||||
// First try to unmarshal as a simple array of integers
|
// First try to unmarshal as a simple array of integers
|
||||||
var simple []int
|
var simple []int
|
||||||
@@ -46,13 +50,18 @@ func (p *ASPath) UnmarshalJSON(data []byte) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// RISLiveMessage represents the outer wrapper from the RIS Live stream
|
// RISLiveMessage represents the outer wrapper message from the RIPE RIS Live stream.
|
||||||
|
// Each message contains a Type field indicating the message type and a Data field
|
||||||
|
// containing the actual BGP message payload.
|
||||||
type RISLiveMessage struct {
|
type RISLiveMessage struct {
|
||||||
Type string `json:"type"`
|
Type string `json:"type"`
|
||||||
Data RISMessage `json:"data"`
|
Data RISMessage `json:"data"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// RISMessage represents a message from the RIS Live stream
|
// RISMessage represents a BGP update message from the RIPE RIS Live stream.
|
||||||
|
// It contains metadata about the BGP session (peer, ASN, host) along with
|
||||||
|
// the actual BGP update data including AS path, communities, announcements,
|
||||||
|
// and withdrawals.
|
||||||
type RISMessage struct {
|
type RISMessage struct {
|
||||||
Type string `json:"type"`
|
Type string `json:"type"`
|
||||||
Timestamp float64 `json:"timestamp"`
|
Timestamp float64 `json:"timestamp"`
|
||||||
@@ -74,7 +83,9 @@ type RISMessage struct {
|
|||||||
Raw string `json:"raw,omitempty"`
|
Raw string `json:"raw,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// RISAnnouncement represents announcement data within a RIS message
|
// RISAnnouncement represents a BGP route announcement within a RIS message.
|
||||||
|
// It contains the next hop IP address and the list of prefixes being announced
|
||||||
|
// via that next hop.
|
||||||
type RISAnnouncement struct {
|
type RISAnnouncement struct {
|
||||||
NextHop string `json:"next_hop"`
|
NextHop string `json:"next_hop"`
|
||||||
Prefixes []string `json:"prefixes"`
|
Prefixes []string `json:"prefixes"`
|
||||||
|
|||||||
@@ -43,6 +43,8 @@ type RouteWatch struct {
|
|||||||
peerHandler *PeerHandler
|
peerHandler *PeerHandler
|
||||||
prefixHandler *PrefixHandler
|
prefixHandler *PrefixHandler
|
||||||
peeringHandler *PeeringHandler
|
peeringHandler *PeeringHandler
|
||||||
|
asnFetcher *ASNFetcher
|
||||||
|
dbMaintainer *DBMaintainer
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new RouteWatch instance
|
// New creates a new RouteWatch instance
|
||||||
@@ -109,6 +111,15 @@ func (rw *RouteWatch) Run(ctx context.Context) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Start ASN WHOIS fetcher for background updates
|
||||||
|
rw.asnFetcher = NewASNFetcher(rw.db, rw.logger.Logger)
|
||||||
|
rw.asnFetcher.Start()
|
||||||
|
rw.server.SetASNFetcher(rw.asnFetcher)
|
||||||
|
|
||||||
|
// Start database maintenance goroutine
|
||||||
|
rw.dbMaintainer = NewDBMaintainer(rw.db, rw.logger.Logger)
|
||||||
|
rw.dbMaintainer.Start()
|
||||||
|
|
||||||
// Wait for context cancellation
|
// Wait for context cancellation
|
||||||
<-ctx.Done()
|
<-ctx.Done()
|
||||||
|
|
||||||
@@ -144,6 +155,16 @@ func (rw *RouteWatch) Shutdown() {
|
|||||||
rw.peeringHandler.Stop()
|
rw.peeringHandler.Stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Stop ASN WHOIS fetcher
|
||||||
|
if rw.asnFetcher != nil {
|
||||||
|
rw.asnFetcher.Stop()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop database maintainer
|
||||||
|
if rw.dbMaintainer != nil {
|
||||||
|
rw.dbMaintainer.Stop()
|
||||||
|
}
|
||||||
|
|
||||||
// Stop services
|
// Stop services
|
||||||
rw.streamer.Stop()
|
rw.streamer.Stop()
|
||||||
|
|
||||||
|
|||||||
@@ -61,8 +61,7 @@ func (m *mockStore) GetOrCreateASN(number int, timestamp time.Time) (*database.A
|
|||||||
}
|
}
|
||||||
|
|
||||||
asn := &database.ASN{
|
asn := &database.ASN{
|
||||||
ID: uuid.New(),
|
ASN: number,
|
||||||
Number: number,
|
|
||||||
FirstSeen: timestamp,
|
FirstSeen: timestamp,
|
||||||
LastSeen: timestamp,
|
LastSeen: timestamp,
|
||||||
}
|
}
|
||||||
@@ -72,6 +71,37 @@ func (m *mockStore) GetOrCreateASN(number int, timestamp time.Time) (*database.A
|
|||||||
return asn, nil
|
return asn, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// UpdatePrefixesBatch mock implementation
|
||||||
|
func (m *mockStore) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
|
||||||
|
for prefix, timestamp := range prefixes {
|
||||||
|
if p, exists := m.Prefixes[prefix]; exists {
|
||||||
|
p.LastSeen = timestamp
|
||||||
|
} else {
|
||||||
|
const (
|
||||||
|
ipVersionV4 = 4
|
||||||
|
ipVersionV6 = 6
|
||||||
|
)
|
||||||
|
|
||||||
|
ipVersion := ipVersionV4
|
||||||
|
if strings.Contains(prefix, ":") {
|
||||||
|
ipVersion = ipVersionV6
|
||||||
|
}
|
||||||
|
|
||||||
|
m.Prefixes[prefix] = &database.Prefix{
|
||||||
|
ID: uuid.New(),
|
||||||
|
Prefix: prefix,
|
||||||
|
IPVersion: ipVersion,
|
||||||
|
FirstSeen: timestamp,
|
||||||
|
LastSeen: timestamp,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// GetOrCreatePrefix mock implementation
|
// GetOrCreatePrefix mock implementation
|
||||||
func (m *mockStore) GetOrCreatePrefix(prefix string, timestamp time.Time) (*database.Prefix, error) {
|
func (m *mockStore) GetOrCreatePrefix(prefix string, timestamp time.Time) (*database.Prefix, error) {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
@@ -163,6 +193,11 @@ func (m *mockStore) GetStats() (database.Stats, error) {
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetStatsContext returns statistics about the mock store with context support
|
||||||
|
func (m *mockStore) GetStatsContext(ctx context.Context) (database.Stats, error) {
|
||||||
|
return m.GetStats()
|
||||||
|
}
|
||||||
|
|
||||||
// UpsertLiveRoute mock implementation
|
// UpsertLiveRoute mock implementation
|
||||||
func (m *mockStore) UpsertLiveRoute(route *database.LiveRoute) error {
|
func (m *mockStore) UpsertLiveRoute(route *database.LiveRoute) error {
|
||||||
// Simple mock - just return nil
|
// Simple mock - just return nil
|
||||||
@@ -181,12 +216,22 @@ func (m *mockStore) GetPrefixDistribution() (ipv4 []database.PrefixDistribution,
|
|||||||
return nil, nil, nil
|
return nil, nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetPrefixDistributionContext mock implementation with context support
|
||||||
|
func (m *mockStore) GetPrefixDistributionContext(ctx context.Context) (ipv4 []database.PrefixDistribution, ipv6 []database.PrefixDistribution, err error) {
|
||||||
|
return m.GetPrefixDistribution()
|
||||||
|
}
|
||||||
|
|
||||||
// GetLiveRouteCounts mock implementation
|
// GetLiveRouteCounts mock implementation
|
||||||
func (m *mockStore) GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error) {
|
func (m *mockStore) GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error) {
|
||||||
// Return mock counts
|
// Return mock counts
|
||||||
return m.RouteCount / 2, m.RouteCount / 2, nil
|
return m.RouteCount / 2, m.RouteCount / 2, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetLiveRouteCountsContext mock implementation with context support
|
||||||
|
func (m *mockStore) GetLiveRouteCountsContext(ctx context.Context) (ipv4Count, ipv6Count int, err error) {
|
||||||
|
return m.GetLiveRouteCounts()
|
||||||
|
}
|
||||||
|
|
||||||
// GetASInfoForIP mock implementation
|
// GetASInfoForIP mock implementation
|
||||||
func (m *mockStore) GetASInfoForIP(ip string) (*database.ASInfo, error) {
|
func (m *mockStore) GetASInfoForIP(ip string) (*database.ASInfo, error) {
|
||||||
// Simple mock - return a test AS
|
// Simple mock - return a test AS
|
||||||
@@ -201,6 +246,11 @@ func (m *mockStore) GetASInfoForIP(ip string) (*database.ASInfo, error) {
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetASInfoForIPContext mock implementation with context support
|
||||||
|
func (m *mockStore) GetASInfoForIPContext(ctx context.Context, ip string) (*database.ASInfo, error) {
|
||||||
|
return m.GetASInfoForIP(ip)
|
||||||
|
}
|
||||||
|
|
||||||
// GetASDetails mock implementation
|
// GetASDetails mock implementation
|
||||||
func (m *mockStore) GetASDetails(asn int) (*database.ASN, []database.LiveRoute, error) {
|
func (m *mockStore) GetASDetails(asn int) (*database.ASN, []database.LiveRoute, error) {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
@@ -215,12 +265,89 @@ func (m *mockStore) GetASDetails(asn int) (*database.ASN, []database.LiveRoute,
|
|||||||
return nil, nil, database.ErrNoRoute
|
return nil, nil, database.ErrNoRoute
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetASDetailsContext mock implementation with context support
|
||||||
|
func (m *mockStore) GetASDetailsContext(ctx context.Context, asn int) (*database.ASN, []database.LiveRoute, error) {
|
||||||
|
return m.GetASDetails(asn)
|
||||||
|
}
|
||||||
|
|
||||||
// GetPrefixDetails mock implementation
|
// GetPrefixDetails mock implementation
|
||||||
func (m *mockStore) GetPrefixDetails(prefix string) ([]database.LiveRoute, error) {
|
func (m *mockStore) GetPrefixDetails(prefix string) ([]database.LiveRoute, error) {
|
||||||
// Return empty routes for now
|
// Return empty routes for now
|
||||||
return []database.LiveRoute{}, nil
|
return []database.LiveRoute{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetPrefixDetailsContext mock implementation with context support
|
||||||
|
func (m *mockStore) GetPrefixDetailsContext(ctx context.Context, prefix string) ([]database.LiveRoute, error) {
|
||||||
|
return m.GetPrefixDetails(prefix)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockStore) GetRandomPrefixesByLength(maskLength, ipVersion, limit int) ([]database.LiveRoute, error) {
|
||||||
|
// Return empty routes for now
|
||||||
|
return []database.LiveRoute{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetRandomPrefixesByLengthContext mock implementation with context support
|
||||||
|
func (m *mockStore) GetRandomPrefixesByLengthContext(ctx context.Context, maskLength, ipVersion, limit int) ([]database.LiveRoute, error) {
|
||||||
|
return m.GetRandomPrefixesByLength(maskLength, ipVersion, limit)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetASPeers mock implementation
|
||||||
|
func (m *mockStore) GetASPeers(asn int) ([]database.ASPeer, error) {
|
||||||
|
// Return empty peers for now
|
||||||
|
return []database.ASPeer{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetASPeersContext mock implementation with context support
|
||||||
|
func (m *mockStore) GetASPeersContext(ctx context.Context, asn int) ([]database.ASPeer, error) {
|
||||||
|
return m.GetASPeers(asn)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetIPInfo mock implementation
|
||||||
|
func (m *mockStore) GetIPInfo(ip string) (*database.IPInfo, error) {
|
||||||
|
return m.GetIPInfoContext(context.Background(), ip)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetIPInfoContext mock implementation with context support
|
||||||
|
func (m *mockStore) GetIPInfoContext(ctx context.Context, ip string) (*database.IPInfo, error) {
|
||||||
|
now := time.Now()
|
||||||
|
return &database.IPInfo{
|
||||||
|
IP: ip,
|
||||||
|
Netblock: "8.8.8.0/24",
|
||||||
|
MaskLength: 24,
|
||||||
|
IPVersion: 4,
|
||||||
|
NumPeers: 3,
|
||||||
|
ASN: 15169,
|
||||||
|
Handle: "GOOGLE",
|
||||||
|
Description: "Google LLC",
|
||||||
|
CountryCode: "US",
|
||||||
|
FirstSeen: now.Add(-24 * time.Hour),
|
||||||
|
LastSeen: now,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetNextStaleASN mock implementation
|
||||||
|
func (m *mockStore) GetNextStaleASN(ctx context.Context, staleThreshold time.Duration) (int, error) {
|
||||||
|
return 0, database.ErrNoStaleASN
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdateASNWHOIS mock implementation
|
||||||
|
func (m *mockStore) UpdateASNWHOIS(ctx context.Context, update *database.ASNWHOISUpdate) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetWHOISStats mock implementation
|
||||||
|
func (m *mockStore) GetWHOISStats(ctx context.Context, staleThreshold time.Duration) (*database.WHOISStats, error) {
|
||||||
|
m.mu.Lock()
|
||||||
|
defer m.mu.Unlock()
|
||||||
|
|
||||||
|
return &database.WHOISStats{
|
||||||
|
TotalASNs: len(m.ASNs),
|
||||||
|
FreshASNs: 0,
|
||||||
|
StaleASNs: 0,
|
||||||
|
NeverFetched: len(m.ASNs),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
// UpsertLiveRouteBatch mock implementation
|
// UpsertLiveRouteBatch mock implementation
|
||||||
func (m *mockStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) error {
|
func (m *mockStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) error {
|
||||||
m.mu.Lock()
|
m.mu.Lock()
|
||||||
@@ -262,8 +389,7 @@ func (m *mockStore) GetOrCreateASNBatch(asns map[int]time.Time) error {
|
|||||||
for number, timestamp := range asns {
|
for number, timestamp := range asns {
|
||||||
if _, exists := m.ASNs[number]; !exists {
|
if _, exists := m.ASNs[number]; !exists {
|
||||||
m.ASNs[number] = &database.ASN{
|
m.ASNs[number] = &database.ASN{
|
||||||
ID: uuid.New(),
|
ASN: number,
|
||||||
Number: number,
|
|
||||||
FirstSeen: timestamp,
|
FirstSeen: timestamp,
|
||||||
LastSeen: timestamp,
|
LastSeen: timestamp,
|
||||||
}
|
}
|
||||||
@@ -279,6 +405,16 @@ func (m *mockStore) UpdatePeerBatch(peers map[string]database.PeerUpdate) error
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Vacuum mock implementation
|
||||||
|
func (m *mockStore) Vacuum(ctx context.Context) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Analyze mock implementation
|
||||||
|
func (m *mockStore) Analyze(ctx context.Context) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func TestRouteWatchLiveFeed(t *testing.T) {
|
func TestRouteWatchLiveFeed(t *testing.T) {
|
||||||
|
|
||||||
// Create mock database
|
// Create mock database
|
||||||
|
|||||||
@@ -11,16 +11,21 @@ import (
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
// asHandlerQueueSize is the queue capacity for ASN operations
|
// asHandlerQueueSize is the queue capacity for ASN operations
|
||||||
|
// DO NOT set this higher than 100000 without explicit instructions
|
||||||
asHandlerQueueSize = 100000
|
asHandlerQueueSize = 100000
|
||||||
|
|
||||||
// asnBatchSize is the number of ASN operations to batch together
|
// asnBatchSize is the number of ASN operations to batch together
|
||||||
asnBatchSize = 10000
|
asnBatchSize = 30000
|
||||||
|
|
||||||
// asnBatchTimeout is the maximum time to wait before flushing a batch
|
// asnBatchTimeout is the maximum time to wait before flushing a batch
|
||||||
|
// DO NOT reduce this timeout - larger batches are more efficient
|
||||||
asnBatchTimeout = 2 * time.Second
|
asnBatchTimeout = 2 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
// ASHandler handles ASN information from BGP messages using batched operations
|
// ASHandler processes Autonomous System Number (ASN) information extracted from
|
||||||
|
// BGP UPDATE messages. It uses batched database operations to efficiently store
|
||||||
|
// ASN data, collecting operations into batches that are flushed either when the
|
||||||
|
// batch reaches a size threshold or after a timeout period.
|
||||||
type ASHandler struct {
|
type ASHandler struct {
|
||||||
db database.Store
|
db database.Store
|
||||||
logger *logger.Logger
|
logger *logger.Logger
|
||||||
@@ -38,7 +43,11 @@ type asnOp struct {
|
|||||||
timestamp time.Time
|
timestamp time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewASHandler creates a new batched ASN handler
|
// NewASHandler creates and returns a new ASHandler instance. It initializes
|
||||||
|
// the batching system and starts a background goroutine that periodically
|
||||||
|
// flushes accumulated ASN operations to the database. The caller must call
|
||||||
|
// Stop when finished to ensure all pending operations are flushed and the
|
||||||
|
// background goroutine is terminated.
|
||||||
func NewASHandler(db database.Store, logger *logger.Logger) *ASHandler {
|
func NewASHandler(db database.Store, logger *logger.Logger) *ASHandler {
|
||||||
h := &ASHandler{
|
h := &ASHandler{
|
||||||
db: db,
|
db: db,
|
||||||
@@ -55,19 +64,27 @@ func NewASHandler(db database.Store, logger *logger.Logger) *ASHandler {
|
|||||||
return h
|
return h
|
||||||
}
|
}
|
||||||
|
|
||||||
// WantsMessage returns true if this handler wants to process messages of the given type
|
// WantsMessage reports whether this handler should process messages of the
|
||||||
|
// given type. ASHandler only processes "UPDATE" messages, as these contain
|
||||||
|
// the AS path information needed to track autonomous systems.
|
||||||
func (h *ASHandler) WantsMessage(messageType string) bool {
|
func (h *ASHandler) WantsMessage(messageType string) bool {
|
||||||
// We only care about UPDATE messages for the database
|
// We only care about UPDATE messages for the database
|
||||||
return messageType == "UPDATE"
|
return messageType == "UPDATE"
|
||||||
}
|
}
|
||||||
|
|
||||||
// QueueCapacity returns the desired queue capacity for this handler
|
// QueueCapacity returns the recommended message queue size for this handler.
|
||||||
|
// ASHandler uses a large queue capacity to accommodate high-volume BGP streams,
|
||||||
|
// as the batching mechanism allows efficient processing of accumulated messages.
|
||||||
func (h *ASHandler) QueueCapacity() int {
|
func (h *ASHandler) QueueCapacity() int {
|
||||||
// Batching allows us to use a larger queue
|
// Batching allows us to use a larger queue
|
||||||
return asHandlerQueueSize
|
return asHandlerQueueSize
|
||||||
}
|
}
|
||||||
|
|
||||||
// HandleMessage processes a RIS message and queues database operations
|
// HandleMessage processes a RIS Live BGP message by extracting all ASNs from
|
||||||
|
// the AS path and queuing them for batch insertion into the database. The
|
||||||
|
// origin ASN (last element in the path) and all transit ASNs are recorded
|
||||||
|
// with their associated timestamps. The batch is automatically flushed when
|
||||||
|
// it reaches the configured size threshold.
|
||||||
func (h *ASHandler) HandleMessage(msg *ristypes.RISMessage) {
|
func (h *ASHandler) HandleMessage(msg *ristypes.RISMessage) {
|
||||||
// Use the pre-parsed timestamp
|
// Use the pre-parsed timestamp
|
||||||
timestamp := msg.ParsedTimestamp
|
timestamp := msg.ParsedTimestamp
|
||||||
@@ -154,7 +171,11 @@ func (h *ASHandler) flushBatchLocked() {
|
|||||||
h.lastFlush = time.Now()
|
h.lastFlush = time.Now()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop gracefully stops the handler and flushes remaining batches
|
// Stop gracefully shuts down the ASHandler by signaling the background flush
|
||||||
|
// goroutine to terminate and waiting for it to complete. Any pending ASN
|
||||||
|
// operations in the current batch are flushed to the database before Stop
|
||||||
|
// returns. This method should be called during application shutdown to ensure
|
||||||
|
// no data is lost.
|
||||||
func (h *ASHandler) Stop() {
|
func (h *ASHandler) Stop() {
|
||||||
close(h.stopCh)
|
close(h.stopCh)
|
||||||
h.wg.Wait()
|
h.wg.Wait()
|
||||||
|
|||||||
325
internal/routewatch/asnfetcher.go
Normal file
325
internal/routewatch/asnfetcher.go
Normal file
@@ -0,0 +1,325 @@
|
|||||||
|
// Package routewatch contains the ASN WHOIS fetcher for background updates.
|
||||||
|
package routewatch
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/database"
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/server"
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/whois"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ASN fetcher configuration constants.
|
||||||
|
const (
|
||||||
|
// baseInterval is the starting interval between fetch attempts.
|
||||||
|
baseInterval = 15 * time.Second
|
||||||
|
|
||||||
|
// minInterval is the minimum interval after successes (rate limit).
|
||||||
|
minInterval = 1 * time.Second
|
||||||
|
|
||||||
|
// maxInterval is the maximum interval after failures (backoff cap).
|
||||||
|
maxInterval = 5 * time.Minute
|
||||||
|
|
||||||
|
// backoffMultiplier is how much to multiply interval on failure.
|
||||||
|
backoffMultiplier = 2
|
||||||
|
|
||||||
|
// whoisStaleThreshold is how old WHOIS data can be before refresh.
|
||||||
|
whoisStaleThreshold = 30 * 24 * time.Hour // 30 days
|
||||||
|
|
||||||
|
// immediateQueueSize is the buffer size for immediate fetch requests.
|
||||||
|
immediateQueueSize = 100
|
||||||
|
|
||||||
|
// statsWindow is how long to keep stats for.
|
||||||
|
statsWindow = time.Hour
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
// ASNFetcher handles background WHOIS lookups for ASNs.
|
||||||
|
type ASNFetcher struct {
|
||||||
|
db database.Store
|
||||||
|
whoisClient *whois.Client
|
||||||
|
logger *slog.Logger
|
||||||
|
immediateQueue chan int
|
||||||
|
stopCh chan struct{}
|
||||||
|
wg sync.WaitGroup
|
||||||
|
|
||||||
|
// fetchMu ensures only one fetch runs at a time
|
||||||
|
fetchMu sync.Mutex
|
||||||
|
|
||||||
|
// interval tracking with mutex protection
|
||||||
|
intervalMu sync.Mutex
|
||||||
|
currentInterval time.Duration
|
||||||
|
consecutiveFails int
|
||||||
|
|
||||||
|
// hourly stats tracking
|
||||||
|
statsMu sync.Mutex
|
||||||
|
successTimes []time.Time
|
||||||
|
errorTimes []time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewASNFetcher creates a new ASN fetcher.
|
||||||
|
func NewASNFetcher(db database.Store, logger *slog.Logger) *ASNFetcher {
|
||||||
|
return &ASNFetcher{
|
||||||
|
db: db,
|
||||||
|
whoisClient: whois.NewClient(),
|
||||||
|
logger: logger.With("component", "asn_fetcher"),
|
||||||
|
immediateQueue: make(chan int, immediateQueueSize),
|
||||||
|
stopCh: make(chan struct{}),
|
||||||
|
currentInterval: baseInterval,
|
||||||
|
successTimes: make([]time.Time, 0),
|
||||||
|
errorTimes: make([]time.Time, 0),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start begins the background ASN fetcher goroutine.
|
||||||
|
func (f *ASNFetcher) Start() {
|
||||||
|
f.wg.Add(1)
|
||||||
|
go f.run()
|
||||||
|
f.logger.Info("ASN fetcher started",
|
||||||
|
"base_interval", baseInterval,
|
||||||
|
"min_interval", minInterval,
|
||||||
|
"max_interval", maxInterval,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop gracefully shuts down the fetcher.
|
||||||
|
func (f *ASNFetcher) Stop() {
|
||||||
|
close(f.stopCh)
|
||||||
|
f.wg.Wait()
|
||||||
|
f.logger.Info("ASN fetcher stopped")
|
||||||
|
}
|
||||||
|
|
||||||
|
// QueueImmediate queues an ASN for immediate WHOIS lookup.
|
||||||
|
// Non-blocking - if queue is full, the request is dropped.
|
||||||
|
func (f *ASNFetcher) QueueImmediate(asn int) {
|
||||||
|
select {
|
||||||
|
case f.immediateQueue <- asn:
|
||||||
|
f.logger.Debug("Queued immediate WHOIS lookup", "asn", asn)
|
||||||
|
default:
|
||||||
|
f.logger.Debug("Immediate queue full, dropping request", "asn", asn)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetStats returns statistics about fetcher activity.
|
||||||
|
func (f *ASNFetcher) GetStats() server.ASNFetcherStats {
|
||||||
|
f.statsMu.Lock()
|
||||||
|
defer f.statsMu.Unlock()
|
||||||
|
|
||||||
|
f.intervalMu.Lock()
|
||||||
|
interval := f.currentInterval
|
||||||
|
fails := f.consecutiveFails
|
||||||
|
f.intervalMu.Unlock()
|
||||||
|
|
||||||
|
// Prune old entries and count
|
||||||
|
cutoff := time.Now().Add(-statsWindow)
|
||||||
|
f.successTimes = pruneOldTimes(f.successTimes, cutoff)
|
||||||
|
f.errorTimes = pruneOldTimes(f.errorTimes, cutoff)
|
||||||
|
|
||||||
|
return server.ASNFetcherStats{
|
||||||
|
SuccessesLastHour: len(f.successTimes),
|
||||||
|
ErrorsLastHour: len(f.errorTimes),
|
||||||
|
CurrentInterval: interval,
|
||||||
|
ConsecutiveFails: fails,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// pruneOldTimes removes times older than cutoff and returns the pruned slice.
|
||||||
|
func pruneOldTimes(times []time.Time, cutoff time.Time) []time.Time {
|
||||||
|
result := make([]time.Time, 0, len(times))
|
||||||
|
for _, t := range times {
|
||||||
|
if t.After(cutoff) {
|
||||||
|
result = append(result, t)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// getInterval returns the current fetch interval.
|
||||||
|
func (f *ASNFetcher) getInterval() time.Duration {
|
||||||
|
f.intervalMu.Lock()
|
||||||
|
defer f.intervalMu.Unlock()
|
||||||
|
|
||||||
|
return f.currentInterval
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordSuccess decreases the interval on successful fetch.
|
||||||
|
func (f *ASNFetcher) recordSuccess() {
|
||||||
|
f.intervalMu.Lock()
|
||||||
|
f.consecutiveFails = 0
|
||||||
|
|
||||||
|
// Decrease interval by half, but not below minimum
|
||||||
|
newInterval := f.currentInterval / backoffMultiplier
|
||||||
|
if newInterval < minInterval {
|
||||||
|
newInterval = minInterval
|
||||||
|
}
|
||||||
|
|
||||||
|
if newInterval != f.currentInterval {
|
||||||
|
f.logger.Debug("Decreased fetch interval",
|
||||||
|
"old_interval", f.currentInterval,
|
||||||
|
"new_interval", newInterval,
|
||||||
|
)
|
||||||
|
f.currentInterval = newInterval
|
||||||
|
}
|
||||||
|
f.intervalMu.Unlock()
|
||||||
|
|
||||||
|
// Record success time for stats
|
||||||
|
f.statsMu.Lock()
|
||||||
|
f.successTimes = append(f.successTimes, time.Now())
|
||||||
|
f.statsMu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// recordFailure increases the interval on failed fetch using exponential backoff.
|
||||||
|
func (f *ASNFetcher) recordFailure() {
|
||||||
|
f.intervalMu.Lock()
|
||||||
|
f.consecutiveFails++
|
||||||
|
|
||||||
|
// Exponential backoff: multiply by 2, capped at max
|
||||||
|
newInterval := f.currentInterval * backoffMultiplier
|
||||||
|
if newInterval > maxInterval {
|
||||||
|
newInterval = maxInterval
|
||||||
|
}
|
||||||
|
|
||||||
|
if newInterval != f.currentInterval {
|
||||||
|
f.logger.Debug("Increased fetch interval due to failure",
|
||||||
|
"old_interval", f.currentInterval,
|
||||||
|
"new_interval", newInterval,
|
||||||
|
"consecutive_failures", f.consecutiveFails,
|
||||||
|
)
|
||||||
|
f.currentInterval = newInterval
|
||||||
|
}
|
||||||
|
f.intervalMu.Unlock()
|
||||||
|
|
||||||
|
// Record error time for stats
|
||||||
|
f.statsMu.Lock()
|
||||||
|
f.errorTimes = append(f.errorTimes, time.Now())
|
||||||
|
f.statsMu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// run is the main background loop.
|
||||||
|
func (f *ASNFetcher) run() {
|
||||||
|
defer f.wg.Done()
|
||||||
|
|
||||||
|
timer := time.NewTimer(f.getInterval())
|
||||||
|
defer timer.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-f.stopCh:
|
||||||
|
return
|
||||||
|
|
||||||
|
case asn := <-f.immediateQueue:
|
||||||
|
// Process immediate request (respects lock)
|
||||||
|
f.tryFetch(asn)
|
||||||
|
// Reset timer after immediate fetch
|
||||||
|
timer.Reset(f.getInterval())
|
||||||
|
|
||||||
|
case <-timer.C:
|
||||||
|
// Background fetch of stale/missing ASN
|
||||||
|
f.fetchNextStale()
|
||||||
|
// Reset timer with potentially updated interval
|
||||||
|
timer.Reset(f.getInterval())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// tryFetch attempts to fetch and update an ASN, respecting the fetch lock.
|
||||||
|
// Returns true if fetch was successful.
|
||||||
|
func (f *ASNFetcher) tryFetch(asn int) bool {
|
||||||
|
// Try to acquire lock, skip if another fetch is running
|
||||||
|
if !f.fetchMu.TryLock() {
|
||||||
|
f.logger.Debug("Skipping fetch, another fetch in progress", "asn", asn)
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
defer f.fetchMu.Unlock()
|
||||||
|
|
||||||
|
return f.fetchAndUpdate(asn)
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchNextStale finds and fetches the next ASN needing WHOIS data.
|
||||||
|
func (f *ASNFetcher) fetchNextStale() {
|
||||||
|
// Try to acquire lock, skip if another fetch is running
|
||||||
|
if !f.fetchMu.TryLock() {
|
||||||
|
f.logger.Debug("Skipping stale fetch, another fetch in progress")
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer f.fetchMu.Unlock()
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
asn, err := f.db.GetNextStaleASN(ctx, whoisStaleThreshold)
|
||||||
|
if err != nil {
|
||||||
|
if err != database.ErrNoStaleASN {
|
||||||
|
f.logger.Error("Failed to get stale ASN", "error", err)
|
||||||
|
f.recordFailure()
|
||||||
|
}
|
||||||
|
// No stale ASN is not a failure, just nothing to do
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
f.fetchAndUpdate(asn)
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchAndUpdate performs a WHOIS lookup and updates the database.
|
||||||
|
// Returns true if successful.
|
||||||
|
func (f *ASNFetcher) fetchAndUpdate(asn int) bool {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
f.logger.Info("Fetching WHOIS data", "asn", asn)
|
||||||
|
|
||||||
|
info, err := f.whoisClient.LookupASN(ctx, asn)
|
||||||
|
if err != nil {
|
||||||
|
f.logger.Error("WHOIS lookup failed", "asn", asn, "error", err)
|
||||||
|
f.recordFailure()
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update database with WHOIS data
|
||||||
|
err = f.db.UpdateASNWHOIS(ctx, &database.ASNWHOISUpdate{
|
||||||
|
ASN: asn,
|
||||||
|
ASName: info.ASName,
|
||||||
|
OrgName: info.OrgName,
|
||||||
|
OrgID: info.OrgID,
|
||||||
|
Address: info.Address,
|
||||||
|
CountryCode: info.CountryCode,
|
||||||
|
AbuseEmail: info.AbuseEmail,
|
||||||
|
AbusePhone: info.AbusePhone,
|
||||||
|
TechEmail: info.TechEmail,
|
||||||
|
TechPhone: info.TechPhone,
|
||||||
|
RIR: info.RIR,
|
||||||
|
RIRRegDate: info.RegDate,
|
||||||
|
RIRLastMod: info.LastMod,
|
||||||
|
WHOISRaw: info.RawResponse,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
f.logger.Error("Failed to update ASN WHOIS data", "asn", asn, "error", err)
|
||||||
|
f.recordFailure()
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
f.recordSuccess()
|
||||||
|
f.logger.Info("Updated ASN WHOIS data",
|
||||||
|
"asn", asn,
|
||||||
|
"org_name", info.OrgName,
|
||||||
|
"country", info.CountryCode,
|
||||||
|
"rir", info.RIR,
|
||||||
|
"next_interval", f.getInterval(),
|
||||||
|
)
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetStaleThreshold returns the WHOIS stale threshold duration.
|
||||||
|
func GetStaleThreshold() time.Duration {
|
||||||
|
return whoisStaleThreshold
|
||||||
|
}
|
||||||
@@ -53,7 +53,11 @@ func logDebugStats(logger *logger.Logger) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// CLIEntry is the main entry point for the CLI
|
// CLIEntry is the main entry point for the routewatch command-line interface.
|
||||||
|
// It initializes the application using the fx dependency injection framework,
|
||||||
|
// sets up signal handling for graceful shutdown, and starts the RouteWatch service.
|
||||||
|
// This function blocks until the application receives a shutdown signal or encounters
|
||||||
|
// a fatal error.
|
||||||
func CLIEntry() {
|
func CLIEntry() {
|
||||||
app := fx.New(
|
app := fx.New(
|
||||||
getModule(),
|
getModule(),
|
||||||
|
|||||||
147
internal/routewatch/dbmaintainer.go
Normal file
147
internal/routewatch/dbmaintainer.go
Normal file
@@ -0,0 +1,147 @@
|
|||||||
|
// Package routewatch contains the database maintainer for background maintenance tasks.
|
||||||
|
package routewatch
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.eeqj.de/sneak/routewatch/internal/database"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Database maintenance configuration constants.
|
||||||
|
const (
|
||||||
|
// vacuumInterval is how often to run incremental vacuum.
|
||||||
|
// Since incremental vacuum only frees ~1000 pages (~4MB) per run,
|
||||||
|
// we run it frequently to keep up with deletions.
|
||||||
|
vacuumInterval = 10 * time.Minute
|
||||||
|
|
||||||
|
// analyzeInterval is how often to run ANALYZE.
|
||||||
|
analyzeInterval = 1 * time.Hour
|
||||||
|
|
||||||
|
// vacuumTimeout is the max time for incremental vacuum (should be quick).
|
||||||
|
vacuumTimeout = 30 * time.Second
|
||||||
|
|
||||||
|
// analyzeTimeout is the max time for ANALYZE.
|
||||||
|
analyzeTimeout = 5 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
// DBMaintainer handles background database maintenance tasks.
|
||||||
|
type DBMaintainer struct {
|
||||||
|
db database.Store
|
||||||
|
logger *slog.Logger
|
||||||
|
stopCh chan struct{}
|
||||||
|
wg sync.WaitGroup
|
||||||
|
|
||||||
|
// Stats tracking
|
||||||
|
statsMu sync.Mutex
|
||||||
|
lastVacuum time.Time
|
||||||
|
lastAnalyze time.Time
|
||||||
|
vacuumCount int
|
||||||
|
analyzeCount int
|
||||||
|
lastVacuumError error
|
||||||
|
lastAnalyzeError error
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewDBMaintainer creates a new database maintainer.
|
||||||
|
func NewDBMaintainer(db database.Store, logger *slog.Logger) *DBMaintainer {
|
||||||
|
return &DBMaintainer{
|
||||||
|
db: db,
|
||||||
|
logger: logger.With("component", "db_maintainer"),
|
||||||
|
stopCh: make(chan struct{}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start begins the background maintenance goroutine.
|
||||||
|
func (m *DBMaintainer) Start() {
|
||||||
|
m.wg.Add(1)
|
||||||
|
go m.run()
|
||||||
|
m.logger.Info("Database maintainer started",
|
||||||
|
"vacuum_interval", vacuumInterval,
|
||||||
|
"analyze_interval", analyzeInterval,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop gracefully shuts down the maintainer.
|
||||||
|
func (m *DBMaintainer) Stop() {
|
||||||
|
close(m.stopCh)
|
||||||
|
m.wg.Wait()
|
||||||
|
m.logger.Info("Database maintainer stopped")
|
||||||
|
}
|
||||||
|
|
||||||
|
// run is the main background loop.
|
||||||
|
func (m *DBMaintainer) run() {
|
||||||
|
defer m.wg.Done()
|
||||||
|
|
||||||
|
// Use different timers for each task
|
||||||
|
vacuumTimer := time.NewTimer(vacuumInterval)
|
||||||
|
analyzeTimer := time.NewTimer(analyzeInterval)
|
||||||
|
defer vacuumTimer.Stop()
|
||||||
|
defer analyzeTimer.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-m.stopCh:
|
||||||
|
return
|
||||||
|
|
||||||
|
case <-vacuumTimer.C:
|
||||||
|
m.runVacuum()
|
||||||
|
vacuumTimer.Reset(vacuumInterval)
|
||||||
|
|
||||||
|
case <-analyzeTimer.C:
|
||||||
|
m.runAnalyze()
|
||||||
|
analyzeTimer.Reset(analyzeInterval)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// runVacuum performs an incremental vacuum operation on the database.
|
||||||
|
func (m *DBMaintainer) runVacuum() {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), vacuumTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
m.logger.Debug("Running incremental vacuum")
|
||||||
|
startTime := time.Now()
|
||||||
|
|
||||||
|
err := m.db.Vacuum(ctx)
|
||||||
|
|
||||||
|
m.statsMu.Lock()
|
||||||
|
m.lastVacuum = time.Now()
|
||||||
|
m.lastVacuumError = err
|
||||||
|
if err == nil {
|
||||||
|
m.vacuumCount++
|
||||||
|
}
|
||||||
|
m.statsMu.Unlock()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
m.logger.Error("Incremental vacuum failed", "error", err, "duration", time.Since(startTime))
|
||||||
|
} else {
|
||||||
|
m.logger.Debug("Incremental vacuum completed", "duration", time.Since(startTime))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// runAnalyze performs an ANALYZE operation on the database.
|
||||||
|
func (m *DBMaintainer) runAnalyze() {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), analyzeTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
m.logger.Info("Starting database ANALYZE")
|
||||||
|
startTime := time.Now()
|
||||||
|
|
||||||
|
err := m.db.Analyze(ctx)
|
||||||
|
|
||||||
|
m.statsMu.Lock()
|
||||||
|
m.lastAnalyze = time.Now()
|
||||||
|
m.lastAnalyzeError = err
|
||||||
|
if err == nil {
|
||||||
|
m.analyzeCount++
|
||||||
|
}
|
||||||
|
m.statsMu.Unlock()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
m.logger.Error("ANALYZE failed", "error", err, "duration", time.Since(startTime))
|
||||||
|
} else {
|
||||||
|
m.logger.Info("ANALYZE completed", "duration", time.Since(startTime))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,14 +5,20 @@ import (
|
|||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
||||||
)
|
)
|
||||||
|
|
||||||
// SimpleHandler is a basic implementation of streamer.MessageHandler
|
// SimpleHandler is a basic implementation of streamer.MessageHandler that
|
||||||
|
// filters messages by type and delegates processing to a callback function.
|
||||||
|
// It provides a simple way to handle specific RIS message types without
|
||||||
|
// implementing the full MessageHandler interface from scratch.
|
||||||
type SimpleHandler struct {
|
type SimpleHandler struct {
|
||||||
logger *logger.Logger
|
logger *logger.Logger
|
||||||
messageTypes []string
|
messageTypes []string
|
||||||
callback func(*ristypes.RISMessage)
|
callback func(*ristypes.RISMessage)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewSimpleHandler creates a handler that accepts specific message types
|
// NewSimpleHandler creates a new SimpleHandler that accepts specific message types.
|
||||||
|
// The messageTypes parameter specifies which RIS message types this handler will process.
|
||||||
|
// If messageTypes is empty, the handler will accept all message types.
|
||||||
|
// The callback function is invoked for each message that passes the type filter.
|
||||||
func NewSimpleHandler(
|
func NewSimpleHandler(
|
||||||
logger *logger.Logger,
|
logger *logger.Logger,
|
||||||
messageTypes []string,
|
messageTypes []string,
|
||||||
@@ -25,7 +31,9 @@ func NewSimpleHandler(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// WantsMessage returns true if this handler wants to process messages of the given type
|
// WantsMessage returns true if this handler wants to process messages of the given type.
|
||||||
|
// It checks whether messageType is in the handler's configured list of accepted types.
|
||||||
|
// If no specific types were configured (empty messageTypes slice), it returns true for all types.
|
||||||
func (h *SimpleHandler) WantsMessage(messageType string) bool {
|
func (h *SimpleHandler) WantsMessage(messageType string) bool {
|
||||||
// If no specific types are set, accept all messages
|
// If no specific types are set, accept all messages
|
||||||
if len(h.messageTypes) == 0 {
|
if len(h.messageTypes) == 0 {
|
||||||
@@ -41,7 +49,8 @@ func (h *SimpleHandler) WantsMessage(messageType string) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// HandleMessage processes a RIS message
|
// HandleMessage processes a RIS message by invoking the configured callback function.
|
||||||
|
// If no callback was provided during construction, the message is silently ignored.
|
||||||
func (h *SimpleHandler) HandleMessage(msg *ristypes.RISMessage) {
|
func (h *SimpleHandler) HandleMessage(msg *ristypes.RISMessage) {
|
||||||
if h.callback != nil {
|
if h.callback != nil {
|
||||||
h.callback(msg)
|
h.callback(msg)
|
||||||
|
|||||||
@@ -1,5 +1,8 @@
|
|||||||
package routewatch
|
package routewatch
|
||||||
|
|
||||||
|
// peerhandler.go provides batched peer tracking functionality for BGP route monitoring.
|
||||||
|
// It tracks BGP peers from all incoming RIS messages and maintains peer state in the database.
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"strconv"
|
"strconv"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -21,7 +24,10 @@ const (
|
|||||||
peerBatchTimeout = 2 * time.Second
|
peerBatchTimeout = 2 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
// PeerHandler tracks BGP peers from all message types using batched operations
|
// PeerHandler tracks BGP peers from all message types using batched operations.
|
||||||
|
// It maintains a queue of peer updates and periodically flushes them to the database
|
||||||
|
// in batches to improve performance. The handler deduplicates peer updates within
|
||||||
|
// each batch, keeping only the most recent update for each peer IP address.
|
||||||
type PeerHandler struct {
|
type PeerHandler struct {
|
||||||
db database.Store
|
db database.Store
|
||||||
logger *logger.Logger
|
logger *logger.Logger
|
||||||
@@ -41,7 +47,10 @@ type peerUpdate struct {
|
|||||||
timestamp time.Time
|
timestamp time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewPeerHandler creates a new batched peer tracking handler
|
// NewPeerHandler creates a new PeerHandler with the given database store and logger.
|
||||||
|
// It initializes the peer batch buffer and starts a background goroutine that
|
||||||
|
// periodically flushes accumulated peer updates to the database. The handler
|
||||||
|
// should be stopped by calling Stop when it is no longer needed.
|
||||||
func NewPeerHandler(db database.Store, logger *logger.Logger) *PeerHandler {
|
func NewPeerHandler(db database.Store, logger *logger.Logger) *PeerHandler {
|
||||||
h := &PeerHandler{
|
h := &PeerHandler{
|
||||||
db: db,
|
db: db,
|
||||||
@@ -58,18 +67,25 @@ func NewPeerHandler(db database.Store, logger *logger.Logger) *PeerHandler {
|
|||||||
return h
|
return h
|
||||||
}
|
}
|
||||||
|
|
||||||
// WantsMessage returns true for all message types since we track peers from all messages
|
// WantsMessage returns true for all message types since peer information
|
||||||
|
// is extracted from every RIS message regardless of type. This satisfies
|
||||||
|
// the MessageHandler interface.
|
||||||
func (h *PeerHandler) WantsMessage(_ string) bool {
|
func (h *PeerHandler) WantsMessage(_ string) bool {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
// QueueCapacity returns the desired queue capacity for this handler
|
// QueueCapacity returns the desired queue capacity for this handler.
|
||||||
|
// The PeerHandler uses a large queue capacity because batching allows
|
||||||
|
// for efficient processing of many updates at once.
|
||||||
func (h *PeerHandler) QueueCapacity() int {
|
func (h *PeerHandler) QueueCapacity() int {
|
||||||
// Batching allows us to use a larger queue
|
// Batching allows us to use a larger queue
|
||||||
return peerHandlerQueueSize
|
return peerHandlerQueueSize
|
||||||
}
|
}
|
||||||
|
|
||||||
// HandleMessage processes a message to track peer information
|
// HandleMessage processes a RIS message to track peer information.
|
||||||
|
// It extracts the peer IP address and ASN from the message and adds
|
||||||
|
// the update to an internal batch. When the batch reaches peerBatchSize
|
||||||
|
// or the batch timeout expires, the batch is flushed to the database.
|
||||||
func (h *PeerHandler) HandleMessage(msg *ristypes.RISMessage) {
|
func (h *PeerHandler) HandleMessage(msg *ristypes.RISMessage) {
|
||||||
// Parse peer ASN from string
|
// Parse peer ASN from string
|
||||||
peerASN := 0
|
peerASN := 0
|
||||||
|
|||||||
@@ -11,23 +11,36 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// peeringHandlerQueueSize is the queue capacity for peering operations
|
// peeringHandlerQueueSize defines the buffer capacity for the peering
|
||||||
|
// handler's message queue. This should be large enough to handle bursts
|
||||||
|
// of BGP UPDATE messages without blocking.
|
||||||
peeringHandlerQueueSize = 100000
|
peeringHandlerQueueSize = 100000
|
||||||
|
|
||||||
// minPathLengthForPeering is the minimum AS path length to extract peerings
|
// minPathLengthForPeering specifies the minimum number of ASNs required
|
||||||
|
// in a BGP AS path to extract peering relationships. A path with fewer
|
||||||
|
// than 2 ASNs cannot contain any peering information.
|
||||||
minPathLengthForPeering = 2
|
minPathLengthForPeering = 2
|
||||||
|
|
||||||
// pathExpirationTime is how long to keep AS paths in memory
|
// pathExpirationTime determines how long AS paths are kept in memory
|
||||||
|
// before being eligible for pruning. Paths older than this are removed
|
||||||
|
// to prevent unbounded memory growth.
|
||||||
pathExpirationTime = 30 * time.Minute
|
pathExpirationTime = 30 * time.Minute
|
||||||
|
|
||||||
// peeringProcessInterval is how often to process AS paths into peerings
|
// peeringProcessInterval controls how frequently the handler processes
|
||||||
peeringProcessInterval = 2 * time.Minute
|
// accumulated AS paths and extracts peering relationships to store
|
||||||
|
// in the database.
|
||||||
|
peeringProcessInterval = 30 * time.Second
|
||||||
|
|
||||||
// pathPruneInterval is how often to prune old AS paths
|
// pathPruneInterval determines how often the handler checks for and
|
||||||
|
// removes expired AS paths from memory.
|
||||||
pathPruneInterval = 5 * time.Minute
|
pathPruneInterval = 5 * time.Minute
|
||||||
)
|
)
|
||||||
|
|
||||||
// PeeringHandler handles AS peering relationships from BGP path data
|
// PeeringHandler processes BGP UPDATE messages to extract and track
|
||||||
|
// AS peering relationships. It accumulates AS paths in memory and
|
||||||
|
// periodically processes them to extract unique peering pairs, which
|
||||||
|
// are then stored in the database. The handler implements the Handler
|
||||||
|
// interface for integration with the message processing pipeline.
|
||||||
type PeeringHandler struct {
|
type PeeringHandler struct {
|
||||||
db database.Store
|
db database.Store
|
||||||
logger *logger.Logger
|
logger *logger.Logger
|
||||||
@@ -39,7 +52,11 @@ type PeeringHandler struct {
|
|||||||
stopCh chan struct{}
|
stopCh chan struct{}
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewPeeringHandler creates a new batched peering handler
|
// NewPeeringHandler creates and initializes a new PeeringHandler with the
|
||||||
|
// provided database store and logger. It starts two background goroutines:
|
||||||
|
// one for periodic processing of accumulated AS paths into peering records,
|
||||||
|
// and one for pruning expired paths from memory. The handler begins
|
||||||
|
// processing immediately upon creation.
|
||||||
func NewPeeringHandler(db database.Store, logger *logger.Logger) *PeeringHandler {
|
func NewPeeringHandler(db database.Store, logger *logger.Logger) *PeeringHandler {
|
||||||
h := &PeeringHandler{
|
h := &PeeringHandler{
|
||||||
db: db,
|
db: db,
|
||||||
@@ -55,18 +72,25 @@ func NewPeeringHandler(db database.Store, logger *logger.Logger) *PeeringHandler
|
|||||||
return h
|
return h
|
||||||
}
|
}
|
||||||
|
|
||||||
// WantsMessage returns true if this handler wants to process messages of the given type
|
// WantsMessage reports whether the handler should receive messages of the
|
||||||
|
// given type. PeeringHandler only processes UPDATE messages, as these contain
|
||||||
|
// the AS path information needed to extract peering relationships.
|
||||||
func (h *PeeringHandler) WantsMessage(messageType string) bool {
|
func (h *PeeringHandler) WantsMessage(messageType string) bool {
|
||||||
// We only care about UPDATE messages that have AS paths
|
// We only care about UPDATE messages that have AS paths
|
||||||
return messageType == "UPDATE"
|
return messageType == "UPDATE"
|
||||||
}
|
}
|
||||||
|
|
||||||
// QueueCapacity returns the desired queue capacity for this handler
|
// QueueCapacity returns the buffer size for the handler's message queue.
|
||||||
|
// This value is used by the message dispatcher to allocate the channel
|
||||||
|
// buffer when registering the handler.
|
||||||
func (h *PeeringHandler) QueueCapacity() int {
|
func (h *PeeringHandler) QueueCapacity() int {
|
||||||
return peeringHandlerQueueSize
|
return peeringHandlerQueueSize
|
||||||
}
|
}
|
||||||
|
|
||||||
// HandleMessage processes a message to extract AS paths
|
// HandleMessage processes a BGP UPDATE message by storing its AS path
|
||||||
|
// in memory for later batch processing. Messages with AS paths shorter
|
||||||
|
// than minPathLengthForPeering are ignored as they cannot contain valid
|
||||||
|
// peering information.
|
||||||
func (h *PeeringHandler) HandleMessage(msg *ristypes.RISMessage) {
|
func (h *PeeringHandler) HandleMessage(msg *ristypes.RISMessage) {
|
||||||
// Skip if no AS path or only one AS
|
// Skip if no AS path or only one AS
|
||||||
if len(msg.Path) < minPathLengthForPeering {
|
if len(msg.Path) < minPathLengthForPeering {
|
||||||
@@ -141,7 +165,9 @@ func (h *PeeringHandler) prunePaths() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ProcessPeeringsNow forces immediate processing of peerings (for testing)
|
// ProcessPeeringsNow triggers immediate processing of all accumulated AS
|
||||||
|
// paths into peering records. This bypasses the normal periodic processing
|
||||||
|
// schedule and is primarily intended for testing purposes.
|
||||||
func (h *PeeringHandler) ProcessPeeringsNow() {
|
func (h *PeeringHandler) ProcessPeeringsNow() {
|
||||||
h.processPeerings()
|
h.processPeerings()
|
||||||
}
|
}
|
||||||
@@ -222,7 +248,10 @@ func (h *PeeringHandler) processPeerings() {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop gracefully stops the handler and processes remaining peerings
|
// Stop gracefully shuts down the handler by signaling the background
|
||||||
|
// goroutines to stop and performing a final synchronous processing of
|
||||||
|
// any remaining AS paths. This ensures no peering data is lost during
|
||||||
|
// shutdown.
|
||||||
func (h *PeeringHandler) Stop() {
|
func (h *PeeringHandler) Stop() {
|
||||||
close(h.stopCh)
|
close(h.stopCh)
|
||||||
// Process any remaining peerings synchronously
|
// Process any remaining peerings synchronously
|
||||||
|
|||||||
@@ -15,12 +15,14 @@ import (
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
// prefixHandlerQueueSize is the queue capacity for prefix tracking operations
|
// prefixHandlerQueueSize is the queue capacity for prefix tracking operations
|
||||||
|
// DO NOT set this higher than 100000 without explicit instructions
|
||||||
prefixHandlerQueueSize = 100000
|
prefixHandlerQueueSize = 100000
|
||||||
|
|
||||||
// prefixBatchSize is the number of prefix updates to batch together
|
// prefixBatchSize is the number of prefix updates to batch together
|
||||||
prefixBatchSize = 5000
|
prefixBatchSize = 25000
|
||||||
|
|
||||||
// prefixBatchTimeout is the maximum time to wait before flushing a batch
|
// prefixBatchTimeout is the maximum time to wait before flushing a batch
|
||||||
|
// DO NOT reduce this timeout - larger batches are more efficient
|
||||||
prefixBatchTimeout = 1 * time.Second
|
prefixBatchTimeout = 1 * time.Second
|
||||||
|
|
||||||
// IP version constants
|
// IP version constants
|
||||||
@@ -180,9 +182,15 @@ func (h *PrefixHandler) flushBatchLocked() {
|
|||||||
var routesToUpsert []*database.LiveRoute
|
var routesToUpsert []*database.LiveRoute
|
||||||
var routesToDelete []database.LiveRouteDeletion
|
var routesToDelete []database.LiveRouteDeletion
|
||||||
|
|
||||||
// Skip the prefix table updates entirely - just update live_routes
|
// Collect unique prefixes to update
|
||||||
// The prefix table is not critical for routing lookups
|
prefixesToUpdate := make(map[string]time.Time)
|
||||||
|
|
||||||
for _, update := range prefixMap {
|
for _, update := range prefixMap {
|
||||||
|
// Track prefix for both announcements and withdrawals
|
||||||
|
if _, exists := prefixesToUpdate[update.prefix]; !exists || update.timestamp.After(prefixesToUpdate[update.prefix]) {
|
||||||
|
prefixesToUpdate[update.prefix] = update.timestamp
|
||||||
|
}
|
||||||
|
|
||||||
if update.messageType == "announcement" && update.originASN > 0 {
|
if update.messageType == "announcement" && update.originASN > 0 {
|
||||||
// Create live route for batch upsert
|
// Create live route for batch upsert
|
||||||
route := h.createLiveRoute(update)
|
route := h.createLiveRoute(update)
|
||||||
@@ -190,11 +198,20 @@ func (h *PrefixHandler) flushBatchLocked() {
|
|||||||
routesToUpsert = append(routesToUpsert, route)
|
routesToUpsert = append(routesToUpsert, route)
|
||||||
}
|
}
|
||||||
} else if update.messageType == "withdrawal" {
|
} else if update.messageType == "withdrawal" {
|
||||||
|
// Parse CIDR to get IP version
|
||||||
|
_, ipVersion, err := parseCIDR(update.prefix)
|
||||||
|
if err != nil {
|
||||||
|
h.logger.Error("Failed to parse CIDR for withdrawal", "prefix", update.prefix, "error", err)
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
// Create deletion record for batch delete
|
// Create deletion record for batch delete
|
||||||
routesToDelete = append(routesToDelete, database.LiveRouteDeletion{
|
routesToDelete = append(routesToDelete, database.LiveRouteDeletion{
|
||||||
Prefix: update.prefix,
|
Prefix: update.prefix,
|
||||||
OriginASN: update.originASN,
|
OriginASN: update.originASN,
|
||||||
PeerIP: update.peer,
|
PeerIP: update.peer,
|
||||||
|
IPVersion: ipVersion,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -217,6 +234,13 @@ func (h *PrefixHandler) flushBatchLocked() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Update prefix tables
|
||||||
|
if len(prefixesToUpdate) > 0 {
|
||||||
|
if err := h.db.UpdatePrefixesBatch(prefixesToUpdate); err != nil {
|
||||||
|
h.logger.Error("Failed to update prefix batch", "error", err, "count", len(prefixesToUpdate))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
elapsed := time.Since(startTime)
|
elapsed := time.Since(startTime)
|
||||||
h.logger.Debug("Flushed prefix batch",
|
h.logger.Debug("Flushed prefix batch",
|
||||||
"batch_size", batchSize,
|
"batch_size", batchSize,
|
||||||
|
|||||||
@@ -11,22 +11,92 @@ import (
|
|||||||
"runtime"
|
"runtime"
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
"git.eeqj.de/sneak/routewatch/internal/database"
|
||||||
"git.eeqj.de/sneak/routewatch/internal/templates"
|
"git.eeqj.de/sneak/routewatch/internal/templates"
|
||||||
|
asinfo "git.eeqj.de/sneak/routewatch/pkg/asinfo"
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
)
|
)
|
||||||
|
|
||||||
// handleRoot returns a handler that redirects to /status
|
const (
|
||||||
|
// statsContextTimeout is the timeout for stats API operations.
|
||||||
|
statsContextTimeout = 4 * time.Second
|
||||||
|
|
||||||
|
// healthCheckTimeout is the timeout for health check operations.
|
||||||
|
healthCheckTimeout = 2 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// HealthCheckResponse represents the health check response.
|
||||||
|
type HealthCheckResponse struct {
|
||||||
|
Status string `json:"status"`
|
||||||
|
Timestamp string `json:"timestamp"`
|
||||||
|
Checks map[string]string `json:"checks"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleHealthCheck returns a handler that performs health checks.
|
||||||
|
// Returns 200 if healthy, 503 if any check fails.
|
||||||
|
func (s *Server) handleHealthCheck() http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
ctx, cancel := context.WithTimeout(r.Context(), healthCheckTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
checks := make(map[string]string)
|
||||||
|
healthy := true
|
||||||
|
|
||||||
|
// Check database connectivity
|
||||||
|
dbStats, err := s.db.GetStatsContext(ctx)
|
||||||
|
if err != nil {
|
||||||
|
checks["database"] = "error: " + err.Error()
|
||||||
|
healthy = false
|
||||||
|
} else if dbStats.ASNs == 0 && dbStats.Prefixes == 0 {
|
||||||
|
checks["database"] = "warning: empty database"
|
||||||
|
} else {
|
||||||
|
checks["database"] = "ok"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check streamer connection
|
||||||
|
metrics := s.streamer.GetMetrics()
|
||||||
|
if metrics.Connected {
|
||||||
|
checks["ris_live"] = "ok"
|
||||||
|
} else {
|
||||||
|
checks["ris_live"] = "disconnected"
|
||||||
|
healthy = false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build response
|
||||||
|
status := "ok"
|
||||||
|
if !healthy {
|
||||||
|
status = "error"
|
||||||
|
}
|
||||||
|
|
||||||
|
response := HealthCheckResponse{
|
||||||
|
Status: status,
|
||||||
|
Timestamp: time.Now().UTC().Format(time.RFC3339),
|
||||||
|
Checks: checks,
|
||||||
|
}
|
||||||
|
|
||||||
|
if !healthy {
|
||||||
|
w.WriteHeader(http.StatusServiceUnavailable)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := writeJSONSuccess(w, response); err != nil {
|
||||||
|
s.logger.Error("Failed to encode health check response", "error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleRoot returns a handler that redirects to /status.
|
||||||
func (s *Server) handleRoot() http.HandlerFunc {
|
func (s *Server) handleRoot() http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
http.Redirect(w, r, "/status", http.StatusSeeOther)
|
http.Redirect(w, r, "/status", http.StatusSeeOther)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeJSONError writes a standardized JSON error response
|
// writeJSONError writes a standardized JSON error response with the given
|
||||||
|
// status code and error message.
|
||||||
func writeJSONError(w http.ResponseWriter, statusCode int, message string) {
|
func writeJSONError(w http.ResponseWriter, statusCode int, message string) {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
w.WriteHeader(statusCode)
|
w.WriteHeader(statusCode)
|
||||||
@@ -39,7 +109,8 @@ func writeJSONError(w http.ResponseWriter, statusCode int, message string) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeJSONSuccess writes a standardized JSON success response
|
// writeJSONSuccess writes a standardized JSON success response containing
|
||||||
|
// the provided data wrapped in a status envelope.
|
||||||
func writeJSONSuccess(w http.ResponseWriter, data interface{}) error {
|
func writeJSONSuccess(w http.ResponseWriter, data interface{}) error {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
|
||||||
@@ -49,15 +120,31 @@ func writeJSONSuccess(w http.ResponseWriter, data interface{}) error {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleStatusJSON returns a handler that serves JSON statistics
|
// WHOISStatsInfo contains WHOIS fetcher statistics for the status page.
|
||||||
|
type WHOISStatsInfo struct {
|
||||||
|
TotalASNs int `json:"total_asns"`
|
||||||
|
FreshASNs int `json:"fresh_asns"`
|
||||||
|
StaleASNs int `json:"stale_asns"`
|
||||||
|
NeverFetched int `json:"never_fetched"`
|
||||||
|
SuccessesLastHour int `json:"successes_last_hour"`
|
||||||
|
ErrorsLastHour int `json:"errors_last_hour"`
|
||||||
|
CurrentInterval string `json:"current_interval"`
|
||||||
|
ConsecutiveFails int `json:"consecutive_fails"`
|
||||||
|
FreshPercent float64 `json:"fresh_percent"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleStatusJSON returns a handler that serves JSON statistics including
|
||||||
|
// uptime, message counts, database stats, and route information.
|
||||||
func (s *Server) handleStatusJSON() http.HandlerFunc {
|
func (s *Server) handleStatusJSON() http.HandlerFunc {
|
||||||
// Stats represents the statistics response
|
// Stats represents the statistics response
|
||||||
type Stats struct {
|
type Stats struct {
|
||||||
Uptime string `json:"uptime"`
|
Uptime string `json:"uptime"`
|
||||||
TotalMessages uint64 `json:"total_messages"`
|
TotalMessages uint64 `json:"total_messages"`
|
||||||
TotalBytes uint64 `json:"total_bytes"`
|
TotalBytes uint64 `json:"total_bytes"`
|
||||||
|
TotalWireBytes uint64 `json:"total_wire_bytes"`
|
||||||
MessagesPerSec float64 `json:"messages_per_sec"`
|
MessagesPerSec float64 `json:"messages_per_sec"`
|
||||||
MbitsPerSec float64 `json:"mbits_per_sec"`
|
MbitsPerSec float64 `json:"mbits_per_sec"`
|
||||||
|
WireMbitsPerSec float64 `json:"wire_mbits_per_sec"`
|
||||||
Connected bool `json:"connected"`
|
Connected bool `json:"connected"`
|
||||||
GoVersion string `json:"go_version"`
|
GoVersion string `json:"go_version"`
|
||||||
Goroutines int `json:"goroutines"`
|
Goroutines int `json:"goroutines"`
|
||||||
@@ -76,11 +163,12 @@ func (s *Server) handleStatusJSON() http.HandlerFunc {
|
|||||||
IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"`
|
IPv6UpdatesPerSec float64 `json:"ipv6_updates_per_sec"`
|
||||||
IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"`
|
IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"`
|
||||||
IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"`
|
IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"`
|
||||||
|
WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
// Create a 1 second timeout context for this request
|
// Create a 4 second timeout context for this request
|
||||||
ctx, cancel := context.WithTimeout(r.Context(), 1*time.Second)
|
ctx, cancel := context.WithTimeout(r.Context(), statsContextTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
metrics := s.streamer.GetMetrics()
|
metrics := s.streamer.GetMetrics()
|
||||||
@@ -90,7 +178,7 @@ func (s *Server) handleStatusJSON() http.HandlerFunc {
|
|||||||
errChan := make(chan error)
|
errChan := make(chan error)
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
dbStats, err := s.db.GetStats()
|
dbStats, err := s.db.GetStatsContext(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Debug("Database stats query failed", "error", err)
|
s.logger.Debug("Database stats query failed", "error", err)
|
||||||
errChan <- err
|
errChan <- err
|
||||||
@@ -124,7 +212,7 @@ func (s *Server) handleStatusJSON() http.HandlerFunc {
|
|||||||
const bitsPerMegabit = 1000000.0
|
const bitsPerMegabit = 1000000.0
|
||||||
|
|
||||||
// Get route counts from database
|
// Get route counts from database
|
||||||
ipv4Routes, ipv6Routes, err := s.db.GetLiveRouteCounts()
|
ipv4Routes, ipv6Routes, err := s.db.GetLiveRouteCountsContext(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Warn("Failed to get live route counts", "error", err)
|
s.logger.Warn("Failed to get live route counts", "error", err)
|
||||||
// Continue with zero counts
|
// Continue with zero counts
|
||||||
@@ -137,12 +225,20 @@ func (s *Server) handleStatusJSON() http.HandlerFunc {
|
|||||||
var memStats runtime.MemStats
|
var memStats runtime.MemStats
|
||||||
runtime.ReadMemStats(&memStats)
|
runtime.ReadMemStats(&memStats)
|
||||||
|
|
||||||
|
// Get WHOIS stats if fetcher is available
|
||||||
|
var whoisStats *WHOISStatsInfo
|
||||||
|
if s.asnFetcher != nil {
|
||||||
|
whoisStats = s.getWHOISStats(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
stats := Stats{
|
stats := Stats{
|
||||||
Uptime: uptime,
|
Uptime: uptime,
|
||||||
TotalMessages: metrics.TotalMessages,
|
TotalMessages: metrics.TotalMessages,
|
||||||
TotalBytes: metrics.TotalBytes,
|
TotalBytes: metrics.TotalBytes,
|
||||||
|
TotalWireBytes: metrics.TotalWireBytes,
|
||||||
MessagesPerSec: metrics.MessagesPerSec,
|
MessagesPerSec: metrics.MessagesPerSec,
|
||||||
MbitsPerSec: metrics.BitsPerSec / bitsPerMegabit,
|
MbitsPerSec: metrics.BitsPerSec / bitsPerMegabit,
|
||||||
|
WireMbitsPerSec: metrics.WireBitsPerSec / bitsPerMegabit,
|
||||||
Connected: metrics.Connected,
|
Connected: metrics.Connected,
|
||||||
GoVersion: runtime.Version(),
|
GoVersion: runtime.Version(),
|
||||||
Goroutines: runtime.NumGoroutine(),
|
Goroutines: runtime.NumGoroutine(),
|
||||||
@@ -161,6 +257,7 @@ func (s *Server) handleStatusJSON() http.HandlerFunc {
|
|||||||
IPv6UpdatesPerSec: routeMetrics.IPv6UpdatesPerSec,
|
IPv6UpdatesPerSec: routeMetrics.IPv6UpdatesPerSec,
|
||||||
IPv4PrefixDistribution: dbStats.IPv4PrefixDistribution,
|
IPv4PrefixDistribution: dbStats.IPv4PrefixDistribution,
|
||||||
IPv6PrefixDistribution: dbStats.IPv6PrefixDistribution,
|
IPv6PrefixDistribution: dbStats.IPv6PrefixDistribution,
|
||||||
|
WHOISStats: whoisStats,
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := writeJSONSuccess(w, stats); err != nil {
|
if err := writeJSONSuccess(w, stats); err != nil {
|
||||||
@@ -169,13 +266,53 @@ func (s *Server) handleStatusJSON() http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleStats returns a handler that serves API v1 statistics
|
// getWHOISStats builds WHOIS statistics from database and fetcher.
|
||||||
|
func (s *Server) getWHOISStats(ctx context.Context) *WHOISStatsInfo {
|
||||||
|
// Get database WHOIS stats
|
||||||
|
dbStats, err := s.db.GetWHOISStats(ctx, whoisStaleThreshold)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Warn("Failed to get WHOIS stats", "error", err)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get fetcher stats
|
||||||
|
fetcherStats := s.asnFetcher.GetStats()
|
||||||
|
|
||||||
|
// Calculate fresh percentage
|
||||||
|
var freshPercent float64
|
||||||
|
if dbStats.TotalASNs > 0 {
|
||||||
|
freshPercent = float64(dbStats.FreshASNs) / float64(dbStats.TotalASNs) * percentMultiplier
|
||||||
|
}
|
||||||
|
|
||||||
|
return &WHOISStatsInfo{
|
||||||
|
TotalASNs: dbStats.TotalASNs,
|
||||||
|
FreshASNs: dbStats.FreshASNs,
|
||||||
|
StaleASNs: dbStats.StaleASNs,
|
||||||
|
NeverFetched: dbStats.NeverFetched,
|
||||||
|
SuccessesLastHour: fetcherStats.SuccessesLastHour,
|
||||||
|
ErrorsLastHour: fetcherStats.ErrorsLastHour,
|
||||||
|
CurrentInterval: fetcherStats.CurrentInterval.String(),
|
||||||
|
ConsecutiveFails: fetcherStats.ConsecutiveFails,
|
||||||
|
FreshPercent: freshPercent,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// whoisStaleThreshold matches the fetcher's threshold for consistency.
|
||||||
|
const whoisStaleThreshold = 30 * 24 * time.Hour
|
||||||
|
|
||||||
|
// percentMultiplier converts a ratio to a percentage.
|
||||||
|
const percentMultiplier = 100
|
||||||
|
|
||||||
|
// handleStats returns a handler that serves API v1 statistics including
|
||||||
|
// detailed handler queue statistics and performance metrics.
|
||||||
func (s *Server) handleStats() http.HandlerFunc {
|
func (s *Server) handleStats() http.HandlerFunc {
|
||||||
// HandlerStatsInfo represents handler statistics in the API response
|
// HandlerStatsInfo represents handler statistics in the API response
|
||||||
type HandlerStatsInfo struct {
|
type HandlerStatsInfo struct {
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
QueueLength int `json:"queue_length"`
|
QueueLength int `json:"queue_length"`
|
||||||
QueueCapacity int `json:"queue_capacity"`
|
QueueCapacity int `json:"queue_capacity"`
|
||||||
|
QueueHighWaterMark int `json:"queue_high_water_mark"`
|
||||||
ProcessedCount uint64 `json:"processed_count"`
|
ProcessedCount uint64 `json:"processed_count"`
|
||||||
DroppedCount uint64 `json:"dropped_count"`
|
DroppedCount uint64 `json:"dropped_count"`
|
||||||
AvgProcessTimeMs float64 `json:"avg_process_time_ms"`
|
AvgProcessTimeMs float64 `json:"avg_process_time_ms"`
|
||||||
@@ -188,9 +325,13 @@ func (s *Server) handleStats() http.HandlerFunc {
|
|||||||
Uptime string `json:"uptime"`
|
Uptime string `json:"uptime"`
|
||||||
TotalMessages uint64 `json:"total_messages"`
|
TotalMessages uint64 `json:"total_messages"`
|
||||||
TotalBytes uint64 `json:"total_bytes"`
|
TotalBytes uint64 `json:"total_bytes"`
|
||||||
|
TotalWireBytes uint64 `json:"total_wire_bytes"`
|
||||||
MessagesPerSec float64 `json:"messages_per_sec"`
|
MessagesPerSec float64 `json:"messages_per_sec"`
|
||||||
MbitsPerSec float64 `json:"mbits_per_sec"`
|
MbitsPerSec float64 `json:"mbits_per_sec"`
|
||||||
|
WireMbitsPerSec float64 `json:"wire_mbits_per_sec"`
|
||||||
Connected bool `json:"connected"`
|
Connected bool `json:"connected"`
|
||||||
|
ConnectionDuration string `json:"connection_duration"`
|
||||||
|
ReconnectCount uint64 `json:"reconnect_count"`
|
||||||
GoVersion string `json:"go_version"`
|
GoVersion string `json:"go_version"`
|
||||||
Goroutines int `json:"goroutines"`
|
Goroutines int `json:"goroutines"`
|
||||||
MemoryUsage string `json:"memory_usage"`
|
MemoryUsage string `json:"memory_usage"`
|
||||||
@@ -209,11 +350,12 @@ func (s *Server) handleStats() http.HandlerFunc {
|
|||||||
HandlerStats []HandlerStatsInfo `json:"handler_stats"`
|
HandlerStats []HandlerStatsInfo `json:"handler_stats"`
|
||||||
IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"`
|
IPv4PrefixDistribution []database.PrefixDistribution `json:"ipv4_prefix_distribution"`
|
||||||
IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"`
|
IPv6PrefixDistribution []database.PrefixDistribution `json:"ipv6_prefix_distribution"`
|
||||||
|
WHOISStats *WHOISStatsInfo `json:"whois_stats,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
// Create a 1 second timeout context for this request
|
// Create a 4 second timeout context for this request
|
||||||
ctx, cancel := context.WithTimeout(r.Context(), 1*time.Second)
|
ctx, cancel := context.WithTimeout(r.Context(), statsContextTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
// Check if context is already cancelled
|
// Check if context is already cancelled
|
||||||
@@ -232,7 +374,7 @@ func (s *Server) handleStats() http.HandlerFunc {
|
|||||||
errChan := make(chan error)
|
errChan := make(chan error)
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
dbStats, err := s.db.GetStats()
|
dbStats, err := s.db.GetStatsContext(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Debug("Database stats query failed", "error", err)
|
s.logger.Debug("Database stats query failed", "error", err)
|
||||||
errChan <- err
|
errChan <- err
|
||||||
@@ -246,12 +388,11 @@ func (s *Server) handleStats() http.HandlerFunc {
|
|||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
s.logger.Error("Database stats timeout")
|
s.logger.Error("Database stats timeout")
|
||||||
http.Error(w, "Database timeout", http.StatusRequestTimeout)
|
// Don't write response here - timeout middleware already handles it
|
||||||
|
|
||||||
return
|
return
|
||||||
case err := <-errChan:
|
case err := <-errChan:
|
||||||
s.logger.Error("Failed to get database stats", "error", err)
|
s.logger.Error("Failed to get database stats", "error", err)
|
||||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
writeJSONError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
|
||||||
return
|
return
|
||||||
case dbStats = <-statsChan:
|
case dbStats = <-statsChan:
|
||||||
@@ -266,7 +407,7 @@ func (s *Server) handleStats() http.HandlerFunc {
|
|||||||
const bitsPerMegabit = 1000000.0
|
const bitsPerMegabit = 1000000.0
|
||||||
|
|
||||||
// Get route counts from database
|
// Get route counts from database
|
||||||
ipv4Routes, ipv6Routes, err := s.db.GetLiveRouteCounts()
|
ipv4Routes, ipv6Routes, err := s.db.GetLiveRouteCountsContext(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Warn("Failed to get live route counts", "error", err)
|
s.logger.Warn("Failed to get live route counts", "error", err)
|
||||||
// Continue with zero counts
|
// Continue with zero counts
|
||||||
@@ -284,6 +425,7 @@ func (s *Server) handleStats() http.HandlerFunc {
|
|||||||
Name: hs.Name,
|
Name: hs.Name,
|
||||||
QueueLength: hs.QueueLength,
|
QueueLength: hs.QueueLength,
|
||||||
QueueCapacity: hs.QueueCapacity,
|
QueueCapacity: hs.QueueCapacity,
|
||||||
|
QueueHighWaterMark: hs.QueueHighWaterMark,
|
||||||
ProcessedCount: hs.ProcessedCount,
|
ProcessedCount: hs.ProcessedCount,
|
||||||
DroppedCount: hs.DroppedCount,
|
DroppedCount: hs.DroppedCount,
|
||||||
AvgProcessTimeMs: float64(hs.AvgProcessTime.Microseconds()) / microsecondsPerMillisecond,
|
AvgProcessTimeMs: float64(hs.AvgProcessTime.Microseconds()) / microsecondsPerMillisecond,
|
||||||
@@ -296,13 +438,29 @@ func (s *Server) handleStats() http.HandlerFunc {
|
|||||||
var memStats runtime.MemStats
|
var memStats runtime.MemStats
|
||||||
runtime.ReadMemStats(&memStats)
|
runtime.ReadMemStats(&memStats)
|
||||||
|
|
||||||
|
// Get WHOIS stats if fetcher is available
|
||||||
|
var whoisStats *WHOISStatsInfo
|
||||||
|
if s.asnFetcher != nil {
|
||||||
|
whoisStats = s.getWHOISStats(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate connection duration
|
||||||
|
connectionDuration := "disconnected"
|
||||||
|
if metrics.Connected && !metrics.ConnectedSince.IsZero() {
|
||||||
|
connectionDuration = time.Since(metrics.ConnectedSince).Truncate(time.Second).String()
|
||||||
|
}
|
||||||
|
|
||||||
stats := StatsResponse{
|
stats := StatsResponse{
|
||||||
Uptime: uptime,
|
Uptime: uptime,
|
||||||
TotalMessages: metrics.TotalMessages,
|
TotalMessages: metrics.TotalMessages,
|
||||||
TotalBytes: metrics.TotalBytes,
|
TotalBytes: metrics.TotalBytes,
|
||||||
|
TotalWireBytes: metrics.TotalWireBytes,
|
||||||
MessagesPerSec: metrics.MessagesPerSec,
|
MessagesPerSec: metrics.MessagesPerSec,
|
||||||
MbitsPerSec: metrics.BitsPerSec / bitsPerMegabit,
|
MbitsPerSec: metrics.BitsPerSec / bitsPerMegabit,
|
||||||
|
WireMbitsPerSec: metrics.WireBitsPerSec / bitsPerMegabit,
|
||||||
Connected: metrics.Connected,
|
Connected: metrics.Connected,
|
||||||
|
ConnectionDuration: connectionDuration,
|
||||||
|
ReconnectCount: metrics.ReconnectCount,
|
||||||
GoVersion: runtime.Version(),
|
GoVersion: runtime.Version(),
|
||||||
Goroutines: runtime.NumGoroutine(),
|
Goroutines: runtime.NumGoroutine(),
|
||||||
MemoryUsage: humanize.Bytes(memStats.Alloc),
|
MemoryUsage: humanize.Bytes(memStats.Alloc),
|
||||||
@@ -321,6 +479,7 @@ func (s *Server) handleStats() http.HandlerFunc {
|
|||||||
HandlerStats: handlerStatsInfo,
|
HandlerStats: handlerStatsInfo,
|
||||||
IPv4PrefixDistribution: dbStats.IPv4PrefixDistribution,
|
IPv4PrefixDistribution: dbStats.IPv4PrefixDistribution,
|
||||||
IPv6PrefixDistribution: dbStats.IPv6PrefixDistribution,
|
IPv6PrefixDistribution: dbStats.IPv6PrefixDistribution,
|
||||||
|
WHOISStats: whoisStats,
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := writeJSONSuccess(w, stats); err != nil {
|
if err := writeJSONSuccess(w, stats); err != nil {
|
||||||
@@ -329,7 +488,8 @@ func (s *Server) handleStats() http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleStatusHTML returns a handler that serves the HTML status page
|
// handleStatusHTML returns a handler that serves the HTML status page,
|
||||||
|
// which displays real-time statistics fetched via JavaScript.
|
||||||
func (s *Server) handleStatusHTML() http.HandlerFunc {
|
func (s *Server) handleStatusHTML() http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, _ *http.Request) {
|
return func(w http.ResponseWriter, _ *http.Request) {
|
||||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||||
@@ -344,33 +504,136 @@ func (s *Server) handleStatusHTML() http.HandlerFunc {
|
|||||||
|
|
||||||
// handleIPLookup returns a handler that looks up AS information for an IP address
|
// handleIPLookup returns a handler that looks up AS information for an IP address
|
||||||
func (s *Server) handleIPLookup() http.HandlerFunc {
|
func (s *Server) handleIPLookup() http.HandlerFunc {
|
||||||
|
return s.handleIPInfo()
|
||||||
|
}
|
||||||
|
|
||||||
|
// IPLookupResponse is the standard response for IP/hostname lookups.
|
||||||
|
type IPLookupResponse struct {
|
||||||
|
Query string `json:"query"`
|
||||||
|
Results []*database.IPInfo `json:"results"`
|
||||||
|
Errors []string `json:"errors,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleIPInfo returns a handler that provides comprehensive IP information.
|
||||||
|
// Used for /ip, /ip/{addr}, and /api/v1/ip/{ip} endpoints.
|
||||||
|
// Accepts IP addresses (single or comma-separated) and hostnames.
|
||||||
|
// Always returns the same response structure with PTR records for each IP.
|
||||||
|
func (s *Server) handleIPInfo() http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
ip := chi.URLParam(r, "ip")
|
// Get IP/hostname from URL param, falling back to client IP
|
||||||
if ip == "" {
|
target := chi.URLParam(r, "ip")
|
||||||
writeJSONError(w, http.StatusBadRequest, "IP parameter is required")
|
if target == "" {
|
||||||
|
target = chi.URLParam(r, "addr")
|
||||||
|
}
|
||||||
|
if target == "" {
|
||||||
|
// Use client IP (RealIP middleware has already processed this)
|
||||||
|
target = extractClientIP(r)
|
||||||
|
}
|
||||||
|
|
||||||
|
if target == "" {
|
||||||
|
writeJSONError(w, http.StatusBadRequest, "Could not determine IP address")
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Look up AS information for the IP
|
ctx := r.Context()
|
||||||
asInfo, err := s.db.GetASInfoForIP(ip)
|
response := IPLookupResponse{
|
||||||
if err != nil {
|
Query: target,
|
||||||
// Check if it's an invalid IP error
|
Results: make([]*database.IPInfo, 0),
|
||||||
if errors.Is(err, database.ErrInvalidIP) {
|
}
|
||||||
writeJSONError(w, http.StatusBadRequest, err.Error())
|
|
||||||
|
// Collect all IPs to look up
|
||||||
|
var ipsToLookup []string
|
||||||
|
|
||||||
|
// Check if target contains commas (multiple IPs)
|
||||||
|
targets := strings.Split(target, ",")
|
||||||
|
for _, t := range targets {
|
||||||
|
t = strings.TrimSpace(t)
|
||||||
|
if t == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if this target is an IP address
|
||||||
|
if parsedIP := net.ParseIP(t); parsedIP != nil {
|
||||||
|
ipsToLookup = append(ipsToLookup, t)
|
||||||
} else {
|
} else {
|
||||||
// All other errors (including ErrNoRoute) are 404
|
// It's a hostname - resolve it
|
||||||
writeJSONError(w, http.StatusNotFound, err.Error())
|
resolved, err := net.DefaultResolver.LookupHost(ctx, t)
|
||||||
|
if err != nil {
|
||||||
|
response.Errors = append(response.Errors, t+": "+err.Error())
|
||||||
|
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
ipsToLookup = append(ipsToLookup, resolved...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(ipsToLookup) == 0 {
|
||||||
|
writeJSONError(w, http.StatusBadRequest, "No valid IPs or hostnames provided")
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Return successful response
|
// Track ASNs that need WHOIS refresh
|
||||||
if err := writeJSONSuccess(w, asInfo); err != nil {
|
refreshASNs := make(map[int]bool)
|
||||||
s.logger.Error("Failed to encode AS info", "error", err)
|
|
||||||
|
// Look up each IP
|
||||||
|
for _, ip := range ipsToLookup {
|
||||||
|
ipInfo, err := s.db.GetIPInfoContext(ctx, ip)
|
||||||
|
if err != nil {
|
||||||
|
response.Errors = append(response.Errors, ip+": "+err.Error())
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Do PTR lookup for this IP
|
||||||
|
ptrs, err := net.DefaultResolver.LookupAddr(ctx, ip)
|
||||||
|
if err == nil && len(ptrs) > 0 {
|
||||||
|
// Remove trailing dots from PTR records
|
||||||
|
for i, ptr := range ptrs {
|
||||||
|
ptrs[i] = strings.TrimSuffix(ptr, ".")
|
||||||
|
}
|
||||||
|
ipInfo.PTR = ptrs
|
||||||
|
}
|
||||||
|
|
||||||
|
response.Results = append(response.Results, ipInfo)
|
||||||
|
|
||||||
|
if ipInfo.NeedsWHOISRefresh {
|
||||||
|
refreshASNs[ipInfo.ASN] = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Queue WHOIS refresh for stale ASNs (non-blocking)
|
||||||
|
if s.asnFetcher != nil {
|
||||||
|
for asn := range refreshASNs {
|
||||||
|
s.asnFetcher.QueueImmediate(asn)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return response (even if no results, include errors)
|
||||||
|
if len(response.Results) == 0 && len(response.Errors) > 0 {
|
||||||
|
writeJSONError(w, http.StatusNotFound, "No routes found: "+response.Errors[0])
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := writeJSONSuccess(w, response); err != nil {
|
||||||
|
s.logger.Error("Failed to encode IP lookup response", "error", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractClientIP extracts the client IP from the request.
|
||||||
|
// Works with chi's RealIP middleware which sets RemoteAddr.
|
||||||
|
func extractClientIP(r *http.Request) string {
|
||||||
|
// RemoteAddr is in the form "IP:port" or just "IP" for unix sockets
|
||||||
|
host, _, err := net.SplitHostPort(r.RemoteAddr)
|
||||||
|
if err != nil {
|
||||||
|
// Might be just an IP without port
|
||||||
|
return r.RemoteAddr
|
||||||
|
}
|
||||||
|
|
||||||
|
return host
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleASDetailJSON returns AS details as JSON
|
// handleASDetailJSON returns AS details as JSON
|
||||||
@@ -384,7 +647,7 @@ func (s *Server) handleASDetailJSON() http.HandlerFunc {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
asInfo, prefixes, err := s.db.GetASDetails(asn)
|
asInfo, prefixes, err := s.db.GetASDetailsContext(r.Context(), asn)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, database.ErrNoRoute) {
|
if errors.Is(err, database.ErrNoRoute) {
|
||||||
writeJSONError(w, http.StatusNotFound, err.Error())
|
writeJSONError(w, http.StatusNotFound, err.Error())
|
||||||
@@ -437,7 +700,7 @@ func (s *Server) handlePrefixDetailJSON() http.HandlerFunc {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
routes, err := s.db.GetPrefixDetails(prefix)
|
routes, err := s.db.GetPrefixDetailsContext(r.Context(), prefix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, database.ErrNoRoute) {
|
if errors.Is(err, database.ErrNoRoute) {
|
||||||
writeJSONError(w, http.StatusNotFound, err.Error())
|
writeJSONError(w, http.StatusNotFound, err.Error())
|
||||||
@@ -479,7 +742,7 @@ func (s *Server) handleASDetail() http.HandlerFunc {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
asInfo, prefixes, err := s.db.GetASDetails(asn)
|
asInfo, prefixes, err := s.db.GetASDetailsContext(r.Context(), asn)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, database.ErrNoRoute) {
|
if errors.Is(err, database.ErrNoRoute) {
|
||||||
http.Error(w, "AS not found", http.StatusNotFound)
|
http.Error(w, "AS not found", http.StatusNotFound)
|
||||||
@@ -491,6 +754,14 @@ func (s *Server) handleASDetail() http.HandlerFunc {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Get peers
|
||||||
|
peers, err := s.db.GetASPeersContext(r.Context(), asn)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Failed to get AS peers", "error", err)
|
||||||
|
// Continue without peers rather than failing the whole request
|
||||||
|
peers = []database.ASPeer{}
|
||||||
|
}
|
||||||
|
|
||||||
// Group prefixes by IP version
|
// Group prefixes by IP version
|
||||||
const ipVersionV4 = 4
|
const ipVersionV4 = 4
|
||||||
var ipv4Prefixes, ipv6Prefixes []database.LiveRoute
|
var ipv4Prefixes, ipv6Prefixes []database.LiveRoute
|
||||||
@@ -547,6 +818,8 @@ func (s *Server) handleASDetail() http.HandlerFunc {
|
|||||||
TotalCount int
|
TotalCount int
|
||||||
IPv4Count int
|
IPv4Count int
|
||||||
IPv6Count int
|
IPv6Count int
|
||||||
|
Peers []database.ASPeer
|
||||||
|
PeerCount int
|
||||||
}{
|
}{
|
||||||
ASN: asInfo,
|
ASN: asInfo,
|
||||||
IPv4Prefixes: ipv4Prefixes,
|
IPv4Prefixes: ipv4Prefixes,
|
||||||
@@ -554,6 +827,16 @@ func (s *Server) handleASDetail() http.HandlerFunc {
|
|||||||
TotalCount: len(prefixes),
|
TotalCount: len(prefixes),
|
||||||
IPv4Count: len(ipv4Prefixes),
|
IPv4Count: len(ipv4Prefixes),
|
||||||
IPv6Count: len(ipv6Prefixes),
|
IPv6Count: len(ipv6Prefixes),
|
||||||
|
Peers: peers,
|
||||||
|
PeerCount: len(peers),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if context is still valid before writing response
|
||||||
|
select {
|
||||||
|
case <-r.Context().Done():
|
||||||
|
// Request was cancelled, don't write response
|
||||||
|
return
|
||||||
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||||
@@ -583,7 +866,7 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
routes, err := s.db.GetPrefixDetails(prefix)
|
routes, err := s.db.GetPrefixDetailsContext(r.Context(), prefix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, database.ErrNoRoute) {
|
if errors.Is(err, database.ErrNoRoute) {
|
||||||
http.Error(w, "Prefix not found", http.StatusNotFound)
|
http.Error(w, "Prefix not found", http.StatusNotFound)
|
||||||
@@ -597,7 +880,7 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc {
|
|||||||
|
|
||||||
// Group by origin AS and collect unique AS info
|
// Group by origin AS and collect unique AS info
|
||||||
type ASNInfo struct {
|
type ASNInfo struct {
|
||||||
Number int
|
ASN int
|
||||||
Handle string
|
Handle string
|
||||||
Description string
|
Description string
|
||||||
PeerCount int
|
PeerCount int
|
||||||
@@ -606,7 +889,7 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc {
|
|||||||
for _, route := range routes {
|
for _, route := range routes {
|
||||||
if _, exists := originMap[route.OriginASN]; !exists {
|
if _, exists := originMap[route.OriginASN]; !exists {
|
||||||
// Get AS info from database
|
// Get AS info from database
|
||||||
asInfo, _, _ := s.db.GetASDetails(route.OriginASN)
|
asInfo, _, _ := s.db.GetASDetailsContext(r.Context(), route.OriginASN)
|
||||||
handle := ""
|
handle := ""
|
||||||
description := ""
|
description := ""
|
||||||
if asInfo != nil {
|
if asInfo != nil {
|
||||||
@@ -614,7 +897,7 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc {
|
|||||||
description = asInfo.Description
|
description = asInfo.Description
|
||||||
}
|
}
|
||||||
originMap[route.OriginASN] = &ASNInfo{
|
originMap[route.OriginASN] = &ASNInfo{
|
||||||
Number: route.OriginASN,
|
ASN: route.OriginASN,
|
||||||
Handle: handle,
|
Handle: handle,
|
||||||
Description: description,
|
Description: description,
|
||||||
PeerCount: 0,
|
PeerCount: 0,
|
||||||
@@ -645,12 +928,41 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc {
|
|||||||
origins = append(origins, origin)
|
origins = append(origins, origin)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Create enhanced routes with AS path handles
|
||||||
|
type ASPathEntry struct {
|
||||||
|
ASN int
|
||||||
|
Handle string
|
||||||
|
}
|
||||||
|
type EnhancedRoute struct {
|
||||||
|
database.LiveRoute
|
||||||
|
ASPathWithHandle []ASPathEntry
|
||||||
|
}
|
||||||
|
|
||||||
|
enhancedRoutes := make([]EnhancedRoute, len(routes))
|
||||||
|
for i, route := range routes {
|
||||||
|
enhancedRoute := EnhancedRoute{
|
||||||
|
LiveRoute: route,
|
||||||
|
ASPathWithHandle: make([]ASPathEntry, len(route.ASPath)),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Look up handle for each AS in the path
|
||||||
|
for j, asn := range route.ASPath {
|
||||||
|
handle := asinfo.GetHandle(asn)
|
||||||
|
enhancedRoute.ASPathWithHandle[j] = ASPathEntry{
|
||||||
|
ASN: asn,
|
||||||
|
Handle: handle,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
enhancedRoutes[i] = enhancedRoute
|
||||||
|
}
|
||||||
|
|
||||||
// Prepare template data
|
// Prepare template data
|
||||||
data := struct {
|
data := struct {
|
||||||
Prefix string
|
Prefix string
|
||||||
MaskLength int
|
MaskLength int
|
||||||
IPVersion int
|
IPVersion int
|
||||||
Routes []database.LiveRoute
|
Routes []EnhancedRoute
|
||||||
Origins []*ASNInfo
|
Origins []*ASNInfo
|
||||||
PeerCount int
|
PeerCount int
|
||||||
OriginCount int
|
OriginCount int
|
||||||
@@ -658,12 +970,20 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc {
|
|||||||
Prefix: prefix,
|
Prefix: prefix,
|
||||||
MaskLength: maskLength,
|
MaskLength: maskLength,
|
||||||
IPVersion: ipVersion,
|
IPVersion: ipVersion,
|
||||||
Routes: routes,
|
Routes: enhancedRoutes,
|
||||||
Origins: origins,
|
Origins: origins,
|
||||||
PeerCount: len(routes),
|
PeerCount: len(routes),
|
||||||
OriginCount: len(originMap),
|
OriginCount: len(originMap),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Check if context is still valid before writing response
|
||||||
|
select {
|
||||||
|
case <-r.Context().Done():
|
||||||
|
// Request was cancelled, don't write response
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||||
tmpl := templates.PrefixDetailTemplate()
|
tmpl := templates.PrefixDetailTemplate()
|
||||||
if err := tmpl.Execute(w, data); err != nil {
|
if err := tmpl.Execute(w, data); err != nil {
|
||||||
@@ -673,32 +993,207 @@ func (s *Server) handlePrefixDetail() http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleIPRedirect looks up the prefix containing the IP and redirects to its detail page
|
// handlePrefixLength shows a random sample of IPv4 prefixes with the specified mask length
|
||||||
func (s *Server) handleIPRedirect() http.HandlerFunc {
|
func (s *Server) handlePrefixLength() http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
ip := chi.URLParam(r, "ip")
|
lengthStr := chi.URLParam(r, "length")
|
||||||
if ip == "" {
|
if lengthStr == "" {
|
||||||
http.Error(w, "IP parameter is required", http.StatusBadRequest)
|
http.Error(w, "Length parameter is required", http.StatusBadRequest)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Look up AS information for the IP (which includes the prefix)
|
maskLength, err := strconv.Atoi(lengthStr)
|
||||||
asInfo, err := s.db.GetASInfoForIP(ip)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, database.ErrInvalidIP) {
|
http.Error(w, "Invalid mask length", http.StatusBadRequest)
|
||||||
http.Error(w, "Invalid IP address", http.StatusBadRequest)
|
|
||||||
} else if errors.Is(err, database.ErrNoRoute) {
|
|
||||||
http.Error(w, "No route found for this IP", http.StatusNotFound)
|
|
||||||
} else {
|
|
||||||
s.logger.Error("Failed to look up IP", "error", err)
|
|
||||||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Redirect to the prefix detail page (URL encode the prefix)
|
// Validate IPv4 mask length
|
||||||
http.Redirect(w, r, "/prefix/"+url.QueryEscape(asInfo.Prefix), http.StatusSeeOther)
|
const maxIPv4MaskLength = 32
|
||||||
|
if maskLength < 0 || maskLength > maxIPv4MaskLength {
|
||||||
|
http.Error(w, "Invalid IPv4 mask length", http.StatusBadRequest)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const ipVersion = 4
|
||||||
|
|
||||||
|
// Get random sample of prefixes
|
||||||
|
const maxPrefixes = 500
|
||||||
|
prefixes, err := s.db.GetRandomPrefixesByLengthContext(r.Context(), maskLength, ipVersion, maxPrefixes)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Failed to get prefixes by length", "error", err)
|
||||||
|
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort prefixes for display
|
||||||
|
sort.Slice(prefixes, func(i, j int) bool {
|
||||||
|
// First compare by IP version
|
||||||
|
if prefixes[i].IPVersion != prefixes[j].IPVersion {
|
||||||
|
return prefixes[i].IPVersion < prefixes[j].IPVersion
|
||||||
|
}
|
||||||
|
// Then by prefix
|
||||||
|
return prefixes[i].Prefix < prefixes[j].Prefix
|
||||||
|
})
|
||||||
|
|
||||||
|
// Create enhanced prefixes with AS descriptions
|
||||||
|
type EnhancedPrefix struct {
|
||||||
|
database.LiveRoute
|
||||||
|
OriginASDescription string
|
||||||
|
Age string
|
||||||
|
}
|
||||||
|
|
||||||
|
enhancedPrefixes := make([]EnhancedPrefix, len(prefixes))
|
||||||
|
for i, prefix := range prefixes {
|
||||||
|
enhancedPrefixes[i] = EnhancedPrefix{
|
||||||
|
LiveRoute: prefix,
|
||||||
|
Age: formatAge(prefix.LastUpdated),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get AS description
|
||||||
|
if asInfo, ok := asinfo.Get(prefix.OriginASN); ok {
|
||||||
|
enhancedPrefixes[i].OriginASDescription = asInfo.Description
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Render template
|
||||||
|
data := map[string]interface{}{
|
||||||
|
"MaskLength": maskLength,
|
||||||
|
"IPVersion": ipVersion,
|
||||||
|
"Prefixes": enhancedPrefixes,
|
||||||
|
"Count": len(prefixes),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if context is still valid before writing response
|
||||||
|
select {
|
||||||
|
case <-r.Context().Done():
|
||||||
|
// Request was cancelled, don't write response
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
tmpl := templates.PrefixLengthTemplate()
|
||||||
|
if err := tmpl.Execute(w, data); err != nil {
|
||||||
|
s.logger.Error("Failed to render prefix length template", "error", err)
|
||||||
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// handlePrefixLength6 shows a random sample of IPv6 prefixes with the specified mask length
|
||||||
|
func (s *Server) handlePrefixLength6() http.HandlerFunc {
|
||||||
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
lengthStr := chi.URLParam(r, "length")
|
||||||
|
if lengthStr == "" {
|
||||||
|
http.Error(w, "Length parameter is required", http.StatusBadRequest)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
maskLength, err := strconv.Atoi(lengthStr)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, "Invalid mask length", http.StatusBadRequest)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate IPv6 mask length
|
||||||
|
const maxIPv6MaskLength = 128
|
||||||
|
if maskLength < 0 || maskLength > maxIPv6MaskLength {
|
||||||
|
http.Error(w, "Invalid IPv6 mask length", http.StatusBadRequest)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const ipVersion = 6
|
||||||
|
|
||||||
|
// Get random sample of prefixes
|
||||||
|
const maxPrefixes = 500
|
||||||
|
prefixes, err := s.db.GetRandomPrefixesByLengthContext(r.Context(), maskLength, ipVersion, maxPrefixes)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Failed to get prefixes by length", "error", err)
|
||||||
|
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort prefixes for display
|
||||||
|
sort.Slice(prefixes, func(i, j int) bool {
|
||||||
|
// First compare by IP version
|
||||||
|
if prefixes[i].IPVersion != prefixes[j].IPVersion {
|
||||||
|
return prefixes[i].IPVersion < prefixes[j].IPVersion
|
||||||
|
}
|
||||||
|
// Then by prefix
|
||||||
|
return prefixes[i].Prefix < prefixes[j].Prefix
|
||||||
|
})
|
||||||
|
|
||||||
|
// Create enhanced prefixes with AS descriptions
|
||||||
|
type EnhancedPrefix struct {
|
||||||
|
database.LiveRoute
|
||||||
|
OriginASDescription string
|
||||||
|
Age string
|
||||||
|
}
|
||||||
|
|
||||||
|
enhancedPrefixes := make([]EnhancedPrefix, len(prefixes))
|
||||||
|
for i, prefix := range prefixes {
|
||||||
|
enhancedPrefixes[i] = EnhancedPrefix{
|
||||||
|
LiveRoute: prefix,
|
||||||
|
Age: formatAge(prefix.LastUpdated),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get AS description
|
||||||
|
if asInfo, ok := asinfo.Get(prefix.OriginASN); ok {
|
||||||
|
enhancedPrefixes[i].OriginASDescription = asInfo.Description
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Render template
|
||||||
|
data := map[string]interface{}{
|
||||||
|
"MaskLength": maskLength,
|
||||||
|
"IPVersion": ipVersion,
|
||||||
|
"Prefixes": enhancedPrefixes,
|
||||||
|
"Count": len(prefixes),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if context is still valid before writing response
|
||||||
|
select {
|
||||||
|
case <-r.Context().Done():
|
||||||
|
// Request was cancelled, don't write response
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
tmpl := templates.PrefixLengthTemplate()
|
||||||
|
if err := tmpl.Execute(w, data); err != nil {
|
||||||
|
s.logger.Error("Failed to render prefix length template", "error", err)
|
||||||
|
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// formatAge returns a human-readable age string
|
||||||
|
func formatAge(timestamp time.Time) string {
|
||||||
|
age := time.Since(timestamp)
|
||||||
|
|
||||||
|
const hoursPerDay = 24
|
||||||
|
|
||||||
|
if age < time.Minute {
|
||||||
|
return "< 1m"
|
||||||
|
} else if age < time.Hour {
|
||||||
|
minutes := int(age.Minutes())
|
||||||
|
|
||||||
|
return strconv.Itoa(minutes) + "m"
|
||||||
|
} else if age < hoursPerDay*time.Hour {
|
||||||
|
hours := int(age.Hours())
|
||||||
|
|
||||||
|
return strconv.Itoa(hours) + "h"
|
||||||
|
}
|
||||||
|
|
||||||
|
days := int(age.Hours() / hoursPerDay)
|
||||||
|
|
||||||
|
return strconv.Itoa(days) + "d"
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@@ -44,7 +45,12 @@ func (rw *responseWriter) Header() http.Header {
|
|||||||
return rw.ResponseWriter.Header()
|
return rw.ResponseWriter.Header()
|
||||||
}
|
}
|
||||||
|
|
||||||
// JSONResponseMiddleware wraps all JSON responses with metadata
|
// JSONResponseMiddleware is an HTTP middleware that wraps all JSON responses
|
||||||
|
// with a @meta field containing execution metadata. The metadata includes the
|
||||||
|
// time zone (always UTC), API version, and request execution time in milliseconds.
|
||||||
|
//
|
||||||
|
// Endpoints "/" and "/status" are excluded from this processing and passed through
|
||||||
|
// unchanged. Non-JSON responses and empty responses are also passed through unchanged.
|
||||||
func JSONResponseMiddleware(next http.Handler) http.Handler {
|
func JSONResponseMiddleware(next http.Handler) http.Handler {
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
// Skip non-JSON endpoints
|
// Skip non-JSON endpoints
|
||||||
@@ -108,6 +114,7 @@ type timeoutWriter struct {
|
|||||||
http.ResponseWriter
|
http.ResponseWriter
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
written bool
|
written bool
|
||||||
|
header http.Header // cached header to prevent concurrent access
|
||||||
}
|
}
|
||||||
|
|
||||||
func (tw *timeoutWriter) Write(b []byte) (int, error) {
|
func (tw *timeoutWriter) Write(b []byte) (int, error) {
|
||||||
@@ -133,6 +140,18 @@ func (tw *timeoutWriter) WriteHeader(statusCode int) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (tw *timeoutWriter) Header() http.Header {
|
func (tw *timeoutWriter) Header() http.Header {
|
||||||
|
tw.mu.Lock()
|
||||||
|
defer tw.mu.Unlock()
|
||||||
|
|
||||||
|
if tw.written {
|
||||||
|
// Return a copy to prevent modifications after timeout
|
||||||
|
if tw.header == nil {
|
||||||
|
tw.header = make(http.Header)
|
||||||
|
}
|
||||||
|
|
||||||
|
return tw.header
|
||||||
|
}
|
||||||
|
|
||||||
return tw.ResponseWriter.Header()
|
return tw.ResponseWriter.Header()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -142,7 +161,13 @@ func (tw *timeoutWriter) markWritten() {
|
|||||||
tw.written = true
|
tw.written = true
|
||||||
}
|
}
|
||||||
|
|
||||||
// TimeoutMiddleware creates a timeout middleware that returns JSON errors
|
// TimeoutMiddleware creates an HTTP middleware that enforces a request timeout.
|
||||||
|
// If the handler does not complete within the specified duration, the middleware
|
||||||
|
// returns a JSON error response with HTTP status 408 (Request Timeout).
|
||||||
|
//
|
||||||
|
// The timeout parameter specifies the maximum duration allowed for request processing.
|
||||||
|
// The returned middleware handles panics from the wrapped handler by re-panicking
|
||||||
|
// after cleanup, and prevents concurrent writes to the response after timeout occurs.
|
||||||
func TimeoutMiddleware(timeout time.Duration) func(http.Handler) http.Handler {
|
func TimeoutMiddleware(timeout time.Duration) func(http.Handler) http.Handler {
|
||||||
return func(next http.Handler) http.Handler {
|
return func(next http.Handler) http.Handler {
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -153,6 +178,7 @@ func TimeoutMiddleware(timeout time.Duration) func(http.Handler) http.Handler {
|
|||||||
|
|
||||||
tw := &timeoutWriter{
|
tw := &timeoutWriter{
|
||||||
ResponseWriter: w,
|
ResponseWriter: w,
|
||||||
|
header: make(http.Header),
|
||||||
}
|
}
|
||||||
|
|
||||||
done := make(chan struct{})
|
done := make(chan struct{})
|
||||||
@@ -178,8 +204,12 @@ func TimeoutMiddleware(timeout time.Duration) func(http.Handler) http.Handler {
|
|||||||
tw.markWritten() // Prevent the handler from writing after timeout
|
tw.markWritten() // Prevent the handler from writing after timeout
|
||||||
execTime := time.Since(startTime)
|
execTime := time.Since(startTime)
|
||||||
|
|
||||||
|
// Write directly to the underlying writer since we've marked tw as written
|
||||||
|
// This is safe because markWritten() prevents the handler from writing
|
||||||
|
tw.mu.Lock()
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
w.WriteHeader(http.StatusRequestTimeout)
|
w.WriteHeader(http.StatusRequestTimeout)
|
||||||
|
tw.mu.Unlock()
|
||||||
|
|
||||||
response := map[string]interface{}{
|
response := map[string]interface{}{
|
||||||
"status": "error",
|
"status": "error",
|
||||||
@@ -199,3 +229,140 @@ func TimeoutMiddleware(timeout time.Duration) func(http.Handler) http.Handler {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// JSONValidationMiddleware is an HTTP middleware that validates JSON API responses.
|
||||||
|
// It ensures that responses with Content-Type "application/json" contain valid JSON.
|
||||||
|
//
|
||||||
|
// If a response is not valid JSON or is empty when JSON is expected, the middleware
|
||||||
|
// returns a properly formatted JSON error response. For timeout errors (status 408),
|
||||||
|
// the error message will be "Request timeout". For other errors, it returns
|
||||||
|
// "Internal server error" with status 500 if the original status was 200.
|
||||||
|
//
|
||||||
|
// Non-JSON responses are passed through unchanged.
|
||||||
|
func JSONValidationMiddleware(next http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
// Create a custom response writer to capture the response
|
||||||
|
rw := &responseWriter{
|
||||||
|
ResponseWriter: w,
|
||||||
|
body: &bytes.Buffer{},
|
||||||
|
statusCode: http.StatusOK,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Serve the request
|
||||||
|
next.ServeHTTP(rw, r)
|
||||||
|
|
||||||
|
// Check if it's meant to be a JSON response
|
||||||
|
contentType := rw.Header().Get("Content-Type")
|
||||||
|
isJSON := contentType == "application/json" || contentType == ""
|
||||||
|
|
||||||
|
// If it's not JSON or has content, pass through
|
||||||
|
if !isJSON && rw.body.Len() > 0 {
|
||||||
|
w.WriteHeader(rw.statusCode)
|
||||||
|
_, _ = w.Write(rw.body.Bytes())
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// For JSON responses, validate the JSON
|
||||||
|
if rw.body.Len() > 0 {
|
||||||
|
var testParse interface{}
|
||||||
|
if err := json.Unmarshal(rw.body.Bytes(), &testParse); err == nil {
|
||||||
|
// Valid JSON, write it out
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.WriteHeader(rw.statusCode)
|
||||||
|
_, _ = w.Write(rw.body.Bytes())
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If we get here, either there's no body or invalid JSON
|
||||||
|
// Write a proper error response
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
// Determine appropriate status code
|
||||||
|
statusCode := rw.statusCode
|
||||||
|
if statusCode == http.StatusOK {
|
||||||
|
statusCode = http.StatusInternalServerError
|
||||||
|
}
|
||||||
|
w.WriteHeader(statusCode)
|
||||||
|
|
||||||
|
errorMsg := "Internal server error"
|
||||||
|
if statusCode == http.StatusRequestTimeout {
|
||||||
|
errorMsg = "Request timeout"
|
||||||
|
}
|
||||||
|
|
||||||
|
response := map[string]interface{}{
|
||||||
|
"status": "error",
|
||||||
|
"error": map[string]interface{}{
|
||||||
|
"msg": errorMsg,
|
||||||
|
"code": statusCode,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
_ = json.NewEncoder(w).Encode(response)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// statusWriter wraps http.ResponseWriter to capture the status code
|
||||||
|
type statusWriter struct {
|
||||||
|
http.ResponseWriter
|
||||||
|
statusCode int
|
||||||
|
written bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sw *statusWriter) WriteHeader(statusCode int) {
|
||||||
|
if !sw.written {
|
||||||
|
sw.statusCode = statusCode
|
||||||
|
sw.written = true
|
||||||
|
}
|
||||||
|
sw.ResponseWriter.WriteHeader(statusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sw *statusWriter) Write(b []byte) (int, error) {
|
||||||
|
if !sw.written {
|
||||||
|
sw.statusCode = http.StatusOK
|
||||||
|
sw.written = true
|
||||||
|
}
|
||||||
|
|
||||||
|
return sw.ResponseWriter.Write(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequestLoggerMiddleware creates a structured logging middleware using slog.
|
||||||
|
func RequestLoggerMiddleware(logger *slog.Logger) func(http.Handler) http.Handler {
|
||||||
|
return func(next http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
start := time.Now()
|
||||||
|
|
||||||
|
// Wrap response writer to capture status
|
||||||
|
sw := &statusWriter{ResponseWriter: w, statusCode: http.StatusOK}
|
||||||
|
|
||||||
|
// Log request start
|
||||||
|
logger.Debug("HTTP request started",
|
||||||
|
"method", r.Method,
|
||||||
|
"path", r.URL.Path,
|
||||||
|
"remote_addr", r.RemoteAddr,
|
||||||
|
"user_agent", r.UserAgent(),
|
||||||
|
)
|
||||||
|
|
||||||
|
// Serve the request
|
||||||
|
next.ServeHTTP(sw, r)
|
||||||
|
|
||||||
|
// Log request completion
|
||||||
|
duration := time.Since(start)
|
||||||
|
logLevel := slog.LevelInfo
|
||||||
|
if sw.statusCode >= http.StatusInternalServerError {
|
||||||
|
logLevel = slog.LevelError
|
||||||
|
} else if sw.statusCode >= http.StatusBadRequest {
|
||||||
|
logLevel = slog.LevelWarn
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Log(r.Context(), logLevel, "HTTP request completed",
|
||||||
|
"method", r.Method,
|
||||||
|
"path", r.URL.Path,
|
||||||
|
"status", sw.statusCode,
|
||||||
|
"duration_ms", duration.Milliseconds(),
|
||||||
|
"remote_addr", r.RemoteAddr,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -14,24 +14,34 @@ func (s *Server) setupRoutes() {
|
|||||||
// Middleware
|
// Middleware
|
||||||
r.Use(middleware.RequestID)
|
r.Use(middleware.RequestID)
|
||||||
r.Use(middleware.RealIP)
|
r.Use(middleware.RealIP)
|
||||||
r.Use(middleware.Logger)
|
r.Use(RequestLoggerMiddleware(s.logger.Logger)) // Structured request logging
|
||||||
r.Use(middleware.Recoverer)
|
r.Use(middleware.Recoverer)
|
||||||
const requestTimeout = 2 * time.Second
|
const requestTimeout = 30 * time.Second // Increased from 8s for slow queries
|
||||||
r.Use(TimeoutMiddleware(requestTimeout))
|
r.Use(TimeoutMiddleware(requestTimeout))
|
||||||
r.Use(JSONResponseMiddleware)
|
r.Use(JSONResponseMiddleware)
|
||||||
|
|
||||||
// Routes
|
// Routes
|
||||||
r.Get("/", s.handleRoot())
|
r.Get("/", s.handleRoot())
|
||||||
r.Get("/status", s.handleStatusHTML())
|
r.Get("/status", s.handleStatusHTML())
|
||||||
r.Get("/status.json", s.handleStatusJSON())
|
r.Get("/status.json", JSONValidationMiddleware(s.handleStatusJSON()).ServeHTTP)
|
||||||
|
r.Get("/.well-known/healthcheck.json", JSONValidationMiddleware(s.handleHealthCheck()).ServeHTTP)
|
||||||
|
|
||||||
// AS and prefix detail pages
|
// AS and prefix detail pages
|
||||||
r.Get("/as/{asn}", s.handleASDetail())
|
r.Get("/as/{asn}", s.handleASDetail())
|
||||||
r.Get("/prefix/{prefix}", s.handlePrefixDetail())
|
r.Get("/prefix/{prefix}", s.handlePrefixDetail())
|
||||||
r.Get("/ip/{ip}", s.handleIPRedirect())
|
r.Get("/prefixlength/{length}", s.handlePrefixLength())
|
||||||
|
r.Get("/prefixlength6/{length}", s.handlePrefixLength6())
|
||||||
|
|
||||||
|
// IP info JSON endpoints (replaces old /ip redirect)
|
||||||
|
r.Route("/ip", func(r chi.Router) {
|
||||||
|
r.Use(JSONValidationMiddleware)
|
||||||
|
r.Get("/", s.handleIPInfo()) // Client IP
|
||||||
|
r.Get("/{addr}", s.handleIPInfo()) // Specified IP
|
||||||
|
})
|
||||||
|
|
||||||
// API routes
|
// API routes
|
||||||
r.Route("/api/v1", func(r chi.Router) {
|
r.Route("/api/v1", func(r chi.Router) {
|
||||||
|
r.Use(JSONValidationMiddleware)
|
||||||
r.Get("/stats", s.handleStats())
|
r.Get("/stats", s.handleStats())
|
||||||
r.Get("/ip/{ip}", s.handleIPLookup())
|
r.Get("/ip/{ip}", s.handleIPLookup())
|
||||||
r.Get("/as/{asn}", s.handleASDetailJSON())
|
r.Get("/as/{asn}", s.handleASDetailJSON())
|
||||||
|
|||||||
@@ -13,6 +13,20 @@ import (
|
|||||||
"github.com/go-chi/chi/v5"
|
"github.com/go-chi/chi/v5"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ASNFetcherStats contains WHOIS fetcher statistics.
|
||||||
|
type ASNFetcherStats struct {
|
||||||
|
SuccessesLastHour int
|
||||||
|
ErrorsLastHour int
|
||||||
|
CurrentInterval time.Duration
|
||||||
|
ConsecutiveFails int
|
||||||
|
}
|
||||||
|
|
||||||
|
// ASNFetcher is an interface for queuing ASN WHOIS lookups.
|
||||||
|
type ASNFetcher interface {
|
||||||
|
QueueImmediate(asn int)
|
||||||
|
GetStats() ASNFetcherStats
|
||||||
|
}
|
||||||
|
|
||||||
// Server provides HTTP endpoints for status monitoring
|
// Server provides HTTP endpoints for status monitoring
|
||||||
type Server struct {
|
type Server struct {
|
||||||
router *chi.Mux
|
router *chi.Mux
|
||||||
@@ -20,6 +34,7 @@ type Server struct {
|
|||||||
streamer *streamer.Streamer
|
streamer *streamer.Streamer
|
||||||
logger *logger.Logger
|
logger *logger.Logger
|
||||||
srv *http.Server
|
srv *http.Server
|
||||||
|
asnFetcher ASNFetcher
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new HTTP server
|
// New creates a new HTTP server
|
||||||
@@ -42,16 +57,27 @@ func (s *Server) Start() error {
|
|||||||
port = "8080"
|
port = "8080"
|
||||||
}
|
}
|
||||||
|
|
||||||
const readHeaderTimeout = 10 * time.Second
|
const (
|
||||||
|
readHeaderTimeout = 40 * time.Second
|
||||||
|
readTimeout = 60 * time.Second
|
||||||
|
writeTimeout = 60 * time.Second
|
||||||
|
idleTimeout = 120 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
s.srv = &http.Server{
|
s.srv = &http.Server{
|
||||||
Addr: ":" + port,
|
Addr: ":" + port,
|
||||||
Handler: s.router,
|
Handler: s.router,
|
||||||
ReadHeaderTimeout: readHeaderTimeout,
|
ReadHeaderTimeout: readHeaderTimeout,
|
||||||
|
ReadTimeout: readTimeout,
|
||||||
|
WriteTimeout: writeTimeout,
|
||||||
|
IdleTimeout: idleTimeout,
|
||||||
}
|
}
|
||||||
|
|
||||||
s.logger.Info("Starting HTTP server", "port", port)
|
s.logger.Info("Starting HTTP server", "port", port, "addr", s.srv.Addr)
|
||||||
|
|
||||||
|
// Start in goroutine but log when actually listening
|
||||||
go func() {
|
go func() {
|
||||||
|
s.logger.Info("HTTP server listening", "addr", s.srv.Addr)
|
||||||
if err := s.srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
if err := s.srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||||
s.logger.Error("HTTP server error", "error", err)
|
s.logger.Error("HTTP server error", "error", err)
|
||||||
}
|
}
|
||||||
@@ -70,3 +96,8 @@ func (s *Server) Stop(ctx context.Context) error {
|
|||||||
|
|
||||||
return s.srv.Shutdown(ctx)
|
return s.srv.Shutdown(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetASNFetcher sets the ASN WHOIS fetcher for on-demand lookups.
|
||||||
|
func (s *Server) SetASNFetcher(fetcher ASNFetcher) {
|
||||||
|
s.asnFetcher = fetcher
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,11 +4,14 @@ package streamer
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
|
"compress/gzip"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"math"
|
||||||
|
"math/rand"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
@@ -18,30 +21,64 @@ import (
|
|||||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// countingReader wraps an io.Reader and counts bytes read
|
||||||
|
type countingReader struct {
|
||||||
|
reader io.Reader
|
||||||
|
count int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read implements io.Reader and counts bytes
|
||||||
|
func (c *countingReader) Read(p []byte) (int, error) {
|
||||||
|
n, err := c.reader.Read(p)
|
||||||
|
atomic.AddInt64(&c.count, int64(n))
|
||||||
|
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count returns the total bytes read
|
||||||
|
func (c *countingReader) Count() int64 {
|
||||||
|
return atomic.LoadInt64(&c.count)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Configuration constants for the RIS Live streamer.
|
||||||
const (
|
const (
|
||||||
risLiveURL = "https://ris-live.ripe.net/v1/stream/?format=json"
|
risLiveURL = "https://ris-live.ripe.net/v1/stream/?format=json&" +
|
||||||
|
"client=https%3A%2F%2Fgit.eeqj.de%2Fsneak%2Froutewatch"
|
||||||
metricsWindowSize = 60 // seconds for rolling average
|
metricsWindowSize = 60 // seconds for rolling average
|
||||||
metricsUpdateRate = time.Second
|
metricsUpdateRate = time.Second
|
||||||
|
minBackoffDelay = 5 * time.Second
|
||||||
|
maxBackoffDelay = 320 * time.Second
|
||||||
metricsLogInterval = 10 * time.Second
|
metricsLogInterval = 10 * time.Second
|
||||||
bytesPerKB = 1024
|
bytesPerKB = 1024
|
||||||
bytesPerMB = 1024 * 1024
|
bytesPerMB = 1024 * 1024
|
||||||
maxConcurrentHandlers = 800 // Maximum number of concurrent message handlers
|
maxConcurrentHandlers = 800 // Maximum number of concurrent message handlers
|
||||||
|
|
||||||
|
// Backpressure constants
|
||||||
|
backpressureThreshold = 0.5 // Start dropping at 50% queue utilization
|
||||||
|
backpressureSlope = 2.0 // Slope for linear drop probability increase
|
||||||
)
|
)
|
||||||
|
|
||||||
// MessageHandler is an interface for handling RIS messages
|
// MessageHandler defines the interface for processing RIS messages.
|
||||||
|
// Implementations must specify which message types they want to receive,
|
||||||
|
// how to process messages, and their desired queue capacity.
|
||||||
type MessageHandler interface {
|
type MessageHandler interface {
|
||||||
// WantsMessage returns true if this handler wants to process messages of the given type
|
// WantsMessage returns true if this handler wants to process messages of the given type.
|
||||||
WantsMessage(messageType string) bool
|
WantsMessage(messageType string) bool
|
||||||
|
|
||||||
// HandleMessage processes a RIS message
|
// HandleMessage processes a RIS message. This method is called from a dedicated
|
||||||
|
// goroutine for each handler, so implementations do not need to be goroutine-safe
|
||||||
|
// with respect to other handlers.
|
||||||
HandleMessage(msg *ristypes.RISMessage)
|
HandleMessage(msg *ristypes.RISMessage)
|
||||||
|
|
||||||
// QueueCapacity returns the desired queue capacity for this handler
|
// QueueCapacity returns the desired queue capacity for this handler.
|
||||||
// Handlers that process quickly can have larger queues
|
// Handlers that process quickly can have larger queues to buffer bursts.
|
||||||
|
// When the queue fills up, messages will be dropped according to the
|
||||||
|
// backpressure algorithm.
|
||||||
QueueCapacity() int
|
QueueCapacity() int
|
||||||
}
|
}
|
||||||
|
|
||||||
// RawMessageHandler is a callback for handling raw JSON lines from the stream
|
// RawMessageHandler is a function type for processing raw JSON lines from the stream.
|
||||||
|
// It receives the unmodified JSON line as a string before any parsing occurs.
|
||||||
type RawMessageHandler func(line string)
|
type RawMessageHandler func(line string)
|
||||||
|
|
||||||
// handlerMetrics tracks performance metrics for a handler
|
// handlerMetrics tracks performance metrics for a handler
|
||||||
@@ -51,6 +88,7 @@ type handlerMetrics struct {
|
|||||||
totalTime time.Duration // Total processing time (for average calculation)
|
totalTime time.Duration // Total processing time (for average calculation)
|
||||||
minTime time.Duration // Minimum processing time
|
minTime time.Duration // Minimum processing time
|
||||||
maxTime time.Duration // Maximum processing time
|
maxTime time.Duration // Maximum processing time
|
||||||
|
queueHighWaterMark int // Maximum queue length seen
|
||||||
mu sync.Mutex // Protects the metrics
|
mu sync.Mutex // Protects the metrics
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -61,7 +99,10 @@ type handlerInfo struct {
|
|||||||
metrics handlerMetrics
|
metrics handlerMetrics
|
||||||
}
|
}
|
||||||
|
|
||||||
// Streamer handles streaming BGP updates from RIS Live
|
// Streamer manages a connection to the RIPE RIS Live streaming API for receiving
|
||||||
|
// real-time BGP UPDATE messages. It handles automatic reconnection with exponential
|
||||||
|
// backoff, dispatches messages to registered handlers via per-handler queues, and
|
||||||
|
// implements backpressure to prevent queue overflow during high traffic periods.
|
||||||
type Streamer struct {
|
type Streamer struct {
|
||||||
logger *logger.Logger
|
logger *logger.Logger
|
||||||
client *http.Client
|
client *http.Client
|
||||||
@@ -72,21 +113,32 @@ type Streamer struct {
|
|||||||
running bool
|
running bool
|
||||||
metrics *metrics.Tracker
|
metrics *metrics.Tracker
|
||||||
totalDropped uint64 // Total dropped messages across all handlers
|
totalDropped uint64 // Total dropped messages across all handlers
|
||||||
|
random *rand.Rand // Random number generator for backpressure drops
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new RIS streamer
|
// New creates a new Streamer instance configured to connect to the RIS Live API.
|
||||||
|
// The logger is used for structured logging of connection events and errors.
|
||||||
|
// The metrics tracker is used to record message counts, bytes received, and connection status.
|
||||||
func New(logger *logger.Logger, metrics *metrics.Tracker) *Streamer {
|
func New(logger *logger.Logger, metrics *metrics.Tracker) *Streamer {
|
||||||
return &Streamer{
|
return &Streamer{
|
||||||
logger: logger,
|
logger: logger,
|
||||||
client: &http.Client{
|
client: &http.Client{
|
||||||
Timeout: 0, // No timeout for streaming
|
Timeout: 0, // No timeout for streaming
|
||||||
|
Transport: &http.Transport{
|
||||||
|
// Disable automatic gzip decompression so we can measure wire bytes
|
||||||
|
DisableCompression: true,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
handlers: make([]*handlerInfo, 0),
|
handlers: make([]*handlerInfo, 0),
|
||||||
metrics: metrics,
|
metrics: metrics,
|
||||||
|
//nolint:gosec // Non-cryptographic randomness is fine for backpressure
|
||||||
|
random: rand.New(rand.NewSource(time.Now().UnixNano())),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// RegisterHandler adds a callback for message processing
|
// RegisterHandler adds a MessageHandler to receive parsed RIS messages.
|
||||||
|
// Each handler gets its own dedicated queue and worker goroutine for processing.
|
||||||
|
// If the streamer is already running, the handler's worker is started immediately.
|
||||||
func (s *Streamer) RegisterHandler(handler MessageHandler) {
|
func (s *Streamer) RegisterHandler(handler MessageHandler) {
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
@@ -95,6 +147,9 @@ func (s *Streamer) RegisterHandler(handler MessageHandler) {
|
|||||||
info := &handlerInfo{
|
info := &handlerInfo{
|
||||||
handler: handler,
|
handler: handler,
|
||||||
queue: make(chan *ristypes.RISMessage, handler.QueueCapacity()),
|
queue: make(chan *ristypes.RISMessage, handler.QueueCapacity()),
|
||||||
|
metrics: handlerMetrics{
|
||||||
|
minTime: time.Duration(math.MaxInt64), // Initialize to max so first value sets the floor
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
s.handlers = append(s.handlers, info)
|
s.handlers = append(s.handlers, info)
|
||||||
@@ -105,14 +160,19 @@ func (s *Streamer) RegisterHandler(handler MessageHandler) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// RegisterRawHandler sets a callback for raw message lines
|
// RegisterRawHandler sets a callback to receive raw JSON lines from the stream
|
||||||
|
// before they are parsed. Only one raw handler can be registered at a time;
|
||||||
|
// subsequent calls will replace the previous handler.
|
||||||
func (s *Streamer) RegisterRawHandler(handler RawMessageHandler) {
|
func (s *Streamer) RegisterRawHandler(handler RawMessageHandler) {
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
s.rawHandler = handler
|
s.rawHandler = handler
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start begins streaming in a goroutine
|
// Start begins streaming BGP updates from the RIS Live API in a background goroutine.
|
||||||
|
// It starts worker goroutines for each registered handler and manages automatic
|
||||||
|
// reconnection with exponential backoff on connection failures.
|
||||||
|
// Returns an error if the streamer is already running.
|
||||||
func (s *Streamer) Start() error {
|
func (s *Streamer) Start() error {
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
defer s.mu.Unlock()
|
defer s.mu.Unlock()
|
||||||
@@ -131,9 +191,7 @@ func (s *Streamer) Start() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
if err := s.stream(ctx); err != nil {
|
s.streamWithReconnect(ctx)
|
||||||
s.logger.Error("Streaming error", "error", err)
|
|
||||||
}
|
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
s.running = false
|
s.running = false
|
||||||
s.mu.Unlock()
|
s.mu.Unlock()
|
||||||
@@ -142,7 +200,9 @@ func (s *Streamer) Start() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop halts the streaming
|
// Stop halts the streaming connection and shuts down all handler workers.
|
||||||
|
// It cancels the streaming context, closes all handler queues, and updates
|
||||||
|
// the connection status in metrics. This method is safe to call multiple times.
|
||||||
func (s *Streamer) Stop() {
|
func (s *Streamer) Stop() {
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
if s.cancel != nil {
|
if s.cancel != nil {
|
||||||
@@ -170,7 +230,7 @@ func (s *Streamer) runHandlerWorker(info *handlerInfo) {
|
|||||||
info.metrics.totalTime += elapsed
|
info.metrics.totalTime += elapsed
|
||||||
|
|
||||||
// Update min time
|
// Update min time
|
||||||
if info.metrics.minTime == 0 || elapsed < info.metrics.minTime {
|
if elapsed < info.metrics.minTime {
|
||||||
info.metrics.minTime = elapsed
|
info.metrics.minTime = elapsed
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -182,7 +242,8 @@ func (s *Streamer) runHandlerWorker(info *handlerInfo) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsRunning returns whether the streamer is currently active
|
// IsRunning reports whether the streamer is currently connected and processing messages.
|
||||||
|
// This is safe to call concurrently from multiple goroutines.
|
||||||
func (s *Streamer) IsRunning() bool {
|
func (s *Streamer) IsRunning() bool {
|
||||||
s.mu.RLock()
|
s.mu.RLock()
|
||||||
defer s.mu.RUnlock()
|
defer s.mu.RUnlock()
|
||||||
@@ -190,21 +251,26 @@ func (s *Streamer) IsRunning() bool {
|
|||||||
return s.running
|
return s.running
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetMetrics returns current streaming metrics
|
// GetMetrics returns the current streaming metrics including message counts,
|
||||||
|
// bytes received, and throughput rates. The returned struct is a snapshot
|
||||||
|
// of the current state and is safe to use without synchronization.
|
||||||
func (s *Streamer) GetMetrics() metrics.StreamMetrics {
|
func (s *Streamer) GetMetrics() metrics.StreamMetrics {
|
||||||
return s.metrics.GetStreamMetrics()
|
return s.metrics.GetStreamMetrics()
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetMetricsTracker returns the metrics tracker instance
|
// GetMetricsTracker returns the underlying metrics.Tracker instance for direct access
|
||||||
|
// to metrics recording and retrieval functionality.
|
||||||
func (s *Streamer) GetMetricsTracker() *metrics.Tracker {
|
func (s *Streamer) GetMetricsTracker() *metrics.Tracker {
|
||||||
return s.metrics
|
return s.metrics
|
||||||
}
|
}
|
||||||
|
|
||||||
// HandlerStats represents metrics for a single handler
|
// HandlerStats contains performance metrics for a single message handler.
|
||||||
|
// It includes queue utilization, message counts, and processing time statistics.
|
||||||
type HandlerStats struct {
|
type HandlerStats struct {
|
||||||
Name string
|
Name string
|
||||||
QueueLength int
|
QueueLength int
|
||||||
QueueCapacity int
|
QueueCapacity int
|
||||||
|
QueueHighWaterMark int
|
||||||
ProcessedCount uint64
|
ProcessedCount uint64
|
||||||
DroppedCount uint64
|
DroppedCount uint64
|
||||||
AvgProcessTime time.Duration
|
AvgProcessTime time.Duration
|
||||||
@@ -212,7 +278,9 @@ type HandlerStats struct {
|
|||||||
MaxProcessTime time.Duration
|
MaxProcessTime time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetHandlerStats returns current handler statistics
|
// GetHandlerStats returns a snapshot of performance statistics for all registered
|
||||||
|
// handlers. The returned slice contains one HandlerStats entry per handler with
|
||||||
|
// current queue depth, processed/dropped counts, and processing time statistics.
|
||||||
func (s *Streamer) GetHandlerStats() []HandlerStats {
|
func (s *Streamer) GetHandlerStats() []HandlerStats {
|
||||||
s.mu.RLock()
|
s.mu.RLock()
|
||||||
defer s.mu.RUnlock()
|
defer s.mu.RUnlock()
|
||||||
@@ -226,6 +294,7 @@ func (s *Streamer) GetHandlerStats() []HandlerStats {
|
|||||||
Name: fmt.Sprintf("%T", info.handler),
|
Name: fmt.Sprintf("%T", info.handler),
|
||||||
QueueLength: len(info.queue),
|
QueueLength: len(info.queue),
|
||||||
QueueCapacity: cap(info.queue),
|
QueueCapacity: cap(info.queue),
|
||||||
|
QueueHighWaterMark: info.metrics.queueHighWaterMark,
|
||||||
ProcessedCount: info.metrics.processedCount,
|
ProcessedCount: info.metrics.processedCount,
|
||||||
DroppedCount: info.metrics.droppedCount,
|
DroppedCount: info.metrics.droppedCount,
|
||||||
MinProcessTime: info.metrics.minTime,
|
MinProcessTime: info.metrics.minTime,
|
||||||
@@ -251,7 +320,9 @@ func (s *Streamer) GetHandlerStats() []HandlerStats {
|
|||||||
return stats
|
return stats
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetDroppedMessages returns the total number of dropped messages
|
// GetDroppedMessages returns the total number of messages dropped across all handlers
|
||||||
|
// due to queue overflow or backpressure. This counter is monotonically increasing
|
||||||
|
// and is safe to call concurrently.
|
||||||
func (s *Streamer) GetDroppedMessages() uint64 {
|
func (s *Streamer) GetDroppedMessages() uint64 {
|
||||||
return atomic.LoadUint64(&s.totalDropped)
|
return atomic.LoadUint64(&s.totalDropped)
|
||||||
}
|
}
|
||||||
@@ -270,16 +341,18 @@ func (s *Streamer) logMetrics() {
|
|||||||
uptime,
|
uptime,
|
||||||
"total_messages",
|
"total_messages",
|
||||||
metrics.TotalMessages,
|
metrics.TotalMessages,
|
||||||
"total_bytes",
|
"wire_bytes",
|
||||||
|
metrics.TotalWireBytes,
|
||||||
|
"wire_mb",
|
||||||
|
fmt.Sprintf("%.2f", float64(metrics.TotalWireBytes)/bytesPerMB),
|
||||||
|
"wire_mbps",
|
||||||
|
fmt.Sprintf("%.2f", metrics.WireBitsPerSec/bitsPerMegabit),
|
||||||
|
"decompressed_bytes",
|
||||||
metrics.TotalBytes,
|
metrics.TotalBytes,
|
||||||
"total_mb",
|
"decompressed_mb",
|
||||||
fmt.Sprintf("%.2f", float64(metrics.TotalBytes)/bytesPerMB),
|
fmt.Sprintf("%.2f", float64(metrics.TotalBytes)/bytesPerMB),
|
||||||
"messages_per_sec",
|
"messages_per_sec",
|
||||||
fmt.Sprintf("%.2f", metrics.MessagesPerSec),
|
fmt.Sprintf("%.2f", metrics.MessagesPerSec),
|
||||||
"bits_per_sec",
|
|
||||||
fmt.Sprintf("%.0f", metrics.BitsPerSec),
|
|
||||||
"mbps",
|
|
||||||
fmt.Sprintf("%.2f", metrics.BitsPerSec/bitsPerMegabit),
|
|
||||||
"total_dropped",
|
"total_dropped",
|
||||||
totalDropped,
|
totalDropped,
|
||||||
)
|
)
|
||||||
@@ -320,12 +393,81 @@ func (s *Streamer) updateMetrics(messageBytes int) {
|
|||||||
s.metrics.RecordMessage(int64(messageBytes))
|
s.metrics.RecordMessage(int64(messageBytes))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// streamWithReconnect handles streaming with automatic reconnection and exponential backoff
|
||||||
|
func (s *Streamer) streamWithReconnect(ctx context.Context) {
|
||||||
|
backoffDelay := minBackoffDelay
|
||||||
|
consecutiveFailures := 0
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
s.logger.Info("Stream context cancelled, stopping reconnection attempts")
|
||||||
|
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
// Attempt to stream
|
||||||
|
startTime := time.Now()
|
||||||
|
err := s.stream(ctx)
|
||||||
|
streamDuration := time.Since(startTime)
|
||||||
|
|
||||||
|
if err == nil {
|
||||||
|
// Clean exit (context cancelled)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Log the error
|
||||||
|
s.logger.Error("Stream disconnected",
|
||||||
|
"error", err,
|
||||||
|
"consecutive_failures", consecutiveFailures+1,
|
||||||
|
"stream_duration", streamDuration)
|
||||||
|
s.metrics.SetConnected(false)
|
||||||
|
|
||||||
|
// Check if context is cancelled
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// If we streamed for more than 30 seconds, reset the backoff
|
||||||
|
// This indicates we had a successful connection that received data
|
||||||
|
if streamDuration > 30*time.Second {
|
||||||
|
s.logger.Info("Resetting backoff delay due to successful connection",
|
||||||
|
"stream_duration", streamDuration)
|
||||||
|
backoffDelay = minBackoffDelay
|
||||||
|
consecutiveFailures = 0
|
||||||
|
} else {
|
||||||
|
// Increment consecutive failures
|
||||||
|
consecutiveFailures++
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait with exponential backoff
|
||||||
|
s.logger.Info("Waiting before reconnection attempt",
|
||||||
|
"delay_seconds", backoffDelay.Seconds(),
|
||||||
|
"consecutive_failures", consecutiveFailures)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-time.After(backoffDelay):
|
||||||
|
// Double the backoff delay for next time, up to max
|
||||||
|
backoffDelay *= 2
|
||||||
|
if backoffDelay > maxBackoffDelay {
|
||||||
|
backoffDelay = maxBackoffDelay
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Streamer) stream(ctx context.Context) error {
|
func (s *Streamer) stream(ctx context.Context) error {
|
||||||
req, err := http.NewRequestWithContext(ctx, "GET", risLiveURL, nil)
|
req, err := http.NewRequestWithContext(ctx, "GET", risLiveURL, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to create request: %w", err)
|
return fmt.Errorf("failed to create request: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Explicitly request gzip compression
|
||||||
|
req.Header.Set("Accept-Encoding", "gzip")
|
||||||
|
|
||||||
resp, err := s.client.Do(req)
|
resp, err := s.client.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to connect to RIS Live: %w", err)
|
return fmt.Errorf("failed to connect to RIS Live: %w", err)
|
||||||
@@ -340,9 +482,28 @@ func (s *Streamer) stream(ctx context.Context) error {
|
|||||||
return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
|
return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
|
||||||
}
|
}
|
||||||
|
|
||||||
s.logger.Info("Connected to RIS Live stream")
|
// Wrap body with counting reader to track actual wire bytes
|
||||||
|
wireCounter := &countingReader{reader: resp.Body}
|
||||||
|
|
||||||
|
// Check if response is gzip-compressed and decompress if needed
|
||||||
|
var reader io.Reader = wireCounter
|
||||||
|
if resp.Header.Get("Content-Encoding") == "gzip" {
|
||||||
|
gzReader, err := gzip.NewReader(wireCounter)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create gzip reader: %w", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = gzReader.Close() }()
|
||||||
|
reader = gzReader
|
||||||
|
s.logger.Info("Connected to RIS Live stream", "compression", "gzip")
|
||||||
|
} else {
|
||||||
|
s.logger.Info("Connected to RIS Live stream", "compression", "none")
|
||||||
|
}
|
||||||
|
|
||||||
s.metrics.SetConnected(true)
|
s.metrics.SetConnected(true)
|
||||||
|
|
||||||
|
// Track wire bytes for metrics updates
|
||||||
|
var lastWireBytes int64
|
||||||
|
|
||||||
// Start metrics logging goroutine
|
// Start metrics logging goroutine
|
||||||
metricsTicker := time.NewTicker(metricsLogInterval)
|
metricsTicker := time.NewTicker(metricsLogInterval)
|
||||||
defer metricsTicker.Stop()
|
defer metricsTicker.Stop()
|
||||||
@@ -358,7 +519,27 @@ func (s *Streamer) stream(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
scanner := bufio.NewScanner(resp.Body)
|
// Wire byte update ticker - update metrics with actual wire bytes periodically
|
||||||
|
wireUpdateTicker := time.NewTicker(time.Second)
|
||||||
|
defer wireUpdateTicker.Stop()
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-wireUpdateTicker.C:
|
||||||
|
currentBytes := wireCounter.Count()
|
||||||
|
delta := currentBytes - lastWireBytes
|
||||||
|
if delta > 0 {
|
||||||
|
s.metrics.RecordWireBytes(delta)
|
||||||
|
lastWireBytes = currentBytes
|
||||||
|
}
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
scanner := bufio.NewScanner(reader)
|
||||||
|
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
select {
|
select {
|
||||||
@@ -374,7 +555,7 @@ func (s *Streamer) stream(ctx context.Context) error {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update metrics with message size
|
// Update metrics with decompressed message size
|
||||||
s.updateMetrics(len(line))
|
s.updateMetrics(len(line))
|
||||||
|
|
||||||
// Call raw handler if registered
|
// Call raw handler if registered
|
||||||
@@ -390,10 +571,13 @@ func (s *Streamer) stream(ctx context.Context) error {
|
|||||||
// Parse the message first
|
// Parse the message first
|
||||||
var wrapper ristypes.RISLiveMessage
|
var wrapper ristypes.RISLiveMessage
|
||||||
if err := json.Unmarshal(line, &wrapper); err != nil {
|
if err := json.Unmarshal(line, &wrapper); err != nil {
|
||||||
// Output the raw line and panic on parse failure
|
// Log the error and return to trigger reconnection
|
||||||
fmt.Fprintf(os.Stderr, "Failed to parse JSON: %v\n", err)
|
s.logger.Error("Failed to parse JSON",
|
||||||
fmt.Fprintf(os.Stderr, "Raw line: %s\n", string(line))
|
"error", err,
|
||||||
panic(fmt.Sprintf("JSON parse error: %v", err))
|
"line", string(line),
|
||||||
|
"line_length", len(line))
|
||||||
|
|
||||||
|
return fmt.Errorf("JSON parse error: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if it's a ris_message wrapper
|
// Check if it's a ris_message wrapper
|
||||||
@@ -443,34 +627,45 @@ func (s *Streamer) stream(ctx context.Context) error {
|
|||||||
// Peer state changes - silently ignore
|
// Peer state changes - silently ignore
|
||||||
continue
|
continue
|
||||||
default:
|
default:
|
||||||
fmt.Fprintf(
|
s.logger.Warn("Unknown message type, skipping",
|
||||||
os.Stderr,
|
"type", msg.Type,
|
||||||
"UNKNOWN MESSAGE TYPE: %s\nRAW MESSAGE: %s\n",
|
|
||||||
msg.Type,
|
|
||||||
string(line),
|
|
||||||
)
|
|
||||||
panic(
|
|
||||||
fmt.Sprintf(
|
|
||||||
"Unknown RIS message type: %s",
|
|
||||||
msg.Type,
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Dispatch to interested handlers
|
// Dispatch to interested handlers
|
||||||
s.mu.RLock()
|
s.mu.RLock()
|
||||||
for _, info := range s.handlers {
|
for _, info := range s.handlers {
|
||||||
if info.handler.WantsMessage(msg.Type) {
|
if !info.handler.WantsMessage(msg.Type) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if we should drop due to backpressure
|
||||||
|
if s.shouldDropForBackpressure(info) {
|
||||||
|
atomic.AddUint64(&info.metrics.droppedCount, 1)
|
||||||
|
atomic.AddUint64(&s.totalDropped, 1)
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to queue the message
|
||||||
select {
|
select {
|
||||||
case info.queue <- &msg:
|
case info.queue <- &msg:
|
||||||
// Message queued successfully
|
// Message queued successfully
|
||||||
|
// Update high water mark if needed
|
||||||
|
queueLen := len(info.queue)
|
||||||
|
info.metrics.mu.Lock()
|
||||||
|
if queueLen > info.metrics.queueHighWaterMark {
|
||||||
|
info.metrics.queueHighWaterMark = queueLen
|
||||||
|
}
|
||||||
|
info.metrics.mu.Unlock()
|
||||||
default:
|
default:
|
||||||
// Queue is full, drop the message
|
// Queue is full, drop the message
|
||||||
atomic.AddUint64(&info.metrics.droppedCount, 1)
|
atomic.AddUint64(&info.metrics.droppedCount, 1)
|
||||||
atomic.AddUint64(&s.totalDropped, 1)
|
atomic.AddUint64(&s.totalDropped, 1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
s.mu.RUnlock()
|
s.mu.RUnlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -480,3 +675,25 @@ func (s *Streamer) stream(ctx context.Context) error {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// shouldDropForBackpressure determines if a message should be dropped based on queue utilization
|
||||||
|
func (s *Streamer) shouldDropForBackpressure(info *handlerInfo) bool {
|
||||||
|
// Calculate queue utilization
|
||||||
|
queueLen := len(info.queue)
|
||||||
|
queueCap := cap(info.queue)
|
||||||
|
utilization := float64(queueLen) / float64(queueCap)
|
||||||
|
|
||||||
|
// No drops below threshold
|
||||||
|
if utilization < backpressureThreshold {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate drop probability (0.0 at threshold, 1.0 at 100% full)
|
||||||
|
dropProbability := (utilization - backpressureThreshold) * backpressureSlope
|
||||||
|
if dropProbability > 1.0 {
|
||||||
|
dropProbability = 1.0
|
||||||
|
}
|
||||||
|
|
||||||
|
// Random drop based on probability
|
||||||
|
return s.random.Float64() < dropProbability
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
<title>AS{{.ASN.Number}} - {{.ASN.Handle}} - RouteWatch</title>
|
<title>AS{{.ASN.ASN}} - {{.ASN.Handle}} - RouteWatch</title>
|
||||||
<style>
|
<style>
|
||||||
body {
|
body {
|
||||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||||
@@ -136,7 +136,7 @@
|
|||||||
<div class="container">
|
<div class="container">
|
||||||
<a href="/status" class="nav-link">← Back to Status</a>
|
<a href="/status" class="nav-link">← Back to Status</a>
|
||||||
|
|
||||||
<h1>AS{{.ASN.Number}}{{if .ASN.Handle}} - {{.ASN.Handle}}{{end}}</h1>
|
<h1>AS{{.ASN.ASN}}{{if .ASN.Handle}} - {{.ASN.Handle}}{{end}}</h1>
|
||||||
{{if .ASN.Description}}
|
{{if .ASN.Description}}
|
||||||
<p class="subtitle">{{.ASN.Description}}</p>
|
<p class="subtitle">{{.ASN.Description}}</p>
|
||||||
{{end}}
|
{{end}}
|
||||||
@@ -154,6 +154,10 @@
|
|||||||
<div class="info-label">IPv6 Prefixes</div>
|
<div class="info-label">IPv6 Prefixes</div>
|
||||||
<div class="info-value">{{.IPv6Count}}</div>
|
<div class="info-value">{{.IPv6Count}}</div>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="info-card">
|
||||||
|
<div class="info-label">Peer ASNs</div>
|
||||||
|
<div class="info-value">{{.PeerCount}}</div>
|
||||||
|
</div>
|
||||||
<div class="info-card">
|
<div class="info-card">
|
||||||
<div class="info-label">First Seen</div>
|
<div class="info-label">First Seen</div>
|
||||||
<div class="info-value">{{.ASN.FirstSeen.Format "2006-01-02"}}</div>
|
<div class="info-value">{{.ASN.FirstSeen.Format "2006-01-02"}}</div>
|
||||||
@@ -223,6 +227,44 @@
|
|||||||
<p>No prefixes announced by this AS</p>
|
<p>No prefixes announced by this AS</p>
|
||||||
</div>
|
</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
||||||
|
{{if .Peers}}
|
||||||
|
<div class="prefix-section">
|
||||||
|
<div class="prefix-header">
|
||||||
|
<h2>Peer ASNs</h2>
|
||||||
|
<span class="prefix-count">{{.PeerCount}}</span>
|
||||||
|
</div>
|
||||||
|
<table class="prefix-table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>ASN</th>
|
||||||
|
<th>Handle</th>
|
||||||
|
<th>Description</th>
|
||||||
|
<th>First Seen</th>
|
||||||
|
<th>Last Seen</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{{range .Peers}}
|
||||||
|
<tr>
|
||||||
|
<td><a href="/as/{{.ASN}}" class="prefix-link">AS{{.ASN}}</a></td>
|
||||||
|
<td>{{if .Handle}}{{.Handle}}{{else}}-{{end}}</td>
|
||||||
|
<td>{{if .Description}}{{.Description}}{{else}}-{{end}}</td>
|
||||||
|
<td>{{.FirstSeen.Format "2006-01-02"}}</td>
|
||||||
|
<td>{{.LastSeen.Format "2006-01-02"}}</td>
|
||||||
|
</tr>
|
||||||
|
{{end}}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
{{else}}
|
||||||
|
<div class="prefix-section">
|
||||||
|
<h2>Peer ASNs</h2>
|
||||||
|
<div class="empty-state">
|
||||||
|
<p>No peering relationships found for this AS</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{{end}}
|
||||||
</div>
|
</div>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
@@ -13,7 +13,8 @@
|
|||||||
color: #333;
|
color: #333;
|
||||||
}
|
}
|
||||||
.container {
|
.container {
|
||||||
max-width: 1200px;
|
width: 90%;
|
||||||
|
max-width: 1600px;
|
||||||
margin: 0 auto;
|
margin: 0 auto;
|
||||||
background: white;
|
background: white;
|
||||||
padding: 30px;
|
padding: 30px;
|
||||||
@@ -91,6 +92,7 @@
|
|||||||
.route-table td {
|
.route-table td {
|
||||||
padding: 12px;
|
padding: 12px;
|
||||||
border-bottom: 1px solid #e0e0e0;
|
border-bottom: 1px solid #e0e0e0;
|
||||||
|
white-space: nowrap;
|
||||||
}
|
}
|
||||||
.route-table tr:hover {
|
.route-table tr:hover {
|
||||||
background: #f8f9fa;
|
background: #f8f9fa;
|
||||||
@@ -114,9 +116,13 @@
|
|||||||
font-family: monospace;
|
font-family: monospace;
|
||||||
font-size: 13px;
|
font-size: 13px;
|
||||||
color: #666;
|
color: #666;
|
||||||
max-width: 300px;
|
max-width: 600px;
|
||||||
overflow-x: auto;
|
word-wrap: break-word;
|
||||||
white-space: nowrap;
|
white-space: normal !important;
|
||||||
|
line-height: 1.5;
|
||||||
|
}
|
||||||
|
.as-path .as-link {
|
||||||
|
font-weight: 600;
|
||||||
}
|
}
|
||||||
.age {
|
.age {
|
||||||
color: #7f8c8d;
|
color: #7f8c8d;
|
||||||
@@ -168,7 +174,7 @@
|
|||||||
font-size: 14px;
|
font-size: 14px;
|
||||||
}
|
}
|
||||||
.as-path {
|
.as-path {
|
||||||
max-width: 150px;
|
max-width: 100%;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
</style>
|
</style>
|
||||||
@@ -201,7 +207,7 @@
|
|||||||
<div class="origin-list">
|
<div class="origin-list">
|
||||||
{{range .Origins}}
|
{{range .Origins}}
|
||||||
<div class="origin-item">
|
<div class="origin-item">
|
||||||
<a href="/as/{{.Number}}" class="as-link">AS{{.Number}}</a>
|
<a href="/as/{{.ASN}}" class="as-link">AS{{.ASN}}</a>
|
||||||
{{if .Handle}} ({{.Handle}}){{end}}
|
{{if .Handle}} ({{.Handle}}){{end}}
|
||||||
<span style="color: #7f8c8d; margin-left: 10px;">{{.PeerCount}} peer{{if ne .PeerCount 1}}s{{end}}</span>
|
<span style="color: #7f8c8d; margin-left: 10px;">{{.PeerCount}} peer{{if ne .PeerCount 1}}s{{end}}</span>
|
||||||
</div>
|
</div>
|
||||||
@@ -234,7 +240,7 @@
|
|||||||
<a href="/as/{{.OriginASN}}" class="as-link">AS{{.OriginASN}}</a>
|
<a href="/as/{{.OriginASN}}" class="as-link">AS{{.OriginASN}}</a>
|
||||||
</td>
|
</td>
|
||||||
<td class="peer-ip">{{.PeerIP}}</td>
|
<td class="peer-ip">{{.PeerIP}}</td>
|
||||||
<td class="as-path">{{range $i, $as := .ASPath}}{{if $i}} → {{end}}{{$as}}{{end}}</td>
|
<td class="as-path">{{range $i, $as := .ASPathWithHandle}}{{if $i}} → {{end}}<a href="/as/{{$as.ASN}}" class="as-link">{{if $as.Handle}}{{$as.Handle}}{{else}}AS{{$as.ASN}}{{end}}</a>{{end}}</td>
|
||||||
<td class="peer-ip">{{.NextHop}}</td>
|
<td class="peer-ip">{{.NextHop}}</td>
|
||||||
<td>{{.LastUpdated.Format "2006-01-02 15:04:05"}}</td>
|
<td>{{.LastUpdated.Format "2006-01-02 15:04:05"}}</td>
|
||||||
<td class="age">{{.LastUpdated | timeSince}}</td>
|
<td class="age">{{.LastUpdated | timeSince}}</td>
|
||||||
|
|||||||
108
internal/templates/prefix_length.html
Normal file
108
internal/templates/prefix_length.html
Normal file
@@ -0,0 +1,108 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<title>Prefixes with /{{ .MaskLength }} - RouteWatch</title>
|
||||||
|
<style>
|
||||||
|
body {
|
||||||
|
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||||
|
max-width: 1200px;
|
||||||
|
margin: 0 auto;
|
||||||
|
padding: 20px;
|
||||||
|
background: #f5f5f5;
|
||||||
|
}
|
||||||
|
h1 {
|
||||||
|
color: #333;
|
||||||
|
margin-bottom: 10px;
|
||||||
|
}
|
||||||
|
.subtitle {
|
||||||
|
color: #666;
|
||||||
|
margin-bottom: 30px;
|
||||||
|
}
|
||||||
|
.info-card {
|
||||||
|
background: white;
|
||||||
|
padding: 20px;
|
||||||
|
border-radius: 8px;
|
||||||
|
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
||||||
|
margin-bottom: 20px;
|
||||||
|
}
|
||||||
|
table {
|
||||||
|
width: 100%;
|
||||||
|
border-collapse: collapse;
|
||||||
|
background: white;
|
||||||
|
box-shadow: 0 2px 4px rgba(0,0,0,0.1);
|
||||||
|
border-radius: 8px;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
th {
|
||||||
|
background: #f8f9fa;
|
||||||
|
padding: 12px;
|
||||||
|
text-align: left;
|
||||||
|
font-weight: 600;
|
||||||
|
color: #333;
|
||||||
|
border-bottom: 2px solid #dee2e6;
|
||||||
|
}
|
||||||
|
td {
|
||||||
|
padding: 12px;
|
||||||
|
border-bottom: 1px solid #eee;
|
||||||
|
}
|
||||||
|
tr:last-child td {
|
||||||
|
border-bottom: none;
|
||||||
|
}
|
||||||
|
tr:hover {
|
||||||
|
background: #f8f9fa;
|
||||||
|
}
|
||||||
|
a {
|
||||||
|
color: #0066cc;
|
||||||
|
text-decoration: none;
|
||||||
|
}
|
||||||
|
a:hover {
|
||||||
|
text-decoration: underline;
|
||||||
|
}
|
||||||
|
.prefix-link {
|
||||||
|
font-family: 'SF Mono', Monaco, 'Cascadia Mono', 'Roboto Mono', Consolas, 'Courier New', monospace;
|
||||||
|
}
|
||||||
|
.as-link {
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
.age {
|
||||||
|
color: #666;
|
||||||
|
font-size: 0.9em;
|
||||||
|
}
|
||||||
|
.back-link {
|
||||||
|
display: inline-block;
|
||||||
|
margin-bottom: 20px;
|
||||||
|
color: #0066cc;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<a href="/status" class="back-link">← Back to Status</a>
|
||||||
|
<h1>IPv{{ .IPVersion }} Prefixes with /{{ .MaskLength }}</h1>
|
||||||
|
<p class="subtitle">Showing {{ .Count }} randomly selected prefixes</p>
|
||||||
|
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Prefix</th>
|
||||||
|
<th>Age</th>
|
||||||
|
<th>Origin AS</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{{ range .Prefixes }}
|
||||||
|
<tr>
|
||||||
|
<td><a href="/prefix/{{ .Prefix | urlEncode }}" class="prefix-link">{{ .Prefix }}</a></td>
|
||||||
|
<td class="age">{{ .Age }}</td>
|
||||||
|
<td>
|
||||||
|
<a href="/as/{{ .OriginASN }}" class="as-link">
|
||||||
|
AS{{ .OriginASN }}{{ if .OriginASDescription }} ({{ .OriginASDescription }}){{ end }}
|
||||||
|
</a>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
{{ end }}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -49,6 +49,16 @@
|
|||||||
font-family: 'SF Mono', Monaco, 'Cascadia Mono', 'Roboto Mono', Consolas, 'Courier New', monospace;
|
font-family: 'SF Mono', Monaco, 'Cascadia Mono', 'Roboto Mono', Consolas, 'Courier New', monospace;
|
||||||
color: #333;
|
color: #333;
|
||||||
}
|
}
|
||||||
|
.metric-value.metric-link {
|
||||||
|
text-decoration: underline;
|
||||||
|
text-decoration-style: dashed;
|
||||||
|
text-underline-offset: 2px;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
.metric-value.metric-link:hover {
|
||||||
|
color: #0066cc;
|
||||||
|
text-decoration-style: solid;
|
||||||
|
}
|
||||||
.connected {
|
.connected {
|
||||||
color: #22c55e;
|
color: #22c55e;
|
||||||
}
|
}
|
||||||
@@ -94,6 +104,14 @@
|
|||||||
|
|
||||||
<div class="status-card">
|
<div class="status-card">
|
||||||
<h2>Stream Statistics</h2>
|
<h2>Stream Statistics</h2>
|
||||||
|
<div class="metric">
|
||||||
|
<span class="metric-label">Connection Duration</span>
|
||||||
|
<span class="metric-value" id="connection_duration">-</span>
|
||||||
|
</div>
|
||||||
|
<div class="metric">
|
||||||
|
<span class="metric-label">Reconnections</span>
|
||||||
|
<span class="metric-value" id="reconnect_count">-</span>
|
||||||
|
</div>
|
||||||
<div class="metric">
|
<div class="metric">
|
||||||
<span class="metric-label">Total Messages</span>
|
<span class="metric-label">Total Messages</span>
|
||||||
<span class="metric-value" id="total_messages">-</span>
|
<span class="metric-value" id="total_messages">-</span>
|
||||||
@@ -104,11 +122,11 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="metric">
|
<div class="metric">
|
||||||
<span class="metric-label">Total Data</span>
|
<span class="metric-label">Total Data</span>
|
||||||
<span class="metric-value" id="total_bytes">-</span>
|
<span class="metric-value" id="total_wire_bytes">-</span>
|
||||||
</div>
|
</div>
|
||||||
<div class="metric">
|
<div class="metric">
|
||||||
<span class="metric-label">Throughput</span>
|
<span class="metric-label">Throughput</span>
|
||||||
<span class="metric-value" id="mbits_per_sec">-</span>
|
<span class="metric-value" id="wire_mbits_per_sec">-</span>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -167,6 +185,38 @@
|
|||||||
<span class="metric-value" id="ipv6_updates_per_sec">-</span>
|
<span class="metric-value" id="ipv6_updates_per_sec">-</span>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="status-card">
|
||||||
|
<h2>WHOIS Fetcher</h2>
|
||||||
|
<div class="metric">
|
||||||
|
<span class="metric-label">Fresh ASNs</span>
|
||||||
|
<span class="metric-value" id="whois_fresh">-</span>
|
||||||
|
</div>
|
||||||
|
<div class="metric">
|
||||||
|
<span class="metric-label">Stale ASNs</span>
|
||||||
|
<span class="metric-value" id="whois_stale">-</span>
|
||||||
|
</div>
|
||||||
|
<div class="metric">
|
||||||
|
<span class="metric-label">Never Fetched</span>
|
||||||
|
<span class="metric-value" id="whois_never">-</span>
|
||||||
|
</div>
|
||||||
|
<div class="metric">
|
||||||
|
<span class="metric-label">Fresh %</span>
|
||||||
|
<span class="metric-value" id="whois_percent">-</span>
|
||||||
|
</div>
|
||||||
|
<div class="metric">
|
||||||
|
<span class="metric-label">Successes (1h)</span>
|
||||||
|
<span class="metric-value" id="whois_successes">-</span>
|
||||||
|
</div>
|
||||||
|
<div class="metric">
|
||||||
|
<span class="metric-label">Errors (1h)</span>
|
||||||
|
<span class="metric-value" id="whois_errors">-</span>
|
||||||
|
</div>
|
||||||
|
<div class="metric">
|
||||||
|
<span class="metric-label">Current Interval</span>
|
||||||
|
<span class="metric-value" id="whois_interval">-</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="status-grid">
|
<div class="status-grid">
|
||||||
@@ -226,12 +276,16 @@
|
|||||||
// Sort by mask length
|
// Sort by mask length
|
||||||
distribution.sort((a, b) => a.mask_length - b.mask_length);
|
distribution.sort((a, b) => a.mask_length - b.mask_length);
|
||||||
|
|
||||||
|
// Determine the URL path based on whether this is IPv4 or IPv6
|
||||||
|
const isIPv6 = elementId.includes('ipv6');
|
||||||
|
const urlPath = isIPv6 ? '/prefixlength6/' : '/prefixlength/';
|
||||||
|
|
||||||
distribution.forEach(item => {
|
distribution.forEach(item => {
|
||||||
const metric = document.createElement('div');
|
const metric = document.createElement('div');
|
||||||
metric.className = 'metric';
|
metric.className = 'metric';
|
||||||
metric.innerHTML = `
|
metric.innerHTML = `
|
||||||
<span class="metric-label">/${item.mask_length}</span>
|
<span class="metric-label">/${item.mask_length}</span>
|
||||||
<span class="metric-value">${formatNumber(item.count)}</span>
|
<a href="${urlPath}${item.mask_length}" class="metric-value metric-link">${formatNumber(item.count)}</a>
|
||||||
`;
|
`;
|
||||||
container.appendChild(metric);
|
container.appendChild(metric);
|
||||||
});
|
});
|
||||||
@@ -254,6 +308,10 @@
|
|||||||
<span class="metric-label">Queue</span>
|
<span class="metric-label">Queue</span>
|
||||||
<span class="metric-value">${handler.queue_length}/${handler.queue_capacity}</span>
|
<span class="metric-value">${handler.queue_length}/${handler.queue_capacity}</span>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="metric">
|
||||||
|
<span class="metric-label">High Water Mark</span>
|
||||||
|
<span class="metric-value">${handler.queue_high_water_mark}/${handler.queue_capacity} (${Math.round(handler.queue_high_water_mark * 100 / handler.queue_capacity)}%)</span>
|
||||||
|
</div>
|
||||||
<div class="metric">
|
<div class="metric">
|
||||||
<span class="metric-label">Processed</span>
|
<span class="metric-label">Processed</span>
|
||||||
<span class="metric-value">${formatNumber(handler.processed_count)}</span>
|
<span class="metric-value">${formatNumber(handler.processed_count)}</span>
|
||||||
@@ -276,6 +334,49 @@
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function resetAllFields() {
|
||||||
|
// Reset all metric fields to '-'
|
||||||
|
document.getElementById('connected').textContent = '-';
|
||||||
|
document.getElementById('connected').className = 'metric-value';
|
||||||
|
document.getElementById('uptime').textContent = '-';
|
||||||
|
document.getElementById('go_version').textContent = '-';
|
||||||
|
document.getElementById('goroutines').textContent = '-';
|
||||||
|
document.getElementById('memory_usage').textContent = '-';
|
||||||
|
document.getElementById('connection_duration').textContent = '-';
|
||||||
|
document.getElementById('reconnect_count').textContent = '-';
|
||||||
|
document.getElementById('total_messages').textContent = '-';
|
||||||
|
document.getElementById('messages_per_sec').textContent = '-';
|
||||||
|
document.getElementById('total_wire_bytes').textContent = '-';
|
||||||
|
document.getElementById('wire_mbits_per_sec').textContent = '-';
|
||||||
|
document.getElementById('asns').textContent = '-';
|
||||||
|
document.getElementById('prefixes').textContent = '-';
|
||||||
|
document.getElementById('ipv4_prefixes').textContent = '-';
|
||||||
|
document.getElementById('ipv6_prefixes').textContent = '-';
|
||||||
|
document.getElementById('peerings').textContent = '-';
|
||||||
|
document.getElementById('peers').textContent = '-';
|
||||||
|
document.getElementById('database_size').textContent = '-';
|
||||||
|
document.getElementById('live_routes').textContent = '-';
|
||||||
|
document.getElementById('ipv4_routes').textContent = '-';
|
||||||
|
document.getElementById('ipv6_routes').textContent = '-';
|
||||||
|
document.getElementById('ipv4_updates_per_sec').textContent = '-';
|
||||||
|
document.getElementById('ipv6_updates_per_sec').textContent = '-';
|
||||||
|
document.getElementById('whois_fresh').textContent = '-';
|
||||||
|
document.getElementById('whois_stale').textContent = '-';
|
||||||
|
document.getElementById('whois_never').textContent = '-';
|
||||||
|
document.getElementById('whois_percent').textContent = '-';
|
||||||
|
document.getElementById('whois_successes').textContent = '-';
|
||||||
|
document.getElementById('whois_errors').textContent = '-';
|
||||||
|
document.getElementById('whois_errors').className = 'metric-value';
|
||||||
|
document.getElementById('whois_interval').textContent = '-';
|
||||||
|
|
||||||
|
// Clear handler stats
|
||||||
|
document.getElementById('handler-stats-container').innerHTML = '';
|
||||||
|
|
||||||
|
// Clear prefix distributions
|
||||||
|
document.getElementById('ipv4-prefix-distribution').innerHTML = '<div class="metric"><span class="metric-label">No data</span></div>';
|
||||||
|
document.getElementById('ipv6-prefix-distribution').innerHTML = '<div class="metric"><span class="metric-label">No data</span></div>';
|
||||||
|
}
|
||||||
|
|
||||||
function updateStatus() {
|
function updateStatus() {
|
||||||
fetch('/api/v1/stats')
|
fetch('/api/v1/stats')
|
||||||
.then(response => response.json())
|
.then(response => response.json())
|
||||||
@@ -284,6 +385,7 @@
|
|||||||
if (response.status === 'error') {
|
if (response.status === 'error') {
|
||||||
document.getElementById('error').textContent = 'Error: ' + response.error.msg;
|
document.getElementById('error').textContent = 'Error: ' + response.error.msg;
|
||||||
document.getElementById('error').style.display = 'block';
|
document.getElementById('error').style.display = 'block';
|
||||||
|
resetAllFields();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -300,10 +402,12 @@
|
|||||||
document.getElementById('go_version').textContent = data.go_version;
|
document.getElementById('go_version').textContent = data.go_version;
|
||||||
document.getElementById('goroutines').textContent = formatNumber(data.goroutines);
|
document.getElementById('goroutines').textContent = formatNumber(data.goroutines);
|
||||||
document.getElementById('memory_usage').textContent = data.memory_usage;
|
document.getElementById('memory_usage').textContent = data.memory_usage;
|
||||||
|
document.getElementById('connection_duration').textContent = data.connection_duration;
|
||||||
|
document.getElementById('reconnect_count').textContent = formatNumber(data.reconnect_count);
|
||||||
document.getElementById('total_messages').textContent = formatNumber(data.total_messages);
|
document.getElementById('total_messages').textContent = formatNumber(data.total_messages);
|
||||||
document.getElementById('messages_per_sec').textContent = data.messages_per_sec.toFixed(1);
|
document.getElementById('messages_per_sec').textContent = data.messages_per_sec.toFixed(1);
|
||||||
document.getElementById('total_bytes').textContent = formatBytes(data.total_bytes);
|
document.getElementById('total_wire_bytes').textContent = formatBytes(data.total_wire_bytes);
|
||||||
document.getElementById('mbits_per_sec').textContent = data.mbits_per_sec.toFixed(2) + ' Mbps';
|
document.getElementById('wire_mbits_per_sec').textContent = data.wire_mbits_per_sec.toFixed(2) + ' Mbps';
|
||||||
document.getElementById('asns').textContent = formatNumber(data.asns);
|
document.getElementById('asns').textContent = formatNumber(data.asns);
|
||||||
document.getElementById('prefixes').textContent = formatNumber(data.prefixes);
|
document.getElementById('prefixes').textContent = formatNumber(data.prefixes);
|
||||||
document.getElementById('ipv4_prefixes').textContent = formatNumber(data.ipv4_prefixes);
|
document.getElementById('ipv4_prefixes').textContent = formatNumber(data.ipv4_prefixes);
|
||||||
@@ -317,6 +421,19 @@
|
|||||||
document.getElementById('ipv4_updates_per_sec').textContent = data.ipv4_updates_per_sec.toFixed(1);
|
document.getElementById('ipv4_updates_per_sec').textContent = data.ipv4_updates_per_sec.toFixed(1);
|
||||||
document.getElementById('ipv6_updates_per_sec').textContent = data.ipv6_updates_per_sec.toFixed(1);
|
document.getElementById('ipv6_updates_per_sec').textContent = data.ipv6_updates_per_sec.toFixed(1);
|
||||||
|
|
||||||
|
// Update WHOIS stats
|
||||||
|
if (data.whois_stats) {
|
||||||
|
document.getElementById('whois_fresh').textContent = formatNumber(data.whois_stats.fresh_asns);
|
||||||
|
document.getElementById('whois_stale').textContent = formatNumber(data.whois_stats.stale_asns);
|
||||||
|
document.getElementById('whois_never').textContent = formatNumber(data.whois_stats.never_fetched);
|
||||||
|
document.getElementById('whois_percent').textContent = data.whois_stats.fresh_percent.toFixed(1) + '%';
|
||||||
|
document.getElementById('whois_successes').textContent = formatNumber(data.whois_stats.successes_last_hour);
|
||||||
|
const errorsEl = document.getElementById('whois_errors');
|
||||||
|
errorsEl.textContent = formatNumber(data.whois_stats.errors_last_hour);
|
||||||
|
errorsEl.className = 'metric-value' + (data.whois_stats.errors_last_hour > 0 ? ' disconnected' : '');
|
||||||
|
document.getElementById('whois_interval').textContent = data.whois_stats.current_interval;
|
||||||
|
}
|
||||||
|
|
||||||
// Update handler stats
|
// Update handler stats
|
||||||
updateHandlerStats(data.handler_stats || []);
|
updateHandlerStats(data.handler_stats || []);
|
||||||
|
|
||||||
@@ -330,12 +447,13 @@
|
|||||||
.catch(error => {
|
.catch(error => {
|
||||||
document.getElementById('error').textContent = 'Error fetching status: ' + error;
|
document.getElementById('error').textContent = 'Error fetching status: ' + error;
|
||||||
document.getElementById('error').style.display = 'block';
|
document.getElementById('error').style.display = 'block';
|
||||||
|
resetAllFields();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update immediately and then every 500ms
|
// Update immediately and then every 2 seconds
|
||||||
updateStatus();
|
updateStatus();
|
||||||
setInterval(updateStatus, 500);
|
setInterval(updateStatus, 2000);
|
||||||
</script>
|
</script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
@@ -18,11 +18,19 @@ var asDetailHTML string
|
|||||||
//go:embed prefix_detail.html
|
//go:embed prefix_detail.html
|
||||||
var prefixDetailHTML string
|
var prefixDetailHTML string
|
||||||
|
|
||||||
|
//go:embed prefix_length.html
|
||||||
|
var prefixLengthHTML string
|
||||||
|
|
||||||
// Templates contains all parsed templates
|
// Templates contains all parsed templates
|
||||||
type Templates struct {
|
type Templates struct {
|
||||||
|
// Status is the template for the main status page
|
||||||
Status *template.Template
|
Status *template.Template
|
||||||
|
// ASDetail is the template for displaying AS (Autonomous System) details
|
||||||
ASDetail *template.Template
|
ASDetail *template.Template
|
||||||
|
// PrefixDetail is the template for displaying prefix details
|
||||||
PrefixDetail *template.Template
|
PrefixDetail *template.Template
|
||||||
|
// PrefixLength is the template for displaying prefixes by length
|
||||||
|
PrefixLength *template.Template
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -99,6 +107,12 @@ func initTemplates() {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
panic("failed to parse prefix detail template: " + err.Error())
|
panic("failed to parse prefix detail template: " + err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Parse prefix length template
|
||||||
|
defaultTemplates.PrefixLength, err = template.New("prefixLength").Funcs(funcs).Parse(prefixLengthHTML)
|
||||||
|
if err != nil {
|
||||||
|
panic("failed to parse prefix length template: " + err.Error())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get returns the singleton Templates instance
|
// Get returns the singleton Templates instance
|
||||||
@@ -122,3 +136,8 @@ func ASDetailTemplate() *template.Template {
|
|||||||
func PrefixDetailTemplate() *template.Template {
|
func PrefixDetailTemplate() *template.Template {
|
||||||
return Get().PrefixDetail
|
return Get().PrefixDetail
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PrefixLengthTemplate returns the parsed prefix length template
|
||||||
|
func PrefixLengthTemplate() *template.Template {
|
||||||
|
return Get().PrefixLength
|
||||||
|
}
|
||||||
|
|||||||
347
internal/whois/whois.go
Normal file
347
internal/whois/whois.go
Normal file
@@ -0,0 +1,347 @@
|
|||||||
|
// Package whois provides WHOIS lookup functionality for ASN information.
|
||||||
|
package whois
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Timeout constants for WHOIS queries.
|
||||||
|
const (
|
||||||
|
dialTimeout = 10 * time.Second
|
||||||
|
readTimeout = 30 * time.Second
|
||||||
|
writeTimeout = 5 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// Parsing constants.
|
||||||
|
const (
|
||||||
|
keyValueParts = 2 // Expected parts when splitting "key: value"
|
||||||
|
lacnicDateFormatLen = 8 // Length of YYYYMMDD date format
|
||||||
|
)
|
||||||
|
|
||||||
|
// WHOIS server addresses.
|
||||||
|
const (
|
||||||
|
whoisServerIANA = "whois.iana.org:43"
|
||||||
|
whoisServerARIN = "whois.arin.net:43"
|
||||||
|
whoisServerRIPE = "whois.ripe.net:43"
|
||||||
|
whoisServerAPNIC = "whois.apnic.net:43"
|
||||||
|
whoisServerLACNIC = "whois.lacnic.net:43"
|
||||||
|
whoisServerAFRINIC = "whois.afrinic.net:43"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RIR identifiers.
|
||||||
|
const (
|
||||||
|
RIRARIN = "ARIN"
|
||||||
|
RIRRIPE = "RIPE"
|
||||||
|
RIRAPNIC = "APNIC"
|
||||||
|
RIRLACNIC = "LACNIC"
|
||||||
|
RIRAFRNIC = "AFRINIC"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ASNInfo contains parsed WHOIS information for an ASN.
|
||||||
|
type ASNInfo struct {
|
||||||
|
ASN int
|
||||||
|
ASName string
|
||||||
|
OrgName string
|
||||||
|
OrgID string
|
||||||
|
Address string
|
||||||
|
CountryCode string
|
||||||
|
AbuseEmail string
|
||||||
|
AbusePhone string
|
||||||
|
TechEmail string
|
||||||
|
TechPhone string
|
||||||
|
RIR string
|
||||||
|
RegDate *time.Time
|
||||||
|
LastMod *time.Time
|
||||||
|
RawResponse string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Client performs WHOIS lookups for ASNs.
|
||||||
|
type Client struct {
|
||||||
|
// Dialer for creating connections (can be overridden for testing)
|
||||||
|
dialer *net.Dialer
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewClient creates a new WHOIS client.
|
||||||
|
func NewClient() *Client {
|
||||||
|
return &Client{
|
||||||
|
dialer: &net.Dialer{
|
||||||
|
Timeout: dialTimeout,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// LookupASN queries WHOIS for the given ASN and returns parsed information.
|
||||||
|
func (c *Client) LookupASN(ctx context.Context, asn int) (*ASNInfo, error) {
|
||||||
|
// Query IANA first to find the authoritative RIR
|
||||||
|
query := fmt.Sprintf("AS%d", asn)
|
||||||
|
|
||||||
|
ianaResp, err := c.query(ctx, whoisServerIANA, query)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("IANA query failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Determine RIR from IANA response
|
||||||
|
rir, whoisServer := c.parseIANAReferral(ianaResp)
|
||||||
|
if whoisServer == "" {
|
||||||
|
// No referral, try to parse what we have
|
||||||
|
return c.parseResponse(asn, rir, ianaResp), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Query the authoritative RIR
|
||||||
|
rirResp, err := c.query(ctx, whoisServer, query)
|
||||||
|
if err != nil {
|
||||||
|
// Return partial data from IANA if RIR query fails
|
||||||
|
info := c.parseResponse(asn, rir, ianaResp)
|
||||||
|
info.RawResponse = ianaResp + "\n--- RIR query failed: " + err.Error() + " ---\n"
|
||||||
|
|
||||||
|
return info, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Combine responses and parse
|
||||||
|
fullResponse := ianaResp + "\n" + rirResp
|
||||||
|
info := c.parseResponse(asn, rir, fullResponse)
|
||||||
|
info.RawResponse = fullResponse
|
||||||
|
|
||||||
|
return info, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// query performs a raw WHOIS query to the specified server.
|
||||||
|
func (c *Client) query(ctx context.Context, server, query string) (string, error) {
|
||||||
|
conn, err := c.dialer.DialContext(ctx, "tcp", server)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("dial %s: %w", server, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = conn.Close() }()
|
||||||
|
|
||||||
|
// Set deadlines
|
||||||
|
if err := conn.SetWriteDeadline(time.Now().Add(writeTimeout)); err != nil {
|
||||||
|
return "", fmt.Errorf("set write deadline: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Send query
|
||||||
|
if _, err := fmt.Fprintf(conn, "%s\r\n", query); err != nil {
|
||||||
|
return "", fmt.Errorf("write query: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read response
|
||||||
|
if err := conn.SetReadDeadline(time.Now().Add(readTimeout)); err != nil {
|
||||||
|
return "", fmt.Errorf("set read deadline: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var sb strings.Builder
|
||||||
|
scanner := bufio.NewScanner(conn)
|
||||||
|
for scanner.Scan() {
|
||||||
|
sb.WriteString(scanner.Text())
|
||||||
|
sb.WriteString("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
return sb.String(), fmt.Errorf("read response: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return sb.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseIANAReferral extracts the RIR and WHOIS server from an IANA response.
|
||||||
|
func (c *Client) parseIANAReferral(response string) (rir, whoisServer string) {
|
||||||
|
lines := strings.Split(response, "\n")
|
||||||
|
|
||||||
|
for _, line := range lines {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
|
||||||
|
// Look for "refer:" line
|
||||||
|
if strings.HasPrefix(strings.ToLower(line), "refer:") {
|
||||||
|
server := strings.TrimSpace(strings.TrimPrefix(line, "refer:"))
|
||||||
|
server = strings.TrimSpace(strings.TrimPrefix(server, "Refer:"))
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case strings.Contains(server, "arin"):
|
||||||
|
return RIRARIN, whoisServerARIN
|
||||||
|
case strings.Contains(server, "ripe"):
|
||||||
|
return RIRRIPE, whoisServerRIPE
|
||||||
|
case strings.Contains(server, "apnic"):
|
||||||
|
return RIRAPNIC, whoisServerAPNIC
|
||||||
|
case strings.Contains(server, "lacnic"):
|
||||||
|
return RIRLACNIC, whoisServerLACNIC
|
||||||
|
case strings.Contains(server, "afrinic"):
|
||||||
|
return RIRAFRNIC, whoisServerAFRINIC
|
||||||
|
default:
|
||||||
|
// Unknown server, add port if missing
|
||||||
|
if !strings.Contains(server, ":") {
|
||||||
|
server += ":43"
|
||||||
|
}
|
||||||
|
|
||||||
|
return "", server
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also check organisation line for RIR hints
|
||||||
|
if strings.HasPrefix(strings.ToLower(line), "organisation:") {
|
||||||
|
org := strings.ToLower(line)
|
||||||
|
switch {
|
||||||
|
case strings.Contains(org, "arin"):
|
||||||
|
rir = RIRARIN
|
||||||
|
case strings.Contains(org, "ripe"):
|
||||||
|
rir = RIRRIPE
|
||||||
|
case strings.Contains(org, "apnic"):
|
||||||
|
rir = RIRAPNIC
|
||||||
|
case strings.Contains(org, "lacnic"):
|
||||||
|
rir = RIRLACNIC
|
||||||
|
case strings.Contains(org, "afrinic"):
|
||||||
|
rir = RIRAFRNIC
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return rir, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseResponse extracts ASN information from a WHOIS response.
|
||||||
|
func (c *Client) parseResponse(asn int, rir, response string) *ASNInfo {
|
||||||
|
info := &ASNInfo{
|
||||||
|
ASN: asn,
|
||||||
|
RIR: rir,
|
||||||
|
RawResponse: response,
|
||||||
|
}
|
||||||
|
|
||||||
|
lines := strings.Split(response, "\n")
|
||||||
|
var addressLines []string
|
||||||
|
|
||||||
|
for _, line := range lines {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" || strings.HasPrefix(line, "%") || strings.HasPrefix(line, "#") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Split on first colon
|
||||||
|
parts := strings.SplitN(line, ":", keyValueParts)
|
||||||
|
if len(parts) != keyValueParts {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
key := strings.TrimSpace(strings.ToLower(parts[0]))
|
||||||
|
value := strings.TrimSpace(parts[1])
|
||||||
|
|
||||||
|
if value == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
switch key {
|
||||||
|
// AS Name (varies by RIR)
|
||||||
|
case "asname", "as-name":
|
||||||
|
if info.ASName == "" {
|
||||||
|
info.ASName = value
|
||||||
|
}
|
||||||
|
|
||||||
|
// Organization
|
||||||
|
case "orgname", "org-name", "owner":
|
||||||
|
if info.OrgName == "" {
|
||||||
|
info.OrgName = value
|
||||||
|
}
|
||||||
|
case "orgid", "org-id", "org":
|
||||||
|
if info.OrgID == "" {
|
||||||
|
info.OrgID = value
|
||||||
|
}
|
||||||
|
|
||||||
|
// Address (collect multiple lines)
|
||||||
|
case "address":
|
||||||
|
addressLines = append(addressLines, value)
|
||||||
|
|
||||||
|
// Country
|
||||||
|
case "country":
|
||||||
|
if info.CountryCode == "" && len(value) == 2 {
|
||||||
|
info.CountryCode = strings.ToUpper(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Abuse contact
|
||||||
|
case "orgabuseemail", "abuse-mailbox":
|
||||||
|
if info.AbuseEmail == "" {
|
||||||
|
info.AbuseEmail = value
|
||||||
|
}
|
||||||
|
case "orgabusephone":
|
||||||
|
if info.AbusePhone == "" {
|
||||||
|
info.AbusePhone = value
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tech contact
|
||||||
|
case "orgtechemail":
|
||||||
|
if info.TechEmail == "" {
|
||||||
|
info.TechEmail = value
|
||||||
|
}
|
||||||
|
case "orgtechphone":
|
||||||
|
if info.TechPhone == "" {
|
||||||
|
info.TechPhone = value
|
||||||
|
}
|
||||||
|
|
||||||
|
// Registration dates
|
||||||
|
case "regdate", "created":
|
||||||
|
if info.RegDate == nil {
|
||||||
|
info.RegDate = c.parseDate(value)
|
||||||
|
}
|
||||||
|
case "updated", "last-modified", "changed":
|
||||||
|
if info.LastMod == nil {
|
||||||
|
info.LastMod = c.parseDate(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Combine address lines
|
||||||
|
if len(addressLines) > 0 {
|
||||||
|
info.Address = strings.Join(addressLines, "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract abuse email from comment lines (common in ARIN responses)
|
||||||
|
if info.AbuseEmail == "" {
|
||||||
|
info.AbuseEmail = c.extractAbuseEmail(response)
|
||||||
|
}
|
||||||
|
|
||||||
|
return info
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseDate attempts to parse various date formats used in WHOIS responses.
|
||||||
|
func (c *Client) parseDate(value string) *time.Time {
|
||||||
|
// Common formats
|
||||||
|
formats := []string{
|
||||||
|
"2006-01-02",
|
||||||
|
"2006-01-02T15:04:05Z",
|
||||||
|
"2006-01-02T15:04:05-07:00",
|
||||||
|
"20060102",
|
||||||
|
"02-Jan-2006",
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up value
|
||||||
|
value = strings.TrimSpace(value)
|
||||||
|
// Handle "YYYYMMDD" format from LACNIC
|
||||||
|
if len(value) == lacnicDateFormatLen {
|
||||||
|
if _, err := time.Parse("20060102", value); err == nil {
|
||||||
|
t, _ := time.Parse("20060102", value)
|
||||||
|
|
||||||
|
return &t
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, format := range formats {
|
||||||
|
if t, err := time.Parse(format, value); err == nil {
|
||||||
|
return &t
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractAbuseEmail extracts abuse email from response using regex.
|
||||||
|
func (c *Client) extractAbuseEmail(response string) string {
|
||||||
|
// Look for "Abuse contact for 'AS...' is 'email@domain'"
|
||||||
|
re := regexp.MustCompile(`[Aa]buse contact.*?is\s+['"]?([^\s'"]+@[^\s'"]+)['"]?`)
|
||||||
|
if matches := re.FindStringSubmatch(response); len(matches) > 1 {
|
||||||
|
return matches[1]
|
||||||
|
}
|
||||||
|
|
||||||
|
return ""
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user