The WAL file was growing to 700MB+ which caused COUNT(*) queries to timeout. Reads must scan the WAL to find current page versions, and a large WAL makes this slow. Add Checkpoint method to database interface and run PASSIVE checkpoints every 30 seconds via the DBMaintainer. This keeps the WAL small and maintains fast read performance under heavy write load.
493 lines
12 KiB
Go
493 lines
12 KiB
Go
package routewatch
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.eeqj.de/sneak/routewatch/internal/config"
|
|
"git.eeqj.de/sneak/routewatch/internal/database"
|
|
"git.eeqj.de/sneak/routewatch/internal/logger"
|
|
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
|
"git.eeqj.de/sneak/routewatch/internal/server"
|
|
"git.eeqj.de/sneak/routewatch/internal/streamer"
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
// mockStore is a mock implementation of database.Store for testing
|
|
type mockStore struct {
|
|
mu sync.Mutex
|
|
|
|
// Counters for tracking calls
|
|
ASNCount int
|
|
PrefixCount int
|
|
PeeringCount int
|
|
RouteCount int
|
|
WithdrawalCount int
|
|
|
|
// Track unique items
|
|
ASNs map[int]*database.ASN
|
|
Prefixes map[string]*database.Prefix
|
|
Peerings map[string]bool // key is "from_to"
|
|
Routes map[string]bool // key is "prefix_origin_peer"
|
|
|
|
// Track IP versions
|
|
IPv4Prefixes int
|
|
IPv6Prefixes int
|
|
}
|
|
|
|
// newMockStore creates a new mock store
|
|
func newMockStore() *mockStore {
|
|
return &mockStore{
|
|
ASNs: make(map[int]*database.ASN),
|
|
Prefixes: make(map[string]*database.Prefix),
|
|
Peerings: make(map[string]bool),
|
|
Routes: make(map[string]bool),
|
|
}
|
|
}
|
|
|
|
// GetOrCreateASN mock implementation
|
|
func (m *mockStore) GetOrCreateASN(number int, timestamp time.Time) (*database.ASN, error) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
if asn, exists := m.ASNs[number]; exists {
|
|
asn.LastSeen = timestamp
|
|
|
|
return asn, nil
|
|
}
|
|
|
|
asn := &database.ASN{
|
|
ASN: number,
|
|
FirstSeen: timestamp,
|
|
LastSeen: timestamp,
|
|
}
|
|
m.ASNs[number] = asn
|
|
m.ASNCount++
|
|
|
|
return asn, nil
|
|
}
|
|
|
|
// UpdatePrefixesBatch mock implementation
|
|
func (m *mockStore) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
for prefix, timestamp := range prefixes {
|
|
if p, exists := m.Prefixes[prefix]; exists {
|
|
p.LastSeen = timestamp
|
|
} else {
|
|
const (
|
|
ipVersionV4 = 4
|
|
ipVersionV6 = 6
|
|
)
|
|
|
|
ipVersion := ipVersionV4
|
|
if strings.Contains(prefix, ":") {
|
|
ipVersion = ipVersionV6
|
|
}
|
|
|
|
m.Prefixes[prefix] = &database.Prefix{
|
|
ID: uuid.New(),
|
|
Prefix: prefix,
|
|
IPVersion: ipVersion,
|
|
FirstSeen: timestamp,
|
|
LastSeen: timestamp,
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// GetOrCreatePrefix mock implementation
|
|
func (m *mockStore) GetOrCreatePrefix(prefix string, timestamp time.Time) (*database.Prefix, error) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
if p, exists := m.Prefixes[prefix]; exists {
|
|
p.LastSeen = timestamp
|
|
|
|
return p, nil
|
|
}
|
|
|
|
const (
|
|
ipVersionV4 = 4
|
|
ipVersionV6 = 6
|
|
)
|
|
|
|
ipVersion := ipVersionV4
|
|
if strings.Contains(prefix, ":") {
|
|
ipVersion = ipVersionV6
|
|
}
|
|
|
|
p := &database.Prefix{
|
|
ID: uuid.New(),
|
|
Prefix: prefix,
|
|
IPVersion: ipVersion,
|
|
FirstSeen: timestamp,
|
|
LastSeen: timestamp,
|
|
}
|
|
m.Prefixes[prefix] = p
|
|
m.PrefixCount++
|
|
|
|
if ipVersion == ipVersionV4 {
|
|
m.IPv4Prefixes++
|
|
} else {
|
|
m.IPv6Prefixes++
|
|
}
|
|
|
|
return p, nil
|
|
}
|
|
|
|
// RecordAnnouncement mock implementation
|
|
func (m *mockStore) RecordAnnouncement(_ *database.Announcement) error {
|
|
// Not tracking announcements in detail for now
|
|
return nil
|
|
}
|
|
|
|
// RecordPeering mock implementation
|
|
func (m *mockStore) RecordPeering(asA, asB int, _ time.Time) error {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
// Normalize
|
|
if asA > asB {
|
|
asA, asB = asB, asA
|
|
}
|
|
|
|
key := fmt.Sprintf("%d_%d", asA, asB)
|
|
if !m.Peerings[key] {
|
|
m.Peerings[key] = true
|
|
m.PeeringCount++
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// UpdatePeer mock implementation
|
|
func (m *mockStore) UpdatePeer(peerIP string, peerASN int, messageType string, timestamp time.Time) error {
|
|
// Simple mock - just return nil
|
|
return nil
|
|
}
|
|
|
|
// Close mock implementation
|
|
func (m *mockStore) Close() error {
|
|
return nil
|
|
}
|
|
|
|
// GetStats returns statistics about the mock store
|
|
func (m *mockStore) GetStats() (database.Stats, error) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
return database.Stats{
|
|
ASNs: len(m.ASNs),
|
|
Prefixes: len(m.Prefixes),
|
|
IPv4Prefixes: m.IPv4Prefixes,
|
|
IPv6Prefixes: m.IPv6Prefixes,
|
|
Peerings: m.PeeringCount,
|
|
Peers: 10, // Mock peer count
|
|
}, nil
|
|
}
|
|
|
|
// GetStatsContext returns statistics about the mock store with context support
|
|
func (m *mockStore) GetStatsContext(ctx context.Context) (database.Stats, error) {
|
|
return m.GetStats()
|
|
}
|
|
|
|
// UpsertLiveRoute mock implementation
|
|
func (m *mockStore) UpsertLiveRoute(route *database.LiveRoute) error {
|
|
// Simple mock - just return nil
|
|
return nil
|
|
}
|
|
|
|
// DeleteLiveRoute mock implementation
|
|
func (m *mockStore) DeleteLiveRoute(prefix string, originASN int, peerIP string) error {
|
|
// Simple mock - just return nil
|
|
return nil
|
|
}
|
|
|
|
// GetPrefixDistribution mock implementation
|
|
func (m *mockStore) GetPrefixDistribution() (ipv4 []database.PrefixDistribution, ipv6 []database.PrefixDistribution, err error) {
|
|
// Return empty distributions for now
|
|
return nil, nil, nil
|
|
}
|
|
|
|
// GetPrefixDistributionContext mock implementation with context support
|
|
func (m *mockStore) GetPrefixDistributionContext(ctx context.Context) (ipv4 []database.PrefixDistribution, ipv6 []database.PrefixDistribution, err error) {
|
|
return m.GetPrefixDistribution()
|
|
}
|
|
|
|
// GetLiveRouteCounts mock implementation
|
|
func (m *mockStore) GetLiveRouteCounts() (ipv4Count, ipv6Count int, err error) {
|
|
// Return mock counts
|
|
return m.RouteCount / 2, m.RouteCount / 2, nil
|
|
}
|
|
|
|
// GetLiveRouteCountsContext mock implementation with context support
|
|
func (m *mockStore) GetLiveRouteCountsContext(ctx context.Context) (ipv4Count, ipv6Count int, err error) {
|
|
return m.GetLiveRouteCounts()
|
|
}
|
|
|
|
// GetASInfoForIP mock implementation
|
|
func (m *mockStore) GetASInfoForIP(ip string) (*database.ASInfo, error) {
|
|
// Simple mock - return a test AS
|
|
now := time.Now()
|
|
return &database.ASInfo{
|
|
ASN: 15169,
|
|
Handle: "GOOGLE",
|
|
Description: "Google LLC",
|
|
Prefix: "8.8.8.0/24",
|
|
LastUpdated: now.Add(-5 * time.Minute),
|
|
Age: "5m0s",
|
|
}, nil
|
|
}
|
|
|
|
// GetASInfoForIPContext mock implementation with context support
|
|
func (m *mockStore) GetASInfoForIPContext(ctx context.Context, ip string) (*database.ASInfo, error) {
|
|
return m.GetASInfoForIP(ip)
|
|
}
|
|
|
|
// GetASDetails mock implementation
|
|
func (m *mockStore) GetASDetails(asn int) (*database.ASN, []database.LiveRoute, error) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
// Check if ASN exists
|
|
if asnInfo, exists := m.ASNs[asn]; exists {
|
|
// Return empty prefixes for now
|
|
return asnInfo, []database.LiveRoute{}, nil
|
|
}
|
|
|
|
return nil, nil, database.ErrNoRoute
|
|
}
|
|
|
|
// GetASDetailsContext mock implementation with context support
|
|
func (m *mockStore) GetASDetailsContext(ctx context.Context, asn int) (*database.ASN, []database.LiveRoute, error) {
|
|
return m.GetASDetails(asn)
|
|
}
|
|
|
|
// GetPrefixDetails mock implementation
|
|
func (m *mockStore) GetPrefixDetails(prefix string) ([]database.LiveRoute, error) {
|
|
// Return empty routes for now
|
|
return []database.LiveRoute{}, nil
|
|
}
|
|
|
|
// GetPrefixDetailsContext mock implementation with context support
|
|
func (m *mockStore) GetPrefixDetailsContext(ctx context.Context, prefix string) ([]database.LiveRoute, error) {
|
|
return m.GetPrefixDetails(prefix)
|
|
}
|
|
|
|
func (m *mockStore) GetRandomPrefixesByLength(maskLength, ipVersion, limit int) ([]database.LiveRoute, error) {
|
|
// Return empty routes for now
|
|
return []database.LiveRoute{}, nil
|
|
}
|
|
|
|
// GetRandomPrefixesByLengthContext mock implementation with context support
|
|
func (m *mockStore) GetRandomPrefixesByLengthContext(ctx context.Context, maskLength, ipVersion, limit int) ([]database.LiveRoute, error) {
|
|
return m.GetRandomPrefixesByLength(maskLength, ipVersion, limit)
|
|
}
|
|
|
|
// GetASPeers mock implementation
|
|
func (m *mockStore) GetASPeers(asn int) ([]database.ASPeer, error) {
|
|
// Return empty peers for now
|
|
return []database.ASPeer{}, nil
|
|
}
|
|
|
|
// GetASPeersContext mock implementation with context support
|
|
func (m *mockStore) GetASPeersContext(ctx context.Context, asn int) ([]database.ASPeer, error) {
|
|
return m.GetASPeers(asn)
|
|
}
|
|
|
|
// GetIPInfo mock implementation
|
|
func (m *mockStore) GetIPInfo(ip string) (*database.IPInfo, error) {
|
|
return m.GetIPInfoContext(context.Background(), ip)
|
|
}
|
|
|
|
// GetIPInfoContext mock implementation with context support
|
|
func (m *mockStore) GetIPInfoContext(ctx context.Context, ip string) (*database.IPInfo, error) {
|
|
now := time.Now()
|
|
return &database.IPInfo{
|
|
IP: ip,
|
|
Netblock: "8.8.8.0/24",
|
|
MaskLength: 24,
|
|
IPVersion: 4,
|
|
NumPeers: 3,
|
|
ASN: 15169,
|
|
Handle: "GOOGLE",
|
|
Description: "Google LLC",
|
|
CountryCode: "US",
|
|
FirstSeen: now.Add(-24 * time.Hour),
|
|
LastSeen: now,
|
|
}, nil
|
|
}
|
|
|
|
// GetNextStaleASN mock implementation
|
|
func (m *mockStore) GetNextStaleASN(ctx context.Context, staleThreshold time.Duration) (int, error) {
|
|
return 0, database.ErrNoStaleASN
|
|
}
|
|
|
|
// UpdateASNWHOIS mock implementation
|
|
func (m *mockStore) UpdateASNWHOIS(ctx context.Context, update *database.ASNWHOISUpdate) error {
|
|
return nil
|
|
}
|
|
|
|
// GetWHOISStats mock implementation
|
|
func (m *mockStore) GetWHOISStats(ctx context.Context, staleThreshold time.Duration) (*database.WHOISStats, error) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
return &database.WHOISStats{
|
|
TotalASNs: len(m.ASNs),
|
|
FreshASNs: 0,
|
|
StaleASNs: 0,
|
|
NeverFetched: len(m.ASNs),
|
|
}, nil
|
|
}
|
|
|
|
// UpsertLiveRouteBatch mock implementation
|
|
func (m *mockStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) error {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
for _, route := range routes {
|
|
// Track prefix
|
|
if _, exists := m.Prefixes[route.Prefix]; !exists {
|
|
m.Prefixes[route.Prefix] = &database.Prefix{
|
|
ID: uuid.New(),
|
|
Prefix: route.Prefix,
|
|
IPVersion: route.IPVersion,
|
|
FirstSeen: route.LastUpdated,
|
|
LastSeen: route.LastUpdated,
|
|
}
|
|
m.PrefixCount++
|
|
if route.IPVersion == 4 {
|
|
m.IPv4Prefixes++
|
|
} else {
|
|
m.IPv6Prefixes++
|
|
}
|
|
}
|
|
m.RouteCount++
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// DeleteLiveRouteBatch mock implementation
|
|
func (m *mockStore) DeleteLiveRouteBatch(deletions []database.LiveRouteDeletion) error {
|
|
// Simple mock - just return nil
|
|
return nil
|
|
}
|
|
|
|
// GetOrCreateASNBatch mock implementation
|
|
func (m *mockStore) GetOrCreateASNBatch(asns map[int]time.Time) error {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
for number, timestamp := range asns {
|
|
if _, exists := m.ASNs[number]; !exists {
|
|
m.ASNs[number] = &database.ASN{
|
|
ASN: number,
|
|
FirstSeen: timestamp,
|
|
LastSeen: timestamp,
|
|
}
|
|
m.ASNCount++
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// UpdatePeerBatch mock implementation
|
|
func (m *mockStore) UpdatePeerBatch(peers map[string]database.PeerUpdate) error {
|
|
// Simple mock - just return nil
|
|
return nil
|
|
}
|
|
|
|
// Vacuum mock implementation
|
|
func (m *mockStore) Vacuum(ctx context.Context) error {
|
|
return nil
|
|
}
|
|
|
|
// Analyze mock implementation
|
|
func (m *mockStore) Analyze(ctx context.Context) error {
|
|
return nil
|
|
}
|
|
|
|
// Checkpoint mock implementation
|
|
func (m *mockStore) Checkpoint(ctx context.Context) error {
|
|
return nil
|
|
}
|
|
|
|
func TestRouteWatchLiveFeed(t *testing.T) {
|
|
|
|
// Create mock database
|
|
mockDB := newMockStore()
|
|
defer mockDB.Close()
|
|
|
|
logger := logger.New()
|
|
|
|
// Create metrics tracker
|
|
metricsTracker := metrics.New()
|
|
|
|
// Create streamer
|
|
s := streamer.New(logger, metricsTracker)
|
|
|
|
// Create test config with empty state dir (no snapshot loading)
|
|
cfg := &config.Config{
|
|
StateDir: "",
|
|
MaxRuntime: 5 * time.Second,
|
|
EnableBatchedDatabaseWrites: true,
|
|
}
|
|
|
|
// Create server
|
|
srv := server.New(mockDB, s, logger)
|
|
|
|
// Create RouteWatch with 5 second limit
|
|
deps := Dependencies{
|
|
DB: mockDB,
|
|
Streamer: s,
|
|
Server: srv,
|
|
Logger: logger,
|
|
Config: cfg,
|
|
}
|
|
rw := New(deps)
|
|
|
|
// Run with context
|
|
ctx := context.Background()
|
|
go func() {
|
|
_ = rw.Run(ctx)
|
|
}()
|
|
|
|
// Wait for the configured duration
|
|
time.Sleep(5 * time.Second)
|
|
|
|
// Force peering processing for test
|
|
if rw.peeringHandler != nil {
|
|
rw.peeringHandler.ProcessPeeringsNow()
|
|
}
|
|
|
|
// Get statistics
|
|
stats, err := mockDB.GetStats()
|
|
if err != nil {
|
|
t.Fatalf("Failed to get stats: %v", err)
|
|
}
|
|
|
|
if stats.ASNs == 0 {
|
|
t.Error("Expected to receive some ASNs from live feed")
|
|
}
|
|
t.Logf("Received %d unique ASNs in 5 seconds", stats.ASNs)
|
|
|
|
if stats.Prefixes == 0 {
|
|
t.Error("Expected to receive some prefixes from live feed")
|
|
}
|
|
t.Logf("Received %d unique prefixes (%d IPv4, %d IPv6) in 5 seconds", stats.Prefixes, stats.IPv4Prefixes, stats.IPv6Prefixes)
|
|
|
|
if stats.Peerings == 0 {
|
|
t.Error("Expected to receive some peerings from live feed")
|
|
}
|
|
t.Logf("Recorded %d AS peering relationships in 5 seconds", stats.Peerings)
|
|
|
|
}
|