Make live route writes cheaper so the prefix handler keeps up (closes #96)
check / check (push) Canceled after 0s
check / check (push) Canceled after 0s
The live route tables lose their random UUID key; the row id the prefix-length sample draws is declared as the integer key. The prefix tables are keyed by prefix and keep only first_seen; a batch inserts only prefixes not stored yet. A route update sets only the columns that change. Each kept index names the queries using it. Every write goes through one connection kept for writing, with a 1 GiB page cache; reading connections get 16 MiB, heap limits rise to 1.5 and 2 GiB (README "Memory"). Batch writes go in key order. A benchmark run by hand writes 250,000 updates on 5 million routes faster than the feed's 10,000 a second; a live run follows. Model: opus-5-5
This commit was merged in pull request #98.
This commit is contained in:
1 parent
5806ddaef7
commit
d84dd56460
10 files changed
+708
-253
No files matched your search
@@ -18,7 +18,6 @@ import (
|
||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
||||
"git.eeqj.de/sneak/routewatch/internal/server"
|
||||
"git.eeqj.de/sneak/routewatch/internal/streamer"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// mockStore is a mock implementation of database.Store for testing
|
||||
@@ -83,9 +82,7 @@ func (m *mockStore) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
|
||||
defer m.mu.Unlock()
|
||||
|
||||
for prefix, timestamp := range prefixes {
|
||||
if p, exists := m.Prefixes[prefix]; exists {
|
||||
p.LastSeen = timestamp
|
||||
} else {
|
||||
if _, exists := m.Prefixes[prefix]; !exists {
|
||||
const (
|
||||
ipVersionV4 = 4
|
||||
ipVersionV6 = 6
|
||||
@@ -97,11 +94,9 @@ func (m *mockStore) UpdatePrefixesBatch(prefixes map[string]time.Time) error {
|
||||
}
|
||||
|
||||
m.Prefixes[prefix] = &database.Prefix{
|
||||
ID: uuid.New(),
|
||||
Prefix: prefix,
|
||||
IPVersion: ipVersion,
|
||||
FirstSeen: timestamp,
|
||||
LastSeen: timestamp,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -117,8 +112,6 @@ func (m *mockStore) GetOrCreatePrefix(
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if p, exists := m.Prefixes[prefix]; exists {
|
||||
p.LastSeen = timestamp
|
||||
|
||||
return p, nil
|
||||
}
|
||||
|
||||
@@ -133,11 +126,9 @@ func (m *mockStore) GetOrCreatePrefix(
|
||||
}
|
||||
|
||||
p := &database.Prefix{
|
||||
ID: uuid.New(),
|
||||
Prefix: prefix,
|
||||
IPVersion: ipVersion,
|
||||
FirstSeen: timestamp,
|
||||
LastSeen: timestamp,
|
||||
}
|
||||
m.Prefixes[prefix] = p
|
||||
m.PrefixCount++
|
||||
@@ -370,11 +361,9 @@ func (m *mockStore) UpsertLiveRouteBatch(routes []*database.LiveRoute) error {
|
||||
// Track prefix
|
||||
if _, exists := m.Prefixes[route.Prefix]; !exists {
|
||||
m.Prefixes[route.Prefix] = &database.Prefix{
|
||||
ID: uuid.New(),
|
||||
Prefix: route.Prefix,
|
||||
IPVersion: route.IPVersion,
|
||||
FirstSeen: route.LastUpdated,
|
||||
LastSeen: route.LastUpdated,
|
||||
}
|
||||
m.PrefixCount++
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"git.eeqj.de/sneak/routewatch/internal/logger"
|
||||
"git.eeqj.de/sneak/routewatch/internal/metrics"
|
||||
"git.eeqj.de/sneak/routewatch/internal/ristypes"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -461,7 +460,6 @@ func (h *PrefixHandler) createLiveRoute(update prefixUpdate) *database.LiveRoute
|
||||
|
||||
// Create live route record
|
||||
liveRoute := &database.LiveRoute{
|
||||
ID: uuid.New(),
|
||||
Prefix: update.prefix,
|
||||
MaskLength: maskLength,
|
||||
IPVersion: ipVersion,
|
||||
|
||||
Reference in new issue
Block a user