1 Commits
Author SHA1 Message Date
sneak 4c2882c19c Write batches through one connection with a 1 GiB cache, in key order (closes #96)
check / check (push) Canceled after 0s
Every write now uses one connection taken from the pool when the
database opens, under the database lock. Its 1 GiB page cache keeps
the pages the route writes touch from one batch to the next. A reading
connection's cache is emptied whenever another connection has written,
so the 9 reading connections drop to 16 MiB each. The soft and hard
heap limits rise to 1.5 and 2 GiB, above the caches; README "Memory"
adds up the new budget. Vacuum, Analyze and Checkpoint check their
context after the lock, which the kept connection does not. Batch
writes sort their rows by peer, then prefix. The benchmark also times
the largest prefix handler batch, 250,000 updates.

Model: opus-5-5
2026-10-19 12:53:52 +00:00
+49 -21
View File
@@ -341,9 +341,7 @@ func writeOriginRoutes(t *testing.T, db *Database, origins map[string]int) {
// TestConnectionPoolPragmas holds several pooled connections open at once and
// checks each one carries the per-connection settings from the DSN, plus the
// process-wide heap limits, and sees the new file with auto_vacuum
// incremental. The connection kept for writing must have the large cache, and
// the soft heap limit must leave room for every connection's cache: SQLite
// shrinks the caches to stay under it.
// incremental.
func TestConnectionPoolPragmas(t *testing.T) {
t.Parallel()
@@ -412,8 +410,23 @@ func TestConnectionPoolPragmas(t *testing.T) {
i, autoVacuum, autoVacuumIncremental)
}
}
}
var writeCacheSize int64
// TestWritingConnectionCache checks that the connection kept for writing has
// the large cache, and that the soft heap limit is above the caches of all the
// connections together: SQLite shrinks the caches to stay under it.
func TestWritingConnectionCache(t *testing.T) {
t.Parallel()
db, err := New(&config.Config{StateDir: t.TempDir()}, logger.New())
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
defer func() { _ = db.Close() }()
ctx := t.Context()
var writeCacheSize, readCacheSize, softHeapLimit int64
err = db.writer.QueryRowContext(ctx, "PRAGMA cache_size").
Scan(&writeCacheSize)
@@ -426,11 +439,18 @@ func TestConnectionPoolPragmas(t *testing.T) {
writeCacheSize, sqliteWriteCacheSizeKiB)
}
readCacheSize := readPragma(ctx, t, conns[0], 0, "cache_size")
softHeapLimit := readPragma(ctx, t, conns[0], 0, "soft_heap_limit")
cachesBytes := -bytesPerKiB *
(writeCacheSize + (maxConns-1)*readCacheSize)
err = db.db.QueryRowContext(ctx, "PRAGMA cache_size").Scan(&readCacheSize)
if err != nil {
t.Fatalf("failed to read a reading connection's cache_size: %v", err)
}
err = db.db.QueryRowContext(ctx, "PRAGMA soft_heap_limit").
Scan(&softHeapLimit)
if err != nil {
t.Fatalf("failed to read soft_heap_limit: %v", err)
}
cachesBytes := -bytesPerKiB * (writeCacheSize + (maxConns-1)*readCacheSize)
if softHeapLimit <= cachesBytes {
t.Errorf("soft_heap_limit = %d, not above the %d bytes of the caches",
softHeapLimit, cachesBytes)
@@ -1278,7 +1298,13 @@ func TestUpsertLiveRouteBatchWritesInKeyOrder(t *testing.T) {
}
defer func() { _ = db.Close() }()
const firstOrigin, laterOrigin = 64500, 64501
const (
firstOrigin, laterOrigin = 64500, 64501
// Three prefixes, in text order.
firstPrefix = "192.0.2.0/24"
secondPrefix = "198.51.100.0/24"
thirdPrefix = "203.0.113.0/24"
)
type row struct {
peer, prefix string
@@ -1287,15 +1313,17 @@ func TestUpsertLiveRouteBatchWritesInKeyOrder(t *testing.T) {
ts := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
var routes []*LiveRoute
announcements := []row{
{secondV4PeerIP, secondPrefix, firstOrigin},
{testV4PeerIP, thirdPrefix, firstOrigin},
{secondV4PeerIP, firstPrefix, firstOrigin},
{testV4PeerIP, secondPrefix, firstOrigin},
{secondV4PeerIP, secondPrefix, laterOrigin},
}
for _, announced := range []row{
{secondV4PeerIP, "198.51.100.0/24", firstOrigin},
{testV4PeerIP, "203.0.113.0/24", firstOrigin},
{secondV4PeerIP, "192.0.2.0/24", firstOrigin},
{testV4PeerIP, "198.51.100.0/24", firstOrigin},
{secondV4PeerIP, "198.51.100.0/24", laterOrigin},
} {
routes := make([]*LiveRoute, 0, len(announcements))
for _, announced := range announcements {
route := mkV4Route(t, announced.prefix, announced.origin, ts)
route.PeerIP = announced.peer
routes = append(routes, route)
@@ -1332,10 +1360,10 @@ func TestUpsertLiveRouteBatchWritesInKeyOrder(t *testing.T) {
}
want := []row{
{testV4PeerIP, "198.51.100.0/24", firstOrigin},
{testV4PeerIP, "203.0.113.0/24", firstOrigin},
{secondV4PeerIP, "192.0.2.0/24", firstOrigin},
{secondV4PeerIP, "198.51.100.0/24", laterOrigin},
{testV4PeerIP, secondPrefix, firstOrigin},
{testV4PeerIP, thirdPrefix, firstOrigin},
{secondV4PeerIP, firstPrefix, firstOrigin},
{secondV4PeerIP, secondPrefix, laterOrigin},
}
if !slices.Equal(stored, want) {