Bound SQLite memory across the whole connection pool (closes #8)

The 3 GiB page cache and temp_store=MEMORY were set once in Initialize, so
only one pooled connection carried them and the other nine got no busy_timeout,
which drove many "database is locked" errors. None of that memory was visible
to the Go runtime.

Move the per-connection settings into the DSN so every pooled connection gets a
64 MiB cache (640 MiB worst case over ten connections), synchronous OFF, a
5 s busy_timeout and WAL. Drop cache_size, temp_store, synchronous and
busy_timeout from the Initialize pragmas; DISTINCT temp B-trees now spill to
disk. Add process-wide soft (1 GiB) and hard (1.5 GiB) heap limits; at the hard
limit a statement returns SQLITE_NOMEM and the existing batch paths already log
and drop, so nothing panics or exits.

Model: opus-4-8
This commit is contained in:
2026-09-21 13:04:03 +00:00
parent 54014c88c8
commit ca58b77e32
2 changed files with 107 additions and 8 deletions
+79
View File
@@ -1,10 +1,23 @@
package database
import (
"context"
"database/sql"
"net"
"testing"
"git.eeqj.de/sneak/routewatch/internal/config"
"git.eeqj.de/sneak/routewatch/internal/logger"
)
// tempStoreMemory is the PRAGMA temp_store value meaning "hold temp B-trees in
// memory"; the DSN change must leave temp_store below this so they spill to disk.
const tempStoreMemory = 2
// heldConnections is how many pooled connections the pragma test holds open at
// once so each is a distinct SQLite connection that parsed the DSN.
const heldConnections = 5
func TestIPToUint32(t *testing.T) {
tests := []struct {
name string
@@ -282,6 +295,72 @@ func TestIPv4RangeIntegration(t *testing.T) {
}
}
// TestConnectionPoolPragmas holds several pooled connections open at once and
// checks each one carries the per-connection settings from the DSN, plus the
// process-wide hard heap limit.
func TestConnectionPoolPragmas(t *testing.T) {
cfg := &config.Config{StateDir: t.TempDir()}
db, err := New(cfg, logger.New())
if err != nil {
t.Fatalf("failed to create database: %v", err)
}
defer func() { _ = db.Close() }()
ctx := context.Background()
// Hold distinct connections open simultaneously so the pool must open a new
// one (each parsing the DSN) rather than hand back the same connection.
conns := make([]*sql.Conn, 0, heldConnections)
defer func() {
for _, c := range conns {
_ = c.Close()
}
}()
for i := 0; i < heldConnections; i++ {
c, err := db.db.Conn(ctx)
if err != nil {
t.Fatalf("failed to open connection %d: %v", i, err)
}
conns = append(conns, c)
}
for i, c := range conns {
var cacheSize int
if err := c.QueryRowContext(ctx, "PRAGMA cache_size").Scan(&cacheSize); err != nil {
t.Fatalf("conn %d: failed to read cache_size: %v", i, err)
}
if cacheSize != sqliteCacheSizeKiB {
t.Errorf("conn %d: cache_size = %d, want %d", i, cacheSize, sqliteCacheSizeKiB)
}
var busyTimeout int
if err := c.QueryRowContext(ctx, "PRAGMA busy_timeout").Scan(&busyTimeout); err != nil {
t.Fatalf("conn %d: failed to read busy_timeout: %v", i, err)
}
if busyTimeout != sqliteBusyTimeoutMs {
t.Errorf("conn %d: busy_timeout = %d, want %d", i, busyTimeout, sqliteBusyTimeoutMs)
}
var tempStore int
if err := c.QueryRowContext(ctx, "PRAGMA temp_store").Scan(&tempStore); err != nil {
t.Fatalf("conn %d: failed to read temp_store: %v", i, err)
}
if tempStore == tempStoreMemory {
t.Errorf("conn %d: temp_store = %d, want anything but %d (MEMORY)", i, tempStore, tempStoreMemory)
}
var hardHeapLimit int64
if err := c.QueryRowContext(ctx, "PRAGMA hard_heap_limit").Scan(&hardHeapLimit); err != nil {
t.Fatalf("conn %d: failed to read hard_heap_limit: %v", i, err)
}
if hardHeapLimit != sqliteHardHeapLimitBytes {
t.Errorf("conn %d: hard_heap_limit = %d, want %d", i, hardHeapLimit, sqliteHardHeapLimitBytes)
}
}
}
func BenchmarkIPToUint32(b *testing.B) {
ip := net.ParseIP("192.168.1.1")
b.ResetTimer()