diff --git a/TODO.md b/TODO.md index b329b6e..ca3f73e 100644 --- a/TODO.md +++ b/TODO.md @@ -28,6 +28,11 @@ The other open issue is https://git.eeqj.de/sneak/routewatch/issues/30. # Completed Steps +- 2026-10-03: a new database is created with `auto_vacuum` set to + incremental, through the connection string so it is set before the file + is first written, and the periodic incremental vacuum now returns free + pages; the late `PRAGMA auto_vacuum` in `Initialize`, which SQLite + ignored, is gone (closes #43) - 2026-10-02: a plain `docker build .` stamps the commit's tag or short commit (`git describe --tags --always`) into the page footer instead of `unknown`: `.dockerignore` sends `.git` without `.git/config`, a `VERSION` diff --git a/internal/database/database.go b/internal/database/database.go index 4007b48..91e6351 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -94,8 +94,13 @@ func New(cfg *config.Config, logger *logger.Logger) (*Database, error) { // a transaction that reads before writing starts as a reader and, when it // then writes while another connection holds the write lock, fails at once // with "database is locked" without waiting for _busy_timeout. + // _auto_vacuum=incremental lets Vacuum return free pages to the filesystem. + // SQLite only accepts it before the database file is first written, and the + // switch to WAL writes it, so it must be here: the driver applies it on open, + // before _journal_mode. On an existing file it changes nothing. dsn := fmt.Sprintf( - "file:%s?_cache_size=%d&_synchronous=OFF&_busy_timeout=%d&_journal_mode=WAL&_txlock=immediate", + "file:%s?_cache_size=%d&_synchronous=OFF&_busy_timeout=%d"+ + "&_auto_vacuum=incremental&_journal_mode=WAL&_txlock=immediate", dbPath, sqliteCacheSizeKiB, sqliteBusyTimeoutMs, @@ -140,9 +145,8 @@ func (d *Database) Initialize() error { // heap. The heap limits below are process-wide, so setting them once here is // enough for the whole pool. pragmas := []string{ - "PRAGMA journal_mode=WAL", // Write-Ahead Logging - "PRAGMA analysis_limit=0", // Disable automatic ANALYZE - "PRAGMA auto_vacuum=INCREMENTAL", // Enable incremental vacuum + "PRAGMA journal_mode=WAL", // Write-Ahead Logging + "PRAGMA analysis_limit=0", // Disable automatic ANALYZE fmt.Sprintf("PRAGMA soft_heap_limit=%d", sqliteSoftHeapLimitBytes), fmt.Sprintf("PRAGMA hard_heap_limit=%d", sqliteHardHeapLimitBytes), } diff --git a/internal/database/database_test.go b/internal/database/database_test.go index fa4bb48..ef7eec8 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -3,6 +3,7 @@ package database import ( "context" "database/sql" + "fmt" "net" "sync" "testing" @@ -16,6 +17,13 @@ import ( // memory"; the DSN change must leave temp_store below this so they spill to disk. const tempStoreMemory = 2 +// autoVacuumIncremental is the PRAGMA auto_vacuum value meaning "incremental". +const autoVacuumIncremental = 2 + +// vacuumTestRoutes is how many routes the vacuum test writes and then deletes, +// enough to leave many free pages in the file. +const vacuumTestRoutes = 2000 + // heldConnections is how many pooled connections the pragma test holds open at // once so each is a distinct SQLite connection that parsed the DSN. const heldConnections = 5 @@ -311,7 +319,8 @@ func TestIPv4RangeIntegration(t *testing.T) { // TestConnectionPoolPragmas holds several pooled connections open at once and // checks each one carries the per-connection settings from the DSN, plus the -// process-wide hard heap limit. +// process-wide hard heap limit, and sees the new file with auto_vacuum +// incremental. func TestConnectionPoolPragmas(t *testing.T) { cfg := &config.Config{StateDir: t.TempDir()} @@ -372,6 +381,72 @@ func TestConnectionPoolPragmas(t *testing.T) { if hardHeapLimit != sqliteHardHeapLimitBytes { t.Errorf("conn %d: hard_heap_limit = %d, want %d", i, hardHeapLimit, sqliteHardHeapLimitBytes) } + + var autoVacuum int + if err := c.QueryRowContext(ctx, "PRAGMA auto_vacuum").Scan(&autoVacuum); err != nil { + t.Fatalf("conn %d: failed to read auto_vacuum: %v", i, err) + } + if autoVacuum != autoVacuumIncremental { + t.Errorf("conn %d: auto_vacuum = %d, want %d (incremental)", i, autoVacuum, autoVacuumIncremental) + } + } +} + +// TestVacuumReturnsFreePages checks that after routes are deleted from a new +// database, Vacuum returns the pages they used. With auto_vacuum off (issue +// https://git.eeqj.de/sneak/routewatch/issues/43) the free pages stayed in the +// file and Vacuum left the count unchanged. +func TestVacuumReturnsFreePages(t *testing.T) { + cfg := &config.Config{StateDir: t.TempDir()} + + db, err := New(cfg, logger.New()) + if err != nil { + t.Fatalf("failed to create database: %v", err) + } + defer func() { _ = db.Close() }() + + ctx := context.Background() + ts := time.Now().UTC() + const asn = 64500 + + routes := make([]*LiveRoute, 0, vacuumTestRoutes) + deletions := make([]LiveRouteDeletion, 0, vacuumTestRoutes) + for i := range vacuumTestRoutes { + route := mkV4Route(t, fmt.Sprintf("10.%d.%d.0/24", i/256, i%256), asn, ts) + routes = append(routes, route) + deletions = append(deletions, LiveRouteDeletion{ + Prefix: route.Prefix, + OriginASN: asn, + PeerIP: route.PeerIP, + IPVersion: ipVersionV4, + }) + } + + if err := db.UpsertLiveRouteBatch(routes); err != nil { + t.Fatalf("failed to write routes: %v", err) + } + if err := db.DeleteLiveRouteBatch(deletions); err != nil { + t.Fatalf("failed to delete routes: %v", err) + } + + var before int + if err := db.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&before); err != nil { + t.Fatalf("failed to read freelist_count: %v", err) + } + if before == 0 { + t.Fatalf("no free pages after deleting %d routes", vacuumTestRoutes) + } + + if err := db.Vacuum(ctx); err != nil { + t.Fatalf("Vacuum failed: %v", err) + } + + var after int + if err := db.db.QueryRowContext(ctx, "PRAGMA freelist_count").Scan(&after); err != nil { + t.Fatalf("failed to read freelist_count: %v", err) + } + if after >= before { + t.Errorf("free pages after Vacuum = %d, want fewer than %d", after, before) } }