From 95428fbb8b66cb2136f12f8f5d124d9650432ce1 Mon Sep 17 00:00:00 2001 From: clawbot Date: Tue, 20 Oct 2026 08:50:14 +0000 Subject: [PATCH] Run SQLite in WAL mode with a busy timeout the driver reads (closes #123) The default database URL asked for WAL mode with `_journal_mode` and `_busy_timeout`, which the SQLite driver does not read, so the database ran in rollback-journal mode. The default now uses the driver's `_pragma=journal_mode(WAL)` and `_pragma=busy_timeout(5000)` parameters, in one constant that both the config default and the database fallback use. The database no longer sets its own busy timeout after opening, so the URL's value applies. A new test opens a file database with those parameters through db.New and checks the journal mode and busy timeout. README.md's database text now matches, and an in-memory test database drops the same ignored parameter. Model: opus-5-5 --- README.md | 61 +++++++++++++------------ internal/config/config.go | 8 +++- internal/db/db.go | 9 +--- internal/db/db_test.go | 75 +++++++++++++++++++++++++++++++ internal/ircserver/server_test.go | 2 +- 5 files changed, 116 insertions(+), 39 deletions(-) create mode 100644 internal/db/db_test.go diff --git a/README.md b/README.md index 3fe6cf4..54a0f23 100644 --- a/README.md +++ b/README.md @@ -2581,28 +2581,28 @@ and Viper also reads a YAML file `neoirc.yaml` from `/etc/neoirc/` or environment variable wins over the file. An environment variable set to the empty string counts as unset, so the file's value or the default applies. -| Variable | Type | Default | Description | -| ---------------------- | ------ | --------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `PORT` | int | `8080` | HTTP listen port | -| `DBURL` | string | `file:///var/lib/neoirc/state.db?_journal_mode=WAL` | SQLite connection string. For file-based: `file:///path/to/db.db?_journal_mode=WAL`. For in-memory (testing): `file::memory:?cache=shared`. The SQLite driver ignores `_journal_mode`, so this does not turn on WAL mode. | -| `DEBUG` | bool | `false` | Switch logging to debug level | -| `MESSAGE_MAX_AGE` | string | `720h` | Maximum age of messages as a Go duration string (e.g. `720h`, `24h`). Messages older than this are pruned. Default is 30 days. | -| `SESSION_IDLE_TIMEOUT` | string | `720h` | Idle timeout as a Go duration string (e.g. `720h`, `24h`). Clients with no authenticated HTTP API request or IRC listener command for this long are removed, and a session left with none is expired and its nick released (see [Data Lifecycle](#data-lifecycle)). Default is 30 days. | -| `QUEUE_MAX_AGE` | string | `720h` | Maximum age of client output queue entries as a Go duration string (e.g. `720h`, `24h`). Entries older than this are pruned. Default is 30 days. | -| `MAX_MESSAGE_SIZE` | int | `4096` | Maximum request body size in bytes for `POST /api/v1/session`, `POST /api/v1/login` and `POST /api/v1/messages`; a larger body gets `400` `invalid request body` | -| `MOTD` | string | a built-in banner | Message of the day, sent to clients as MOTD numerics and shown via `GET /api/v1/server` | -| `SERVER_NAME` | string | `""` | Server display name. If empty, the server calls itself `neoirc` in replies and in `GET /api/v1/server` returns it empty. | -| `FEDERATION_KEY` | string | `""` | Shared key for server federation linking (planned) | -| `SENTRY_DSN` | string | `""` | Sentry error tracking DSN (optional) | -| `METRICS_USERNAME` | string | `""` | Basic auth username for `/metrics` endpoint. If empty, metrics endpoint is disabled. | -| `METRICS_PASSWORD` | string | `""` | Basic auth password for `/metrics` endpoint | -| `NEOIRC_HASHCASH_BITS` | int | `20` | Required hashcash proof-of-work difficulty (leading zero bits in SHA-256) for session creation. Set to `0` to disable. | -| `NEOIRC_OPER_NAME` | string | `""` | Server operator (o-line) username. Both name and password must be set to enable OPER. | -| `NEOIRC_OPER_PASSWORD` | string | `""` | Server operator (o-line) password. Both name and password must be set to enable OPER. | -| `LOGIN_RATE_LIMIT` | float | `1` | Allowed login attempts per second per IP address. | -| `LOGIN_RATE_BURST` | int | `5` | Maximum burst of login attempts per IP before rate limiting kicks in. | -| `IRC_LISTEN_ADDR` | string | `:6667` | TCP address for the traditional IRC protocol listener. Only an empty value in `neoirc.yaml` disables it; an empty environment variable does not. | -| `MAINTENANCE_MODE` | bool | `false` | Maintenance mode flag (reserved) | +| Variable | Type | Default | Description | +| ---------------------- | ------ | -------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `PORT` | int | `8080` | HTTP listen port | +| `DBURL` | string | `file:///var/lib/neoirc/state.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)` | SQLite connection string. For file-based: `file:///path/to/db.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)`. For in-memory (testing): `file::memory:?cache=shared`. See [SQLite Considerations](#sqlite-considerations). | +| `DEBUG` | bool | `false` | Switch logging to debug level | +| `MESSAGE_MAX_AGE` | string | `720h` | Maximum age of messages as a Go duration string (e.g. `720h`, `24h`). Messages older than this are pruned. Default is 30 days. | +| `SESSION_IDLE_TIMEOUT` | string | `720h` | Idle timeout as a Go duration string (e.g. `720h`, `24h`). Clients with no authenticated HTTP API request or IRC listener command for this long are removed, and a session left with none is expired and its nick released (see [Data Lifecycle](#data-lifecycle)). Default is 30 days. | +| `QUEUE_MAX_AGE` | string | `720h` | Maximum age of client output queue entries as a Go duration string (e.g. `720h`, `24h`). Entries older than this are pruned. Default is 30 days. | +| `MAX_MESSAGE_SIZE` | int | `4096` | Maximum request body size in bytes for `POST /api/v1/session`, `POST /api/v1/login` and `POST /api/v1/messages`; a larger body gets `400` `invalid request body` | +| `MOTD` | string | a built-in banner | Message of the day, sent to clients as MOTD numerics and shown via `GET /api/v1/server` | +| `SERVER_NAME` | string | `""` | Server display name. If empty, the server calls itself `neoirc` in replies and in `GET /api/v1/server` returns it empty. | +| `FEDERATION_KEY` | string | `""` | Shared key for server federation linking (planned) | +| `SENTRY_DSN` | string | `""` | Sentry error tracking DSN (optional) | +| `METRICS_USERNAME` | string | `""` | Basic auth username for `/metrics` endpoint. If empty, metrics endpoint is disabled. | +| `METRICS_PASSWORD` | string | `""` | Basic auth password for `/metrics` endpoint | +| `NEOIRC_HASHCASH_BITS` | int | `20` | Required hashcash proof-of-work difficulty (leading zero bits in SHA-256) for session creation. Set to `0` to disable. | +| `NEOIRC_OPER_NAME` | string | `""` | Server operator (o-line) username. Both name and password must be set to enable OPER. | +| `NEOIRC_OPER_PASSWORD` | string | `""` | Server operator (o-line) password. Both name and password must be set to enable OPER. | +| `LOGIN_RATE_LIMIT` | float | `1` | Allowed login attempts per second per IP address. | +| `LOGIN_RATE_BURST` | int | `5` | Maximum burst of login attempts per IP before rate limiting kicks in. | +| `IRC_LISTEN_ADDR` | string | `:6667` | TCP address for the traditional IRC protocol listener. Only an empty value in `neoirc.yaml` disables it; an empty environment variable does not. | +| `MAINTENANCE_MODE` | bool | `false` | Maintenance mode flag (reserved) | ### Example `.env` file @@ -2611,7 +2611,7 @@ PORT=8080 SERVER_NAME=My NeoIRC Server MOTD=Welcome! Be excellent to each other. DEBUG=false -DBURL=file:///var/lib/neoirc/state.db?_journal_mode=WAL +DBURL=file:///var/lib/neoirc/state.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000) SESSION_IDLE_TIMEOUT=720h NEOIRC_HASHCASH_BITS=20 ``` @@ -2821,14 +2821,17 @@ seconds to accommodate long-poll connections. ### SQLite Considerations -- **Journal mode**: the default connection string carries `?_journal_mode=WAL`, - but the SQLite driver ignores that parameter, so the database runs in SQLite's - default rollback-journal mode, not WAL. +- **Journal mode**: the default connection string runs the database in WAL mode + with a 5000 ms busy timeout, through the SQLite driver's + `_pragma=journal_mode(WAL)` and `_pragma=busy_timeout(5000)` parameters. A + `DBURL` of your own should keep both. - **Single writer**: the server uses one database connection, so reads and writes take turns. For high-traffic servers, Postgres support is planned. -- **Backup**: The database is a single file. Back it up with - `sqlite3 /var/lib/neoirc/state.db ".backup backup.db"`, or copy the file while - the server is stopped. +- **Backup**: Back the database up with + `sqlite3 /var/lib/neoirc/state.db ".backup backup.db"`, or copy `state.db` + while the server is stopped, together with `state.db-wal` if one is left + beside it: in WAL mode that file holds recent writes not yet folded into + `state.db`. - **Location**: By default, `state.db` is created in `/var/lib/neoirc/`. Use the `DBURL` env var to place it elsewhere. diff --git a/internal/config/config.go b/internal/config/config.go index db57556..a3db54d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -21,6 +21,12 @@ const defaultMOTD = ` _ __ ___ ___ (_)_ __ ___ Welcome to NeoIRC — IRC semantics over HTTP. Type /help for available commands.` +// DefaultDBURL is the database URL used when DBURL is not set. Its +// _pragma parameters, which the SQLite driver runs on every connection +// it opens, set WAL mode and a 5000 ms busy timeout. +const DefaultDBURL = "file:///var/lib/neoirc/state.db" + + "?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)" + // Params defines the dependencies for creating a Config. type Params struct { fx.In @@ -71,7 +77,7 @@ func New( viper.SetDefault("DEBUG", "false") viper.SetDefault("MAINTENANCE_MODE", "false") viper.SetDefault("PORT", "8080") - viper.SetDefault("DBURL", "file:///var/lib/neoirc/state.db?_journal_mode=WAL") + viper.SetDefault("DBURL", DefaultDBURL) viper.SetDefault("SENTRY_DSN", "") viper.SetDefault("METRICS_USERNAME", "") viper.SetDefault("METRICS_PASSWORD", "") diff --git a/internal/db/db.go b/internal/db/db.go index 43fad60..e2dce3a 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -87,7 +87,7 @@ func (database *Database) GetDB() *sql.DB { func (database *Database) connect(ctx context.Context) error { dbURL := database.params.Config.DBURL if dbURL == "" { - dbURL = "file:///var/lib/neoirc/state.db?_journal_mode=WAL&_busy_timeout=5000" + dbURL = config.DefaultDBURL } database.log.Info( @@ -116,13 +116,6 @@ func (database *Database) connect(ctx context.Context) error { return fmt.Errorf("enable foreign keys: %w", err) } - _, err = database.conn.ExecContext( - ctx, "PRAGMA busy_timeout = 5000", - ) - if err != nil { - return fmt.Errorf("set busy timeout: %w", err) - } - return database.runMigrations(ctx) } diff --git a/internal/db/db_test.go b/internal/db/db_test.go new file mode 100644 index 0000000..c79fdfc --- /dev/null +++ b/internal/db/db_test.go @@ -0,0 +1,75 @@ +package db_test + +import ( + "path/filepath" + "strings" + "testing" + + "go.uber.org/fx/fxtest" + "sneak.berlin/go/neoirc/internal/config" + "sneak.berlin/go/neoirc/internal/db" + "sneak.berlin/go/neoirc/internal/globals" + "sneak.berlin/go/neoirc/internal/logger" +) + +const wantBusyTimeoutMS = 5000 + +// TestDefaultDBURLSetsWALAndBusyTimeout opens a file database +// with the default URL's parameters the way the server does, +// and checks the journal mode and busy timeout it ends up with. +func TestDefaultDBURLSetsWALAndBusyTimeout(t *testing.T) { + t.Parallel() + + _, params, _ := strings.Cut(config.DefaultDBURL, "?") + dbURL := "file://" + filepath.Join(t.TempDir(), "state.db") + + "?" + params + + lifecycle := fxtest.NewLifecycle(t) + + log, err := logger.New(lifecycle, logger.Params{ + Globals: &globals.Globals{}, + }) + if err != nil { + t.Fatal(err) + } + + database, err := db.New(lifecycle, db.Params{ + Logger: log, + Config: &config.Config{DBURL: dbURL}, + }) + if err != nil { + t.Fatal(err) + } + + lifecycle.RequireStart() + t.Cleanup(lifecycle.RequireStop) + + var journalMode string + + err = database.GetDB().QueryRowContext( + t.Context(), "PRAGMA journal_mode", + ).Scan(&journalMode) + if err != nil { + t.Fatal(err) + } + + if journalMode != "wal" { + t.Errorf("journal_mode = %q, want %q", journalMode, "wal") + } + + var busyTimeout int + + err = database.GetDB().QueryRowContext( + t.Context(), "PRAGMA busy_timeout", + ).Scan(&busyTimeout) + if err != nil { + t.Fatal(err) + } + + if busyTimeout != wantBusyTimeoutMS { + t.Errorf( + "busy_timeout = %d, want %d", + busyTimeout, wantBusyTimeoutMS, + ) + } +} diff --git a/internal/ircserver/server_test.go b/internal/ircserver/server_test.go index 70e922b..35ca0a6 100644 --- a/internal/ircserver/server_test.go +++ b/internal/ircserver/server_test.go @@ -73,7 +73,7 @@ func newTestEnvWithConfig( // A random name, so that no other test environment, not // even an earlier run of the same test, can share it. dsn := fmt.Sprintf( - "file:%s?mode=memory&cache=shared&_journal_mode=WAL", + "file:%s?mode=memory&cache=shared", rand.Text(), )