Implement queue pruning and message rotation (closes #40) #67

Merged
sneak merged 6 commits from feat/queue-pruning into main 2026-03-10 15:37:34 +01:00
4 changed files with 136 additions and 18 deletions

View File

@@ -249,8 +249,8 @@ Key properties:
- **Ordered**: Queue entries have monotonically increasing IDs. Messages are - **Ordered**: Queue entries have monotonically increasing IDs. Messages are
always delivered in order within a client's queue. always delivered in order within a client's queue.
- **No delivery/read receipts** for channel messages. DM receipts are planned. - **No delivery/read receipts** for channel messages. DM receipts are planned.
- **Queue depth**: Server-configurable via `QUEUE_MAX_AGE`. Default is 48 - **Client output queue depth**: Server-configurable via `QUEUE_MAX_AGE`.
hours. Entries older than this are pruned. Default is 30 days. Entries older than this are pruned.
### Long-Polling ### Long-Polling
@@ -1788,14 +1788,14 @@ skew issues) and simpler than UUIDs (integer comparison vs. string comparison).
### Data Lifecycle ### Data Lifecycle
- **Messages**: Stored indefinitely in the current implementation. Rotation - **Messages**: Pruned automatically when older than `MESSAGE_MAX_AGE`
per `MAX_HISTORY` is planned. (default 30 days).
- **Queue entries**: Stored until pruned. Pruning by `QUEUE_MAX_AGE` is - **Client output queue entries**: Pruned automatically when older than
planned. `QUEUE_MAX_AGE` (default 30 days).
- **Channels**: Deleted when the last member leaves (ephemeral). - **Channels**: Deleted when the last member leaves (ephemeral).
- **Users/sessions**: Deleted on `QUIT` or `POST /api/v1/logout`. Idle - **Users/sessions**: Deleted on `QUIT` or `POST /api/v1/logout`. Idle
sessions are automatically expired after `SESSION_IDLE_TIMEOUT` (default sessions are automatically expired after `SESSION_IDLE_TIMEOUT` (default
24h) — the server runs a background cleanup loop that parts idle users 30 days) — the server runs a background cleanup loop that parts idle users
from all channels, broadcasts QUIT, and releases their nicks. from all channels, broadcasts QUIT, and releases their nicks.
--- ---
@@ -1812,9 +1812,9 @@ directory is also loaded automatically via
| `PORT` | int | `8080` | HTTP listen port | | `PORT` | int | `8080` | HTTP listen port |
| `DBURL` | string | `file:///var/lib/neoirc/state.db?_journal_mode=WAL` | SQLite connection string. For file-based: `file:///path/to/db.db?_journal_mode=WAL`. For in-memory (testing): `file::memory:?cache=shared`. | | `DBURL` | string | `file:///var/lib/neoirc/state.db?_journal_mode=WAL` | SQLite connection string. For file-based: `file:///path/to/db.db?_journal_mode=WAL`. For in-memory (testing): `file::memory:?cache=shared`. |
| `DEBUG` | bool | `false` | Enable debug logging (verbose request/response logging) | | `DEBUG` | bool | `false` | Enable debug logging (verbose request/response logging) |
| `MAX_HISTORY` | int | `10000` | Maximum messages retained per channel before rotation (planned) | | `MESSAGE_MAX_AGE` | string | `720h` | Maximum age of messages as a Go duration string (e.g. `720h`, `24h`). Messages older than this are pruned. Default is 30 days. |
| `SESSION_IDLE_TIMEOUT` | string | `24h` | Session idle timeout as a Go duration string (e.g. `24h`, `30m`). Sessions with no activity for this long are expired and the nick is released. | | `SESSION_IDLE_TIMEOUT` | string | `720h` | Session idle timeout as a Go duration string (e.g. `720h`, `24h`). Sessions with no activity for this long are expired and the nick is released. Default is 30 days. |
| `QUEUE_MAX_AGE` | int | `172800` | Maximum age of client queue entries in seconds (48h). Entries older than this are pruned (planned). | | `QUEUE_MAX_AGE` | string | `720h` | Maximum age of client output queue entries as a Go duration string (e.g. `720h`, `24h`). Entries older than this are pruned. Default is 30 days. |
| `MAX_MESSAGE_SIZE` | int | `4096` | Maximum message body size in bytes (planned enforcement) | | `MAX_MESSAGE_SIZE` | int | `4096` | Maximum message body size in bytes (planned enforcement) |
| `LONG_POLL_TIMEOUT`| int | `15` | Default long-poll timeout in seconds (client can override via query param, server caps at 30) | | `LONG_POLL_TIMEOUT`| int | `15` | Default long-poll timeout in seconds (client can override via query param, server caps at 30) |
| `MOTD` | string | `""` | Message of the day, shown to clients via `GET /api/v1/server` | | `MOTD` | string | `""` | Message of the day, shown to clients via `GET /api/v1/server` |
@@ -1833,7 +1833,7 @@ SERVER_NAME=My NeoIRC Server
MOTD=Welcome! Be excellent to each other. MOTD=Welcome! Be excellent to each other.
DEBUG=false DEBUG=false
DBURL=file:///var/lib/neoirc/state.db?_journal_mode=WAL DBURL=file:///var/lib/neoirc/state.db?_journal_mode=WAL
SESSION_IDLE_TIMEOUT=24h SESSION_IDLE_TIMEOUT=720h
``` ```
--- ---
@@ -2228,8 +2228,8 @@ GET /api/v1/challenge
### Post-MVP (Planned) ### Post-MVP (Planned)
- [ ] **Hashcash proof-of-work** for session creation (abuse prevention) - [ ] **Hashcash proof-of-work** for session creation (abuse prevention)
- [ ] **Queue pruning** — delete old queue entries per `QUEUE_MAX_AGE` - [x] **Client output queue pruning** — delete old client output queue entries per `QUEUE_MAX_AGE`
- [ ] **Message rotation** — enforce `MAX_HISTORY` per channel - [x] **Message rotation** — prune messages older than `MESSAGE_MAX_AGE`
- [ ] **Channel modes** — enforce `+i`, `+m`, `+s`, `+t`, `+n` - [ ] **Channel modes** — enforce `+i`, `+m`, `+s`, `+t`, `+n`
- [ ] **User channel modes** — `+o` (operator), `+v` (voice) - [ ] **User channel modes** — `+o` (operator), `+v` (voice)
- [x] **MODE command** — query channel and user modes (set not yet implemented) - [x] **MODE command** — query channel and user modes (set not yet implemented)

View File

@@ -38,8 +38,9 @@ type Config struct {
MetricsUsername string MetricsUsername string
Port int Port int
SentryDSN string SentryDSN string
MaxHistory int MessageMaxAge string
MaxMessageSize int MaxMessageSize int
QueueMaxAge string
MOTD string MOTD string
ServerName string ServerName string
FederationKey string FederationKey string
@@ -68,12 +69,13 @@ func New(
viper.SetDefault("SENTRY_DSN", "") viper.SetDefault("SENTRY_DSN", "")
viper.SetDefault("METRICS_USERNAME", "") viper.SetDefault("METRICS_USERNAME", "")
viper.SetDefault("METRICS_PASSWORD", "") viper.SetDefault("METRICS_PASSWORD", "")
viper.SetDefault("MAX_HISTORY", "10000") viper.SetDefault("MESSAGE_MAX_AGE", "720h")
viper.SetDefault("MAX_MESSAGE_SIZE", "4096") viper.SetDefault("MAX_MESSAGE_SIZE", "4096")
viper.SetDefault("QUEUE_MAX_AGE", "720h")
viper.SetDefault("MOTD", defaultMOTD) viper.SetDefault("MOTD", defaultMOTD)
viper.SetDefault("SERVER_NAME", "") viper.SetDefault("SERVER_NAME", "")
viper.SetDefault("FEDERATION_KEY", "") viper.SetDefault("FEDERATION_KEY", "")
viper.SetDefault("SESSION_IDLE_TIMEOUT", "24h") viper.SetDefault("SESSION_IDLE_TIMEOUT", "720h")
err := viper.ReadInConfig() err := viper.ReadInConfig()
if err != nil { if err != nil {
@@ -92,8 +94,9 @@ func New(
MaintenanceMode: viper.GetBool("MAINTENANCE_MODE"), MaintenanceMode: viper.GetBool("MAINTENANCE_MODE"),
MetricsUsername: viper.GetString("METRICS_USERNAME"), MetricsUsername: viper.GetString("METRICS_USERNAME"),
MetricsPassword: viper.GetString("METRICS_PASSWORD"), MetricsPassword: viper.GetString("METRICS_PASSWORD"),
MaxHistory: viper.GetInt("MAX_HISTORY"), MessageMaxAge: viper.GetString("MESSAGE_MAX_AGE"),
MaxMessageSize: viper.GetInt("MAX_MESSAGE_SIZE"), MaxMessageSize: viper.GetInt("MAX_MESSAGE_SIZE"),
QueueMaxAge: viper.GetString("QUEUE_MAX_AGE"),
MOTD: viper.GetString("MOTD"), MOTD: viper.GetString("MOTD"),
ServerName: viper.GetString("SERVER_NAME"), ServerName: viper.GetString("SERVER_NAME"),
FederationKey: viper.GetString("FEDERATION_KEY"), FederationKey: viper.GetString("FEDERATION_KEY"),

View File

@@ -1109,3 +1109,45 @@ func (database *Database) GetSessionCreatedAt(
return createdAt, nil return createdAt, nil
} }
// PruneOldQueueEntries deletes client output queue entries
// older than cutoff and returns the number of rows removed.
func (database *Database) PruneOldQueueEntries(
ctx context.Context,
cutoff time.Time,
) (int64, error) {
res, err := database.conn.ExecContext(ctx,
"DELETE FROM client_queues WHERE created_at < ?",
cutoff,
)
if err != nil {
return 0, fmt.Errorf(
"prune old client output queue entries: %w", err,
)
}
deleted, _ := res.RowsAffected()
return deleted, nil
}
// PruneOldMessages deletes messages older than cutoff and
// returns the number of rows removed.
func (database *Database) PruneOldMessages(
ctx context.Context,
cutoff time.Time,
) (int64, error) {
res, err := database.conn.ExecContext(ctx,
"DELETE FROM messages WHERE created_at < ?",
cutoff,
)
if err != nil {
return 0, fmt.Errorf(
"prune old messages: %w", err,
)
}
deleted, _ := res.RowsAffected()
return deleted, nil
}

View File

@@ -31,7 +31,7 @@ type Params struct {
Healthcheck *healthcheck.Healthcheck Healthcheck *healthcheck.Healthcheck
} }
const defaultIdleTimeout = 24 * time.Hour const defaultIdleTimeout = 30 * 24 * time.Hour
// Handlers manages HTTP request handling. // Handlers manages HTTP request handling.
type Handlers struct { type Handlers struct {
@@ -200,4 +200,77 @@ func (hdlr *Handlers) runCleanup(
"deleted", deleted, "deleted", deleted,
) )
} }
hdlr.pruneQueuesAndMessages(ctx)
}
// parseDurationConfig parses a Go duration string,
// returning zero on empty input and logging on error.
func (hdlr *Handlers) parseDurationConfig(
name, raw string,
) time.Duration {
if raw == "" {
return 0
}
dur, err := time.ParseDuration(raw)
if err != nil {
hdlr.log.Error(
"invalid duration config, skipping",
"name", name, "value", raw, "error", err,
)
return 0
}
return dur
}
// pruneQueuesAndMessages removes old client output queue
// entries per QUEUE_MAX_AGE and old messages per
// MESSAGE_MAX_AGE.
func (hdlr *Handlers) pruneQueuesAndMessages(
ctx context.Context,
) {
queueMaxAge := hdlr.parseDurationConfig(
"QUEUE_MAX_AGE",
hdlr.params.Config.QueueMaxAge,
)
if queueMaxAge > 0 {
queueCutoff := time.Now().Add(-queueMaxAge)
pruned, err := hdlr.params.Database.
PruneOldQueueEntries(ctx, queueCutoff)
if err != nil {
hdlr.log.Error(
"client output queue pruning failed", "error", err,
)
} else if pruned > 0 {
hdlr.log.Info(
"pruned old client output queue entries",
"deleted", pruned,
)
}
}
messageMaxAge := hdlr.parseDurationConfig(
"MESSAGE_MAX_AGE",
hdlr.params.Config.MessageMaxAge,
)
if messageMaxAge > 0 {
msgCutoff := time.Now().Add(-messageMaxAge)
pruned, err := hdlr.params.Database.
PruneOldMessages(ctx, msgCutoff)
if err != nil {
hdlr.log.Error(
"message pruning failed", "error", err,
)
} else if pruned > 0 {
hdlr.log.Info(
"pruned old messages",
"deleted", pruned,
)
}
}
} }