Add production hardening: health check, streamer panic fix, db maintenance
- Add health check endpoint at /.well-known/healthcheck.json that verifies database and RIS Live connectivity, returns 200/503 - Fix panic in streamer when encountering unknown RIS message types by logging a warning and continuing instead of crashing - Add DBMaintainer for periodic database maintenance: - VACUUM every 6 hours to reclaim space - ANALYZE every hour to update query statistics - Graceful shutdown support - Add Vacuum() and Analyze() methods to database interface
This commit is contained in:
142
internal/routewatch/dbmaintainer.go
Normal file
142
internal/routewatch/dbmaintainer.go
Normal file
@@ -0,0 +1,142 @@
|
||||
// Package routewatch contains the database maintainer for background maintenance tasks.
|
||||
package routewatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.eeqj.de/sneak/routewatch/internal/database"
|
||||
)
|
||||
|
||||
// Database maintenance configuration constants.
|
||||
const (
|
||||
// vacuumInterval is how often to run VACUUM.
|
||||
vacuumInterval = 6 * time.Hour
|
||||
|
||||
// analyzeInterval is how often to run ANALYZE.
|
||||
analyzeInterval = 1 * time.Hour
|
||||
|
||||
// maintenanceTimeout is the max time for a maintenance operation.
|
||||
maintenanceTimeout = 5 * time.Minute
|
||||
)
|
||||
|
||||
// DBMaintainer handles background database maintenance tasks.
|
||||
type DBMaintainer struct {
|
||||
db database.Store
|
||||
logger *slog.Logger
|
||||
stopCh chan struct{}
|
||||
wg sync.WaitGroup
|
||||
|
||||
// Stats tracking
|
||||
statsMu sync.Mutex
|
||||
lastVacuum time.Time
|
||||
lastAnalyze time.Time
|
||||
vacuumCount int
|
||||
analyzeCount int
|
||||
lastVacuumError error
|
||||
lastAnalyzeError error
|
||||
}
|
||||
|
||||
// NewDBMaintainer creates a new database maintainer.
|
||||
func NewDBMaintainer(db database.Store, logger *slog.Logger) *DBMaintainer {
|
||||
return &DBMaintainer{
|
||||
db: db,
|
||||
logger: logger.With("component", "db_maintainer"),
|
||||
stopCh: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// Start begins the background maintenance goroutine.
|
||||
func (m *DBMaintainer) Start() {
|
||||
m.wg.Add(1)
|
||||
go m.run()
|
||||
m.logger.Info("Database maintainer started",
|
||||
"vacuum_interval", vacuumInterval,
|
||||
"analyze_interval", analyzeInterval,
|
||||
)
|
||||
}
|
||||
|
||||
// Stop gracefully shuts down the maintainer.
|
||||
func (m *DBMaintainer) Stop() {
|
||||
close(m.stopCh)
|
||||
m.wg.Wait()
|
||||
m.logger.Info("Database maintainer stopped")
|
||||
}
|
||||
|
||||
// run is the main background loop.
|
||||
func (m *DBMaintainer) run() {
|
||||
defer m.wg.Done()
|
||||
|
||||
// Use different timers for each task
|
||||
vacuumTimer := time.NewTimer(vacuumInterval)
|
||||
analyzeTimer := time.NewTimer(analyzeInterval)
|
||||
defer vacuumTimer.Stop()
|
||||
defer analyzeTimer.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-m.stopCh:
|
||||
return
|
||||
|
||||
case <-vacuumTimer.C:
|
||||
m.runVacuum()
|
||||
vacuumTimer.Reset(vacuumInterval)
|
||||
|
||||
case <-analyzeTimer.C:
|
||||
m.runAnalyze()
|
||||
analyzeTimer.Reset(analyzeInterval)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// runVacuum performs a VACUUM operation on the database.
|
||||
func (m *DBMaintainer) runVacuum() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), maintenanceTimeout)
|
||||
defer cancel()
|
||||
|
||||
m.logger.Info("Starting database VACUUM")
|
||||
startTime := time.Now()
|
||||
|
||||
err := m.db.Vacuum(ctx)
|
||||
|
||||
m.statsMu.Lock()
|
||||
m.lastVacuum = time.Now()
|
||||
m.lastVacuumError = err
|
||||
if err == nil {
|
||||
m.vacuumCount++
|
||||
}
|
||||
m.statsMu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
m.logger.Error("VACUUM failed", "error", err, "duration", time.Since(startTime))
|
||||
} else {
|
||||
m.logger.Info("VACUUM completed", "duration", time.Since(startTime))
|
||||
}
|
||||
}
|
||||
|
||||
// runAnalyze performs an ANALYZE operation on the database.
|
||||
func (m *DBMaintainer) runAnalyze() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), maintenanceTimeout)
|
||||
defer cancel()
|
||||
|
||||
m.logger.Info("Starting database ANALYZE")
|
||||
startTime := time.Now()
|
||||
|
||||
err := m.db.Analyze(ctx)
|
||||
|
||||
m.statsMu.Lock()
|
||||
m.lastAnalyze = time.Now()
|
||||
m.lastAnalyzeError = err
|
||||
if err == nil {
|
||||
m.analyzeCount++
|
||||
}
|
||||
m.statsMu.Unlock()
|
||||
|
||||
if err != nil {
|
||||
m.logger.Error("ANALYZE failed", "error", err, "duration", time.Since(startTime))
|
||||
} else {
|
||||
m.logger.Info("ANALYZE completed", "duration", time.Since(startTime))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user