feat: implement per-webhook event databases
All checks were successful
check / check (push) Successful in 1m50s
All checks were successful
check / check (push) Successful in 1m50s
Split data storage into main application DB (config only) and
per-webhook event databases (one SQLite file per webhook).
Architecture changes:
- New WebhookDBManager component manages per-webhook DB lifecycle
(create, open, cache, delete) with lazy connection pooling via sync.Map
- Main DB (DBURL) stores only config: Users, Webhooks, Entrypoints,
Targets, APIKeys
- Per-webhook DBs (DATA_DIR) store Events, Deliveries, DeliveryResults
in files named events-{webhook_uuid}.db
- New DATA_DIR env var (default: ./data dev, /data/events prod)
Behavioral changes:
- Webhook creation creates per-webhook DB file
- Webhook deletion hard-deletes per-webhook DB file (config soft-deleted)
- Event ingestion writes to per-webhook DB, not main DB
- Delivery engine polls all per-webhook DBs for pending deliveries
- Database target type marks delivery as immediately successful (events
are already in the dedicated per-webhook DB)
- Event log UI reads from per-webhook DBs with targets from main DB
- Existing webhooks without DB files get them created lazily
Removed:
- ArchivedEvent model (was a half-measure, replaced by per-webhook DBs)
- Event/Delivery/DeliveryResult removed from main DB migrations
Added:
- Comprehensive tests for WebhookDBManager (create, delete, lazy
creation, delivery workflow, multiple webhooks, close all)
- Dockerfile creates /data/events directory
README updates:
- Per-webhook event databases documented as implemented (was Phase 2)
- DATA_DIR added to configuration table
- Docker instructions updated with data volume mount
- Data model diagram updated
- TODO updated (database separation moved to completed)
Closes #15
This commit is contained in:
@@ -40,7 +40,13 @@ func (h *Handlers) HandleSourceList() http.HandlerFunc {
|
||||
items[i].Webhook = webhooks[i]
|
||||
h.db.DB().Model(&database.Entrypoint{}).Where("webhook_id = ?", webhooks[i].ID).Count(&items[i].EntrypointCount)
|
||||
h.db.DB().Model(&database.Target{}).Where("webhook_id = ?", webhooks[i].ID).Count(&items[i].TargetCount)
|
||||
h.db.DB().Model(&database.Event{}).Where("webhook_id = ?", webhooks[i].ID).Count(&items[i].EventCount)
|
||||
|
||||
// Event count comes from per-webhook DB
|
||||
if h.dbMgr.DBExists(webhooks[i].ID) {
|
||||
if webhookDB, err := h.dbMgr.GetDB(webhooks[i].ID); err == nil {
|
||||
webhookDB.Model(&database.Event{}).Count(&items[i].EventCount)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
data := map[string]interface{}{
|
||||
@@ -136,6 +142,15 @@ func (h *Handlers) HandleSourceCreateSubmit() http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
// Create per-webhook event database
|
||||
if err := h.dbMgr.CreateDB(webhook.ID); err != nil {
|
||||
h.log.Error("failed to create webhook event database",
|
||||
"webhook_id", webhook.ID,
|
||||
"error", err,
|
||||
)
|
||||
// Non-fatal: the DB will be created lazily on first event
|
||||
}
|
||||
|
||||
h.log.Info("webhook created",
|
||||
"webhook_id", webhook.ID,
|
||||
"name", name,
|
||||
@@ -169,9 +184,13 @@ func (h *Handlers) HandleSourceDetail() http.HandlerFunc {
|
||||
var targets []database.Target
|
||||
h.db.DB().Where("webhook_id = ?", webhook.ID).Find(&targets)
|
||||
|
||||
// Recent events with delivery info
|
||||
// Recent events from per-webhook database
|
||||
var events []database.Event
|
||||
h.db.DB().Where("webhook_id = ?", webhook.ID).Order("created_at DESC").Limit(20).Find(&events)
|
||||
if h.dbMgr.DBExists(webhook.ID) {
|
||||
if webhookDB, err := h.dbMgr.GetDB(webhook.ID); err == nil {
|
||||
webhookDB.Where("webhook_id = ?", webhook.ID).Order("created_at DESC").Limit(20).Find(&events)
|
||||
}
|
||||
}
|
||||
|
||||
// Build host URL for display
|
||||
host := r.Host
|
||||
@@ -271,7 +290,9 @@ func (h *Handlers) HandleSourceEditSubmit() http.HandlerFunc {
|
||||
}
|
||||
}
|
||||
|
||||
// HandleSourceDelete handles webhook deletion (soft delete).
|
||||
// HandleSourceDelete handles webhook deletion.
|
||||
// Configuration data is soft-deleted in the main DB.
|
||||
// The per-webhook event database file is hard-deleted (permanently removed).
|
||||
func (h *Handlers) HandleSourceDelete() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
userID, ok := h.getUserID(r)
|
||||
@@ -288,6 +309,7 @@ func (h *Handlers) HandleSourceDelete() http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
// Soft-delete configuration in the main application database
|
||||
tx := h.db.DB().Begin()
|
||||
if tx.Error != nil {
|
||||
h.log.Error("failed to begin transaction", "error", tx.Error)
|
||||
@@ -295,28 +317,7 @@ func (h *Handlers) HandleSourceDelete() http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
// Soft-delete child records in dependency order (deepest first).
|
||||
|
||||
// Collect event IDs for this webhook
|
||||
var eventIDs []string
|
||||
tx.Model(&database.Event{}).Where("webhook_id = ?", webhook.ID).Pluck("id", &eventIDs)
|
||||
|
||||
if len(eventIDs) > 0 {
|
||||
// Collect delivery IDs for these events
|
||||
var deliveryIDs []string
|
||||
tx.Model(&database.Delivery{}).Where("event_id IN ?", eventIDs).Pluck("id", &deliveryIDs)
|
||||
|
||||
if len(deliveryIDs) > 0 {
|
||||
// Soft-delete delivery results
|
||||
tx.Where("delivery_id IN ?", deliveryIDs).Delete(&database.DeliveryResult{})
|
||||
}
|
||||
|
||||
// Soft-delete deliveries
|
||||
tx.Where("event_id IN ?", eventIDs).Delete(&database.Delivery{})
|
||||
}
|
||||
|
||||
// Soft-delete events, entrypoints, targets, and the webhook itself
|
||||
tx.Where("webhook_id = ?", webhook.ID).Delete(&database.Event{})
|
||||
// Soft-delete entrypoints and targets (config tier)
|
||||
tx.Where("webhook_id = ?", webhook.ID).Delete(&database.Entrypoint{})
|
||||
tx.Where("webhook_id = ?", webhook.ID).Delete(&database.Target{})
|
||||
tx.Delete(&webhook)
|
||||
@@ -327,12 +328,23 @@ func (h *Handlers) HandleSourceDelete() http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
// Hard-delete the per-webhook event database file
|
||||
if err := h.dbMgr.DeleteDB(webhook.ID); err != nil {
|
||||
h.log.Error("failed to delete webhook event database",
|
||||
"webhook_id", webhook.ID,
|
||||
"error", err,
|
||||
)
|
||||
// Non-fatal: file may not exist if no events were ever received
|
||||
}
|
||||
|
||||
h.log.Info("webhook deleted", "webhook_id", webhook.ID, "user_id", userID)
|
||||
http.Redirect(w, r, "/sources", http.StatusSeeOther)
|
||||
}
|
||||
}
|
||||
|
||||
// HandleSourceLogs shows the request/response logs for a webhook.
|
||||
// Events and deliveries are read from the per-webhook database.
|
||||
// Target information is loaded from the main application database.
|
||||
func (h *Handlers) HandleSourceLogs() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
userID, ok := h.getUserID(r)
|
||||
@@ -349,6 +361,14 @@ func (h *Handlers) HandleSourceLogs() http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
// Load targets from main DB for display
|
||||
var targets []database.Target
|
||||
h.db.DB().Where("webhook_id = ?", webhook.ID).Find(&targets)
|
||||
targetMap := make(map[string]database.Target, len(targets))
|
||||
for _, t := range targets {
|
||||
targetMap[t.ID] = t
|
||||
}
|
||||
|
||||
// Pagination
|
||||
page := 1
|
||||
if p := r.URL.Query().Get("page"); p != "" {
|
||||
@@ -359,25 +379,48 @@ func (h *Handlers) HandleSourceLogs() http.HandlerFunc {
|
||||
perPage := 25
|
||||
offset := (page - 1) * perPage
|
||||
|
||||
var totalEvents int64
|
||||
h.db.DB().Model(&database.Event{}).Where("webhook_id = ?", webhook.ID).Count(&totalEvents)
|
||||
|
||||
var events []database.Event
|
||||
h.db.DB().Where("webhook_id = ?", webhook.ID).
|
||||
Order("created_at DESC").
|
||||
Offset(offset).
|
||||
Limit(perPage).
|
||||
Find(&events)
|
||||
|
||||
// Load deliveries for each event
|
||||
// EventWithDeliveries holds an event with its associated deliveries
|
||||
type EventWithDeliveries struct {
|
||||
database.Event
|
||||
Deliveries []database.Delivery
|
||||
}
|
||||
eventsWithDeliveries := make([]EventWithDeliveries, len(events))
|
||||
for i := range events {
|
||||
eventsWithDeliveries[i].Event = events[i]
|
||||
h.db.DB().Where("event_id = ?", events[i].ID).Preload("Target").Find(&eventsWithDeliveries[i].Deliveries)
|
||||
|
||||
var totalEvents int64
|
||||
var eventsWithDeliveries []EventWithDeliveries
|
||||
|
||||
// Read events and deliveries from per-webhook database
|
||||
if h.dbMgr.DBExists(webhook.ID) {
|
||||
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
|
||||
if err != nil {
|
||||
h.log.Error("failed to get webhook database",
|
||||
"webhook_id", webhook.ID,
|
||||
"error", err,
|
||||
)
|
||||
http.Error(w, "Internal server error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
webhookDB.Model(&database.Event{}).Where("webhook_id = ?", webhook.ID).Count(&totalEvents)
|
||||
|
||||
var events []database.Event
|
||||
webhookDB.Where("webhook_id = ?", webhook.ID).
|
||||
Order("created_at DESC").
|
||||
Offset(offset).
|
||||
Limit(perPage).
|
||||
Find(&events)
|
||||
|
||||
eventsWithDeliveries = make([]EventWithDeliveries, len(events))
|
||||
for i := range events {
|
||||
eventsWithDeliveries[i].Event = events[i]
|
||||
// Load deliveries from per-webhook DB (without Target preload)
|
||||
webhookDB.Where("event_id = ?", events[i].ID).Find(&eventsWithDeliveries[i].Deliveries)
|
||||
// Manually assign targets from main DB
|
||||
for j := range eventsWithDeliveries[i].Deliveries {
|
||||
if target, ok := targetMap[eventsWithDeliveries[i].Deliveries[j].TargetID]; ok {
|
||||
eventsWithDeliveries[i].Deliveries[j].Target = target
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
totalPages := int(totalEvents) / perPage
|
||||
|
||||
Reference in New Issue
Block a user