feat: implement per-webhook event databases
All checks were successful
check / check (push) Successful in 1m50s

Split data storage into main application DB (config only) and
per-webhook event databases (one SQLite file per webhook).

Architecture changes:
- New WebhookDBManager component manages per-webhook DB lifecycle
  (create, open, cache, delete) with lazy connection pooling via sync.Map
- Main DB (DBURL) stores only config: Users, Webhooks, Entrypoints,
  Targets, APIKeys
- Per-webhook DBs (DATA_DIR) store Events, Deliveries, DeliveryResults
  in files named events-{webhook_uuid}.db
- New DATA_DIR env var (default: ./data dev, /data/events prod)

Behavioral changes:
- Webhook creation creates per-webhook DB file
- Webhook deletion hard-deletes per-webhook DB file (config soft-deleted)
- Event ingestion writes to per-webhook DB, not main DB
- Delivery engine polls all per-webhook DBs for pending deliveries
- Database target type marks delivery as immediately successful (events
  are already in the dedicated per-webhook DB)
- Event log UI reads from per-webhook DBs with targets from main DB
- Existing webhooks without DB files get them created lazily

Removed:
- ArchivedEvent model (was a half-measure, replaced by per-webhook DBs)
- Event/Delivery/DeliveryResult removed from main DB migrations

Added:
- Comprehensive tests for WebhookDBManager (create, delete, lazy
  creation, delivery workflow, multiple webhooks, close all)
- Dockerfile creates /data/events directory

README updates:
- Per-webhook event databases documented as implemented (was Phase 2)
- DATA_DIR added to configuration table
- Docker instructions updated with data volume mount
- Data model diagram updated
- TODO updated (database separation moved to completed)

Closes #15
This commit is contained in:
clawbot
2026-03-01 17:06:43 -08:00
parent 6c393ccb78
commit 43c22a9e9a
13 changed files with 814 additions and 198 deletions

View File

@@ -40,7 +40,13 @@ func (h *Handlers) HandleSourceList() http.HandlerFunc {
items[i].Webhook = webhooks[i]
h.db.DB().Model(&database.Entrypoint{}).Where("webhook_id = ?", webhooks[i].ID).Count(&items[i].EntrypointCount)
h.db.DB().Model(&database.Target{}).Where("webhook_id = ?", webhooks[i].ID).Count(&items[i].TargetCount)
h.db.DB().Model(&database.Event{}).Where("webhook_id = ?", webhooks[i].ID).Count(&items[i].EventCount)
// Event count comes from per-webhook DB
if h.dbMgr.DBExists(webhooks[i].ID) {
if webhookDB, err := h.dbMgr.GetDB(webhooks[i].ID); err == nil {
webhookDB.Model(&database.Event{}).Count(&items[i].EventCount)
}
}
}
data := map[string]interface{}{
@@ -136,6 +142,15 @@ func (h *Handlers) HandleSourceCreateSubmit() http.HandlerFunc {
return
}
// Create per-webhook event database
if err := h.dbMgr.CreateDB(webhook.ID); err != nil {
h.log.Error("failed to create webhook event database",
"webhook_id", webhook.ID,
"error", err,
)
// Non-fatal: the DB will be created lazily on first event
}
h.log.Info("webhook created",
"webhook_id", webhook.ID,
"name", name,
@@ -169,9 +184,13 @@ func (h *Handlers) HandleSourceDetail() http.HandlerFunc {
var targets []database.Target
h.db.DB().Where("webhook_id = ?", webhook.ID).Find(&targets)
// Recent events with delivery info
// Recent events from per-webhook database
var events []database.Event
h.db.DB().Where("webhook_id = ?", webhook.ID).Order("created_at DESC").Limit(20).Find(&events)
if h.dbMgr.DBExists(webhook.ID) {
if webhookDB, err := h.dbMgr.GetDB(webhook.ID); err == nil {
webhookDB.Where("webhook_id = ?", webhook.ID).Order("created_at DESC").Limit(20).Find(&events)
}
}
// Build host URL for display
host := r.Host
@@ -271,7 +290,9 @@ func (h *Handlers) HandleSourceEditSubmit() http.HandlerFunc {
}
}
// HandleSourceDelete handles webhook deletion (soft delete).
// HandleSourceDelete handles webhook deletion.
// Configuration data is soft-deleted in the main DB.
// The per-webhook event database file is hard-deleted (permanently removed).
func (h *Handlers) HandleSourceDelete() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
userID, ok := h.getUserID(r)
@@ -288,6 +309,7 @@ func (h *Handlers) HandleSourceDelete() http.HandlerFunc {
return
}
// Soft-delete configuration in the main application database
tx := h.db.DB().Begin()
if tx.Error != nil {
h.log.Error("failed to begin transaction", "error", tx.Error)
@@ -295,28 +317,7 @@ func (h *Handlers) HandleSourceDelete() http.HandlerFunc {
return
}
// Soft-delete child records in dependency order (deepest first).
// Collect event IDs for this webhook
var eventIDs []string
tx.Model(&database.Event{}).Where("webhook_id = ?", webhook.ID).Pluck("id", &eventIDs)
if len(eventIDs) > 0 {
// Collect delivery IDs for these events
var deliveryIDs []string
tx.Model(&database.Delivery{}).Where("event_id IN ?", eventIDs).Pluck("id", &deliveryIDs)
if len(deliveryIDs) > 0 {
// Soft-delete delivery results
tx.Where("delivery_id IN ?", deliveryIDs).Delete(&database.DeliveryResult{})
}
// Soft-delete deliveries
tx.Where("event_id IN ?", eventIDs).Delete(&database.Delivery{})
}
// Soft-delete events, entrypoints, targets, and the webhook itself
tx.Where("webhook_id = ?", webhook.ID).Delete(&database.Event{})
// Soft-delete entrypoints and targets (config tier)
tx.Where("webhook_id = ?", webhook.ID).Delete(&database.Entrypoint{})
tx.Where("webhook_id = ?", webhook.ID).Delete(&database.Target{})
tx.Delete(&webhook)
@@ -327,12 +328,23 @@ func (h *Handlers) HandleSourceDelete() http.HandlerFunc {
return
}
// Hard-delete the per-webhook event database file
if err := h.dbMgr.DeleteDB(webhook.ID); err != nil {
h.log.Error("failed to delete webhook event database",
"webhook_id", webhook.ID,
"error", err,
)
// Non-fatal: file may not exist if no events were ever received
}
h.log.Info("webhook deleted", "webhook_id", webhook.ID, "user_id", userID)
http.Redirect(w, r, "/sources", http.StatusSeeOther)
}
}
// HandleSourceLogs shows the request/response logs for a webhook.
// Events and deliveries are read from the per-webhook database.
// Target information is loaded from the main application database.
func (h *Handlers) HandleSourceLogs() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
userID, ok := h.getUserID(r)
@@ -349,6 +361,14 @@ func (h *Handlers) HandleSourceLogs() http.HandlerFunc {
return
}
// Load targets from main DB for display
var targets []database.Target
h.db.DB().Where("webhook_id = ?", webhook.ID).Find(&targets)
targetMap := make(map[string]database.Target, len(targets))
for _, t := range targets {
targetMap[t.ID] = t
}
// Pagination
page := 1
if p := r.URL.Query().Get("page"); p != "" {
@@ -359,25 +379,48 @@ func (h *Handlers) HandleSourceLogs() http.HandlerFunc {
perPage := 25
offset := (page - 1) * perPage
var totalEvents int64
h.db.DB().Model(&database.Event{}).Where("webhook_id = ?", webhook.ID).Count(&totalEvents)
var events []database.Event
h.db.DB().Where("webhook_id = ?", webhook.ID).
Order("created_at DESC").
Offset(offset).
Limit(perPage).
Find(&events)
// Load deliveries for each event
// EventWithDeliveries holds an event with its associated deliveries
type EventWithDeliveries struct {
database.Event
Deliveries []database.Delivery
}
eventsWithDeliveries := make([]EventWithDeliveries, len(events))
for i := range events {
eventsWithDeliveries[i].Event = events[i]
h.db.DB().Where("event_id = ?", events[i].ID).Preload("Target").Find(&eventsWithDeliveries[i].Deliveries)
var totalEvents int64
var eventsWithDeliveries []EventWithDeliveries
// Read events and deliveries from per-webhook database
if h.dbMgr.DBExists(webhook.ID) {
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
if err != nil {
h.log.Error("failed to get webhook database",
"webhook_id", webhook.ID,
"error", err,
)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
webhookDB.Model(&database.Event{}).Where("webhook_id = ?", webhook.ID).Count(&totalEvents)
var events []database.Event
webhookDB.Where("webhook_id = ?", webhook.ID).
Order("created_at DESC").
Offset(offset).
Limit(perPage).
Find(&events)
eventsWithDeliveries = make([]EventWithDeliveries, len(events))
for i := range events {
eventsWithDeliveries[i].Event = events[i]
// Load deliveries from per-webhook DB (without Target preload)
webhookDB.Where("event_id = ?", events[i].ID).Find(&eventsWithDeliveries[i].Deliveries)
// Manually assign targets from main DB
for j := range eventsWithDeliveries[i].Deliveries {
if target, ok := targetMap[eventsWithDeliveries[i].Deliveries[j].TargetID]; ok {
eventsWithDeliveries[i].Deliveries[j].Target = target
}
}
}
}
totalPages := int(totalEvents) / perPage