feat: implement per-webhook event databases
All checks were successful
check / check (push) Successful in 1m50s

Split data storage into main application DB (config only) and
per-webhook event databases (one SQLite file per webhook).

Architecture changes:
- New WebhookDBManager component manages per-webhook DB lifecycle
  (create, open, cache, delete) with lazy connection pooling via sync.Map
- Main DB (DBURL) stores only config: Users, Webhooks, Entrypoints,
  Targets, APIKeys
- Per-webhook DBs (DATA_DIR) store Events, Deliveries, DeliveryResults
  in files named events-{webhook_uuid}.db
- New DATA_DIR env var (default: ./data dev, /data/events prod)

Behavioral changes:
- Webhook creation creates per-webhook DB file
- Webhook deletion hard-deletes per-webhook DB file (config soft-deleted)
- Event ingestion writes to per-webhook DB, not main DB
- Delivery engine polls all per-webhook DBs for pending deliveries
- Database target type marks delivery as immediately successful (events
  are already in the dedicated per-webhook DB)
- Event log UI reads from per-webhook DBs with targets from main DB
- Existing webhooks without DB files get them created lazily

Removed:
- ArchivedEvent model (was a half-measure, replaced by per-webhook DBs)
- Event/Delivery/DeliveryResult removed from main DB migrations

Added:
- Comprehensive tests for WebhookDBManager (create, delete, lazy
  creation, delivery workflow, multiple webhooks, close all)
- Dockerfile creates /data/events directory

README updates:
- Per-webhook event databases documented as implemented (was Phase 2)
- DATA_DIR added to configuration table
- Docker instructions updated with data volume mount
- Data model diagram updated
- TODO updated (database separation moved to completed)

Closes #15
This commit is contained in:
clawbot
2026-03-01 17:06:43 -08:00
parent 6c393ccb78
commit 43c22a9e9a
13 changed files with 814 additions and 198 deletions

View File

@@ -16,6 +16,7 @@ const (
// HandleWebhook handles incoming webhook requests at entrypoint URLs.
// Only POST requests are accepted; all other methods return 405 Method Not Allowed.
// Events and deliveries are stored in the per-webhook database.
func (h *Handlers) HandleWebhook() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
@@ -36,7 +37,7 @@ func (h *Handlers) HandleWebhook() http.HandlerFunc {
"remote_addr", r.RemoteAddr,
)
// Look up entrypoint by path
// Look up entrypoint by path (from main application DB)
var entrypoint database.Entrypoint
result := h.db.DB().Where("path = ?", entrypointUUID).First(&entrypoint)
if result.Error != nil {
@@ -71,8 +72,27 @@ func (h *Handlers) HandleWebhook() http.HandlerFunc {
return
}
// Create the event in a transaction
tx := h.db.DB().Begin()
// Find all active targets for this webhook (from main application DB)
var targets []database.Target
if targetErr := h.db.DB().Where("webhook_id = ? AND active = ?", entrypoint.WebhookID, true).Find(&targets).Error; targetErr != nil {
h.log.Error("failed to query targets", "error", targetErr)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
// Get the per-webhook database for event storage
webhookDB, err := h.dbMgr.GetDB(entrypoint.WebhookID)
if err != nil {
h.log.Error("failed to get webhook database",
"webhook_id", entrypoint.WebhookID,
"error", err,
)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
// Create the event and deliveries in a transaction on the per-webhook DB
tx := webhookDB.Begin()
if tx.Error != nil {
h.log.Error("failed to begin transaction", "error", tx.Error)
http.Error(w, "Internal server error", http.StatusInternalServerError)
@@ -95,15 +115,6 @@ func (h *Handlers) HandleWebhook() http.HandlerFunc {
return
}
// Find all active targets for this webhook
var targets []database.Target
if err := tx.Where("webhook_id = ? AND active = ?", entrypoint.WebhookID, true).Find(&targets).Error; err != nil {
tx.Rollback()
h.log.Error("failed to query targets", "error", err)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
// Create delivery records for each active target
for i := range targets {
delivery := &database.Delivery{