webhooker/internal/handlers/source_management.go
clawbot 43c22a9e9a
All checks were successful
check / check (push) Successful in 1m50s
feat: implement per-webhook event databases
Split data storage into main application DB (config only) and
per-webhook event databases (one SQLite file per webhook).

Architecture changes:
- New WebhookDBManager component manages per-webhook DB lifecycle
  (create, open, cache, delete) with lazy connection pooling via sync.Map
- Main DB (DBURL) stores only config: Users, Webhooks, Entrypoints,
  Targets, APIKeys
- Per-webhook DBs (DATA_DIR) store Events, Deliveries, DeliveryResults
  in files named events-{webhook_uuid}.db
- New DATA_DIR env var (default: ./data dev, /data/events prod)

Behavioral changes:
- Webhook creation creates per-webhook DB file
- Webhook deletion hard-deletes per-webhook DB file (config soft-deleted)
- Event ingestion writes to per-webhook DB, not main DB
- Delivery engine polls all per-webhook DBs for pending deliveries
- Database target type marks delivery as immediately successful (events
  are already in the dedicated per-webhook DB)
- Event log UI reads from per-webhook DBs with targets from main DB
- Existing webhooks without DB files get them created lazily

Removed:
- ArchivedEvent model (was a half-measure, replaced by per-webhook DBs)
- Event/Delivery/DeliveryResult removed from main DB migrations

Added:
- Comprehensive tests for WebhookDBManager (create, delete, lazy
  creation, delivery workflow, multiple webhooks, close all)
- Dockerfile creates /data/events directory

README updates:
- Per-webhook event databases documented as implemented (was Phase 2)
- DATA_DIR added to configuration table
- Docker instructions updated with data volume mount
- Data model diagram updated
- TODO updated (database separation moved to completed)

Closes #15
2026-03-01 17:06:43 -08:00

722 lines
20 KiB
Go

package handlers
import (
"encoding/json"
"net/http"
"strconv"
"github.com/go-chi/chi"
"github.com/google/uuid"
"sneak.berlin/go/webhooker/internal/database"
)
// WebhookListItem holds data for the webhook list view.
type WebhookListItem struct {
database.Webhook
EntrypointCount int64
TargetCount int64
EventCount int64
}
// HandleSourceList shows a list of user's webhooks.
func (h *Handlers) HandleSourceList() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
userID, ok := h.getUserID(r)
if !ok {
http.Redirect(w, r, "/pages/login", http.StatusSeeOther)
return
}
var webhooks []database.Webhook
if err := h.db.DB().Where("user_id = ?", userID).Order("created_at DESC").Find(&webhooks).Error; err != nil {
h.log.Error("failed to list webhooks", "error", err)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
// Build list items with counts
items := make([]WebhookListItem, len(webhooks))
for i := range webhooks {
items[i].Webhook = webhooks[i]
h.db.DB().Model(&database.Entrypoint{}).Where("webhook_id = ?", webhooks[i].ID).Count(&items[i].EntrypointCount)
h.db.DB().Model(&database.Target{}).Where("webhook_id = ?", webhooks[i].ID).Count(&items[i].TargetCount)
// Event count comes from per-webhook DB
if h.dbMgr.DBExists(webhooks[i].ID) {
if webhookDB, err := h.dbMgr.GetDB(webhooks[i].ID); err == nil {
webhookDB.Model(&database.Event{}).Count(&items[i].EventCount)
}
}
}
data := map[string]interface{}{
"Webhooks": items,
}
h.renderTemplate(w, r, "sources_list.html", data)
}
}
// HandleSourceCreate shows the form to create a new webhook.
func (h *Handlers) HandleSourceCreate() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
data := map[string]interface{}{
"Error": "",
}
h.renderTemplate(w, r, "sources_new.html", data)
}
}
// HandleSourceCreateSubmit handles the webhook creation form submission.
func (h *Handlers) HandleSourceCreateSubmit() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
userID, ok := h.getUserID(r)
if !ok {
http.Redirect(w, r, "/pages/login", http.StatusSeeOther)
return
}
if err := r.ParseForm(); err != nil {
http.Error(w, "Bad request", http.StatusBadRequest)
return
}
name := r.FormValue("name")
description := r.FormValue("description")
retentionStr := r.FormValue("retention_days")
if name == "" {
data := map[string]interface{}{
"Error": "Name is required",
}
w.WriteHeader(http.StatusBadRequest)
h.renderTemplate(w, r, "sources_new.html", data)
return
}
retentionDays := 30
if retentionStr != "" {
if v, err := strconv.Atoi(retentionStr); err == nil && v > 0 {
retentionDays = v
}
}
tx := h.db.DB().Begin()
if tx.Error != nil {
h.log.Error("failed to begin transaction", "error", tx.Error)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
webhook := &database.Webhook{
UserID: userID,
Name: name,
Description: description,
RetentionDays: retentionDays,
}
if err := tx.Create(webhook).Error; err != nil {
tx.Rollback()
h.log.Error("failed to create webhook", "error", err)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
// Auto-create one entrypoint
entrypoint := &database.Entrypoint{
WebhookID: webhook.ID,
Path: uuid.New().String(),
Description: "Default entrypoint",
Active: true,
}
if err := tx.Create(entrypoint).Error; err != nil {
tx.Rollback()
h.log.Error("failed to create entrypoint", "error", err)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
if err := tx.Commit().Error; err != nil {
h.log.Error("failed to commit transaction", "error", err)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
// Create per-webhook event database
if err := h.dbMgr.CreateDB(webhook.ID); err != nil {
h.log.Error("failed to create webhook event database",
"webhook_id", webhook.ID,
"error", err,
)
// Non-fatal: the DB will be created lazily on first event
}
h.log.Info("webhook created",
"webhook_id", webhook.ID,
"name", name,
"user_id", userID,
)
http.Redirect(w, r, "/source/"+webhook.ID, http.StatusSeeOther)
}
}
// HandleSourceDetail shows details for a specific webhook.
func (h *Handlers) HandleSourceDetail() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
userID, ok := h.getUserID(r)
if !ok {
http.Redirect(w, r, "/pages/login", http.StatusSeeOther)
return
}
sourceID := chi.URLParam(r, "sourceID")
var webhook database.Webhook
if err := h.db.DB().Where("id = ? AND user_id = ?", sourceID, userID).First(&webhook).Error; err != nil {
http.NotFound(w, r)
return
}
var entrypoints []database.Entrypoint
h.db.DB().Where("webhook_id = ?", webhook.ID).Find(&entrypoints)
var targets []database.Target
h.db.DB().Where("webhook_id = ?", webhook.ID).Find(&targets)
// Recent events from per-webhook database
var events []database.Event
if h.dbMgr.DBExists(webhook.ID) {
if webhookDB, err := h.dbMgr.GetDB(webhook.ID); err == nil {
webhookDB.Where("webhook_id = ?", webhook.ID).Order("created_at DESC").Limit(20).Find(&events)
}
}
// Build host URL for display
host := r.Host
scheme := "https"
if r.TLS == nil {
scheme = "http"
}
// Check X-Forwarded headers
if fwdProto := r.Header.Get("X-Forwarded-Proto"); fwdProto != "" {
scheme = fwdProto
}
data := map[string]interface{}{
"Webhook": webhook,
"Entrypoints": entrypoints,
"Targets": targets,
"Events": events,
"BaseURL": scheme + "://" + host,
}
h.renderTemplate(w, r, "source_detail.html", data)
}
}
// HandleSourceEdit shows the form to edit a webhook.
func (h *Handlers) HandleSourceEdit() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
userID, ok := h.getUserID(r)
if !ok {
http.Redirect(w, r, "/pages/login", http.StatusSeeOther)
return
}
sourceID := chi.URLParam(r, "sourceID")
var webhook database.Webhook
if err := h.db.DB().Where("id = ? AND user_id = ?", sourceID, userID).First(&webhook).Error; err != nil {
http.NotFound(w, r)
return
}
data := map[string]interface{}{
"Webhook": webhook,
"Error": "",
}
h.renderTemplate(w, r, "source_edit.html", data)
}
}
// HandleSourceEditSubmit handles the webhook edit form submission.
func (h *Handlers) HandleSourceEditSubmit() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
userID, ok := h.getUserID(r)
if !ok {
http.Redirect(w, r, "/pages/login", http.StatusSeeOther)
return
}
sourceID := chi.URLParam(r, "sourceID")
var webhook database.Webhook
if err := h.db.DB().Where("id = ? AND user_id = ?", sourceID, userID).First(&webhook).Error; err != nil {
http.NotFound(w, r)
return
}
if err := r.ParseForm(); err != nil {
http.Error(w, "Bad request", http.StatusBadRequest)
return
}
name := r.FormValue("name")
if name == "" {
data := map[string]interface{}{
"Webhook": webhook,
"Error": "Name is required",
}
w.WriteHeader(http.StatusBadRequest)
h.renderTemplate(w, r, "source_edit.html", data)
return
}
webhook.Name = name
webhook.Description = r.FormValue("description")
if retStr := r.FormValue("retention_days"); retStr != "" {
if v, err := strconv.Atoi(retStr); err == nil && v > 0 {
webhook.RetentionDays = v
}
}
if err := h.db.DB().Save(&webhook).Error; err != nil {
h.log.Error("failed to update webhook", "error", err)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
http.Redirect(w, r, "/source/"+webhook.ID, http.StatusSeeOther)
}
}
// HandleSourceDelete handles webhook deletion.
// Configuration data is soft-deleted in the main DB.
// The per-webhook event database file is hard-deleted (permanently removed).
func (h *Handlers) HandleSourceDelete() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
userID, ok := h.getUserID(r)
if !ok {
http.Redirect(w, r, "/pages/login", http.StatusSeeOther)
return
}
sourceID := chi.URLParam(r, "sourceID")
var webhook database.Webhook
if err := h.db.DB().Where("id = ? AND user_id = ?", sourceID, userID).First(&webhook).Error; err != nil {
http.NotFound(w, r)
return
}
// Soft-delete configuration in the main application database
tx := h.db.DB().Begin()
if tx.Error != nil {
h.log.Error("failed to begin transaction", "error", tx.Error)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
// Soft-delete entrypoints and targets (config tier)
tx.Where("webhook_id = ?", webhook.ID).Delete(&database.Entrypoint{})
tx.Where("webhook_id = ?", webhook.ID).Delete(&database.Target{})
tx.Delete(&webhook)
if err := tx.Commit().Error; err != nil {
h.log.Error("failed to commit deletion", "error", err)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
// Hard-delete the per-webhook event database file
if err := h.dbMgr.DeleteDB(webhook.ID); err != nil {
h.log.Error("failed to delete webhook event database",
"webhook_id", webhook.ID,
"error", err,
)
// Non-fatal: file may not exist if no events were ever received
}
h.log.Info("webhook deleted", "webhook_id", webhook.ID, "user_id", userID)
http.Redirect(w, r, "/sources", http.StatusSeeOther)
}
}
// HandleSourceLogs shows the request/response logs for a webhook.
// Events and deliveries are read from the per-webhook database.
// Target information is loaded from the main application database.
func (h *Handlers) HandleSourceLogs() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
userID, ok := h.getUserID(r)
if !ok {
http.Redirect(w, r, "/pages/login", http.StatusSeeOther)
return
}
sourceID := chi.URLParam(r, "sourceID")
var webhook database.Webhook
if err := h.db.DB().Where("id = ? AND user_id = ?", sourceID, userID).First(&webhook).Error; err != nil {
http.NotFound(w, r)
return
}
// Load targets from main DB for display
var targets []database.Target
h.db.DB().Where("webhook_id = ?", webhook.ID).Find(&targets)
targetMap := make(map[string]database.Target, len(targets))
for _, t := range targets {
targetMap[t.ID] = t
}
// Pagination
page := 1
if p := r.URL.Query().Get("page"); p != "" {
if v, err := strconv.Atoi(p); err == nil && v > 0 {
page = v
}
}
perPage := 25
offset := (page - 1) * perPage
// EventWithDeliveries holds an event with its associated deliveries
type EventWithDeliveries struct {
database.Event
Deliveries []database.Delivery
}
var totalEvents int64
var eventsWithDeliveries []EventWithDeliveries
// Read events and deliveries from per-webhook database
if h.dbMgr.DBExists(webhook.ID) {
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
if err != nil {
h.log.Error("failed to get webhook database",
"webhook_id", webhook.ID,
"error", err,
)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
webhookDB.Model(&database.Event{}).Where("webhook_id = ?", webhook.ID).Count(&totalEvents)
var events []database.Event
webhookDB.Where("webhook_id = ?", webhook.ID).
Order("created_at DESC").
Offset(offset).
Limit(perPage).
Find(&events)
eventsWithDeliveries = make([]EventWithDeliveries, len(events))
for i := range events {
eventsWithDeliveries[i].Event = events[i]
// Load deliveries from per-webhook DB (without Target preload)
webhookDB.Where("event_id = ?", events[i].ID).Find(&eventsWithDeliveries[i].Deliveries)
// Manually assign targets from main DB
for j := range eventsWithDeliveries[i].Deliveries {
if target, ok := targetMap[eventsWithDeliveries[i].Deliveries[j].TargetID]; ok {
eventsWithDeliveries[i].Deliveries[j].Target = target
}
}
}
}
totalPages := int(totalEvents) / perPage
if int(totalEvents)%perPage != 0 {
totalPages++
}
data := map[string]interface{}{
"Webhook": webhook,
"Events": eventsWithDeliveries,
"Page": page,
"TotalPages": totalPages,
"TotalEvents": totalEvents,
"HasPrev": page > 1,
"HasNext": page < totalPages,
"PrevPage": page - 1,
"NextPage": page + 1,
}
h.renderTemplate(w, r, "source_logs.html", data)
}
}
// HandleEntrypointCreate handles adding a new entrypoint to a webhook.
func (h *Handlers) HandleEntrypointCreate() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
userID, ok := h.getUserID(r)
if !ok {
http.Redirect(w, r, "/pages/login", http.StatusSeeOther)
return
}
sourceID := chi.URLParam(r, "sourceID")
// Verify ownership
var webhook database.Webhook
if err := h.db.DB().Where("id = ? AND user_id = ?", sourceID, userID).First(&webhook).Error; err != nil {
http.NotFound(w, r)
return
}
if err := r.ParseForm(); err != nil {
http.Error(w, "Bad request", http.StatusBadRequest)
return
}
description := r.FormValue("description")
entrypoint := &database.Entrypoint{
WebhookID: webhook.ID,
Path: uuid.New().String(),
Description: description,
Active: true,
}
if err := h.db.DB().Create(entrypoint).Error; err != nil {
h.log.Error("failed to create entrypoint", "error", err)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
http.Redirect(w, r, "/source/"+webhook.ID, http.StatusSeeOther)
}
}
// HandleTargetCreate handles adding a new target to a webhook.
func (h *Handlers) HandleTargetCreate() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
userID, ok := h.getUserID(r)
if !ok {
http.Redirect(w, r, "/pages/login", http.StatusSeeOther)
return
}
sourceID := chi.URLParam(r, "sourceID")
var webhook database.Webhook
if err := h.db.DB().Where("id = ? AND user_id = ?", sourceID, userID).First(&webhook).Error; err != nil {
http.NotFound(w, r)
return
}
if err := r.ParseForm(); err != nil {
http.Error(w, "Bad request", http.StatusBadRequest)
return
}
name := r.FormValue("name")
targetType := database.TargetType(r.FormValue("type"))
url := r.FormValue("url")
maxRetriesStr := r.FormValue("max_retries")
if name == "" {
http.Error(w, "Name is required", http.StatusBadRequest)
return
}
// Validate target type
switch targetType {
case database.TargetTypeHTTP, database.TargetTypeRetry, database.TargetTypeDatabase, database.TargetTypeLog:
// valid
default:
http.Error(w, "Invalid target type", http.StatusBadRequest)
return
}
// Build config JSON for HTTP-based targets
var configJSON string
if targetType == database.TargetTypeHTTP || targetType == database.TargetTypeRetry {
if url == "" {
http.Error(w, "URL is required for HTTP targets", http.StatusBadRequest)
return
}
cfg := map[string]interface{}{
"url": url,
}
configBytes, err := json.Marshal(cfg)
if err != nil {
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
configJSON = string(configBytes)
}
maxRetries := 5
if maxRetriesStr != "" {
if v, err := strconv.Atoi(maxRetriesStr); err == nil && v > 0 {
maxRetries = v
}
}
target := &database.Target{
WebhookID: webhook.ID,
Name: name,
Type: targetType,
Active: true,
Config: configJSON,
MaxRetries: maxRetries,
}
if err := h.db.DB().Create(target).Error; err != nil {
h.log.Error("failed to create target", "error", err)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
http.Redirect(w, r, "/source/"+webhook.ID, http.StatusSeeOther)
}
}
// HandleEntrypointDelete handles deleting an individual entrypoint.
func (h *Handlers) HandleEntrypointDelete() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
userID, ok := h.getUserID(r)
if !ok {
http.Redirect(w, r, "/pages/login", http.StatusSeeOther)
return
}
sourceID := chi.URLParam(r, "sourceID")
entrypointID := chi.URLParam(r, "entrypointID")
// Verify webhook ownership
var webhook database.Webhook
if err := h.db.DB().Where("id = ? AND user_id = ?", sourceID, userID).First(&webhook).Error; err != nil {
http.NotFound(w, r)
return
}
// Delete entrypoint (must belong to this webhook)
result := h.db.DB().Where("id = ? AND webhook_id = ?", entrypointID, webhook.ID).Delete(&database.Entrypoint{})
if result.Error != nil {
h.log.Error("failed to delete entrypoint", "error", result.Error)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
http.Redirect(w, r, "/source/"+webhook.ID, http.StatusSeeOther)
}
}
// HandleEntrypointToggle handles toggling the active state of an entrypoint.
func (h *Handlers) HandleEntrypointToggle() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
userID, ok := h.getUserID(r)
if !ok {
http.Redirect(w, r, "/pages/login", http.StatusSeeOther)
return
}
sourceID := chi.URLParam(r, "sourceID")
entrypointID := chi.URLParam(r, "entrypointID")
// Verify webhook ownership
var webhook database.Webhook
if err := h.db.DB().Where("id = ? AND user_id = ?", sourceID, userID).First(&webhook).Error; err != nil {
http.NotFound(w, r)
return
}
// Find the entrypoint
var entrypoint database.Entrypoint
if err := h.db.DB().Where("id = ? AND webhook_id = ?", entrypointID, webhook.ID).First(&entrypoint).Error; err != nil {
http.NotFound(w, r)
return
}
// Toggle active state
entrypoint.Active = !entrypoint.Active
if err := h.db.DB().Save(&entrypoint).Error; err != nil {
h.log.Error("failed to toggle entrypoint", "error", err)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
http.Redirect(w, r, "/source/"+webhook.ID, http.StatusSeeOther)
}
}
// HandleTargetDelete handles deleting an individual target.
func (h *Handlers) HandleTargetDelete() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
userID, ok := h.getUserID(r)
if !ok {
http.Redirect(w, r, "/pages/login", http.StatusSeeOther)
return
}
sourceID := chi.URLParam(r, "sourceID")
targetID := chi.URLParam(r, "targetID")
// Verify webhook ownership
var webhook database.Webhook
if err := h.db.DB().Where("id = ? AND user_id = ?", sourceID, userID).First(&webhook).Error; err != nil {
http.NotFound(w, r)
return
}
// Delete target (must belong to this webhook)
result := h.db.DB().Where("id = ? AND webhook_id = ?", targetID, webhook.ID).Delete(&database.Target{})
if result.Error != nil {
h.log.Error("failed to delete target", "error", result.Error)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
http.Redirect(w, r, "/source/"+webhook.ID, http.StatusSeeOther)
}
}
// HandleTargetToggle handles toggling the active state of a target.
func (h *Handlers) HandleTargetToggle() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
userID, ok := h.getUserID(r)
if !ok {
http.Redirect(w, r, "/pages/login", http.StatusSeeOther)
return
}
sourceID := chi.URLParam(r, "sourceID")
targetID := chi.URLParam(r, "targetID")
// Verify webhook ownership
var webhook database.Webhook
if err := h.db.DB().Where("id = ? AND user_id = ?", sourceID, userID).First(&webhook).Error; err != nil {
http.NotFound(w, r)
return
}
// Find the target
var target database.Target
if err := h.db.DB().Where("id = ? AND webhook_id = ?", targetID, webhook.ID).First(&target).Error; err != nil {
http.NotFound(w, r)
return
}
// Toggle active state
target.Active = !target.Active
if err := h.db.DB().Save(&target).Error; err != nil {
h.log.Error("failed to toggle target", "error", err)
http.Error(w, "Internal server error", http.StatusInternalServerError)
return
}
http.Redirect(w, r, "/source/"+webhook.ID, http.StatusSeeOther)
}
}
// getUserID extracts the user ID from the session.
func (h *Handlers) getUserID(r *http.Request) (string, bool) {
sess, err := h.session.Get(r)
if err != nil {
return "", false
}
if !h.session.IsAuthenticated(sess) {
return "", false
}
return h.session.GetUserID(sess)
}