feat: implement per-webhook event databases
All checks were successful
check / check (push) Successful in 1m50s
All checks were successful
check / check (push) Successful in 1m50s
Split data storage into main application DB (config only) and
per-webhook event databases (one SQLite file per webhook).
Architecture changes:
- New WebhookDBManager component manages per-webhook DB lifecycle
(create, open, cache, delete) with lazy connection pooling via sync.Map
- Main DB (DBURL) stores only config: Users, Webhooks, Entrypoints,
Targets, APIKeys
- Per-webhook DBs (DATA_DIR) store Events, Deliveries, DeliveryResults
in files named events-{webhook_uuid}.db
- New DATA_DIR env var (default: ./data dev, /data/events prod)
Behavioral changes:
- Webhook creation creates per-webhook DB file
- Webhook deletion hard-deletes per-webhook DB file (config soft-deleted)
- Event ingestion writes to per-webhook DB, not main DB
- Delivery engine polls all per-webhook DBs for pending deliveries
- Database target type marks delivery as immediately successful (events
are already in the dedicated per-webhook DB)
- Event log UI reads from per-webhook DBs with targets from main DB
- Existing webhooks without DB files get them created lazily
Removed:
- ArchivedEvent model (was a half-measure, replaced by per-webhook DBs)
- Event/Delivery/DeliveryResult removed from main DB migrations
Added:
- Comprehensive tests for WebhookDBManager (create, delete, lazy
creation, delivery workflow, multiple webhooks, close all)
- Dockerfile creates /data/events directory
README updates:
- Per-webhook event databases documented as implemented (was Phase 2)
- DATA_DIR added to configuration table
- Docker instructions updated with data volume mount
- Data model diagram updated
- TODO updated (database separation moved to completed)
Closes #15
This commit is contained in:
294
internal/database/webhook_db_manager_test.go
Normal file
294
internal/database/webhook_db_manager_test.go
Normal file
@@ -0,0 +1,294 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/spf13/afero"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/fx/fxtest"
|
||||
"sneak.berlin/go/webhooker/internal/config"
|
||||
"sneak.berlin/go/webhooker/internal/globals"
|
||||
"sneak.berlin/go/webhooker/internal/logger"
|
||||
pkgconfig "sneak.berlin/go/webhooker/pkg/config"
|
||||
)
|
||||
|
||||
func setupTestWebhookDBManager(t *testing.T) (*WebhookDBManager, *fxtest.Lifecycle) {
|
||||
t.Helper()
|
||||
|
||||
fs := afero.NewMemMapFs()
|
||||
testConfigYAML := `
|
||||
environments:
|
||||
dev:
|
||||
config:
|
||||
port: 8080
|
||||
debug: false
|
||||
dburl: "file::memory:?cache=shared"
|
||||
secrets:
|
||||
sessionKey: d2ViaG9va2VyLWRldi1zZXNzaW9uLWtleS1pbnNlY3VyZSE=
|
||||
configDefaults:
|
||||
port: 8080
|
||||
`
|
||||
require.NoError(t, afero.WriteFile(fs, "config.yaml", []byte(testConfigYAML), 0644))
|
||||
pkgconfig.SetFs(fs)
|
||||
|
||||
lc := fxtest.NewLifecycle(t)
|
||||
|
||||
globals.Appname = "webhooker-test"
|
||||
globals.Version = "test"
|
||||
globals.Buildarch = "test"
|
||||
|
||||
g, err := globals.New(lc)
|
||||
require.NoError(t, err)
|
||||
|
||||
l, err := logger.New(lc, logger.LoggerParams{Globals: g})
|
||||
require.NoError(t, err)
|
||||
|
||||
dataDir := filepath.Join(t.TempDir(), "events")
|
||||
|
||||
cfg := &config.Config{
|
||||
DBURL: "file::memory:?cache=shared",
|
||||
DataDir: dataDir,
|
||||
SessionKey: "d2ViaG9va2VyLWRldi1zZXNzaW9uLWtleS1pbnNlY3VyZSE=",
|
||||
}
|
||||
_ = cfg
|
||||
|
||||
mgr, err := NewWebhookDBManager(lc, WebhookDBManagerParams{
|
||||
Config: cfg,
|
||||
Logger: l,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
return mgr, lc
|
||||
}
|
||||
|
||||
func TestWebhookDBManager_CreateAndGetDB(t *testing.T) {
|
||||
mgr, lc := setupTestWebhookDBManager(t)
|
||||
ctx := context.Background()
|
||||
require.NoError(t, lc.Start(ctx))
|
||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
||||
|
||||
webhookID := uuid.New().String()
|
||||
|
||||
// DB should not exist yet
|
||||
assert.False(t, mgr.DBExists(webhookID))
|
||||
|
||||
// Create the DB
|
||||
err := mgr.CreateDB(webhookID)
|
||||
require.NoError(t, err)
|
||||
|
||||
// DB file should now exist
|
||||
assert.True(t, mgr.DBExists(webhookID))
|
||||
|
||||
// Get the DB again (should use cached connection)
|
||||
db, err := mgr.GetDB(webhookID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, db)
|
||||
|
||||
// Verify we can write an event
|
||||
event := &Event{
|
||||
WebhookID: webhookID,
|
||||
EntrypointID: uuid.New().String(),
|
||||
Method: "POST",
|
||||
Headers: `{"Content-Type":["application/json"]}`,
|
||||
Body: `{"test": true}`,
|
||||
ContentType: "application/json",
|
||||
}
|
||||
require.NoError(t, db.Create(event).Error)
|
||||
assert.NotEmpty(t, event.ID)
|
||||
|
||||
// Verify we can read it back
|
||||
var readEvent Event
|
||||
require.NoError(t, db.First(&readEvent, "id = ?", event.ID).Error)
|
||||
assert.Equal(t, webhookID, readEvent.WebhookID)
|
||||
assert.Equal(t, "POST", readEvent.Method)
|
||||
assert.Equal(t, `{"test": true}`, readEvent.Body)
|
||||
}
|
||||
|
||||
func TestWebhookDBManager_DeleteDB(t *testing.T) {
|
||||
mgr, lc := setupTestWebhookDBManager(t)
|
||||
ctx := context.Background()
|
||||
require.NoError(t, lc.Start(ctx))
|
||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
||||
|
||||
webhookID := uuid.New().String()
|
||||
|
||||
// Create the DB and write some data
|
||||
require.NoError(t, mgr.CreateDB(webhookID))
|
||||
db, err := mgr.GetDB(webhookID)
|
||||
require.NoError(t, err)
|
||||
|
||||
event := &Event{
|
||||
WebhookID: webhookID,
|
||||
EntrypointID: uuid.New().String(),
|
||||
Method: "POST",
|
||||
Body: `{"test": true}`,
|
||||
ContentType: "application/json",
|
||||
}
|
||||
require.NoError(t, db.Create(event).Error)
|
||||
|
||||
// Delete the DB
|
||||
require.NoError(t, mgr.DeleteDB(webhookID))
|
||||
|
||||
// File should no longer exist
|
||||
assert.False(t, mgr.DBExists(webhookID))
|
||||
|
||||
// Verify the file is actually gone from disk
|
||||
dbPath := mgr.dbPath(webhookID)
|
||||
_, err = os.Stat(dbPath)
|
||||
assert.True(t, os.IsNotExist(err))
|
||||
}
|
||||
|
||||
func TestWebhookDBManager_LazyCreation(t *testing.T) {
|
||||
mgr, lc := setupTestWebhookDBManager(t)
|
||||
ctx := context.Background()
|
||||
require.NoError(t, lc.Start(ctx))
|
||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
||||
|
||||
webhookID := uuid.New().String()
|
||||
|
||||
// GetDB should lazily create the database
|
||||
db, err := mgr.GetDB(webhookID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, db)
|
||||
|
||||
// File should now exist
|
||||
assert.True(t, mgr.DBExists(webhookID))
|
||||
}
|
||||
|
||||
func TestWebhookDBManager_DeliveryWorkflow(t *testing.T) {
|
||||
mgr, lc := setupTestWebhookDBManager(t)
|
||||
ctx := context.Background()
|
||||
require.NoError(t, lc.Start(ctx))
|
||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
||||
|
||||
webhookID := uuid.New().String()
|
||||
targetID := uuid.New().String()
|
||||
|
||||
db, err := mgr.GetDB(webhookID)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create an event
|
||||
event := &Event{
|
||||
WebhookID: webhookID,
|
||||
EntrypointID: uuid.New().String(),
|
||||
Method: "POST",
|
||||
Headers: `{"Content-Type":["application/json"]}`,
|
||||
Body: `{"payload": "test"}`,
|
||||
ContentType: "application/json",
|
||||
}
|
||||
require.NoError(t, db.Create(event).Error)
|
||||
|
||||
// Create a delivery
|
||||
delivery := &Delivery{
|
||||
EventID: event.ID,
|
||||
TargetID: targetID,
|
||||
Status: DeliveryStatusPending,
|
||||
}
|
||||
require.NoError(t, db.Create(delivery).Error)
|
||||
|
||||
// Query pending deliveries
|
||||
var pending []Delivery
|
||||
require.NoError(t, db.Where("status = ?", DeliveryStatusPending).
|
||||
Preload("Event").
|
||||
Find(&pending).Error)
|
||||
require.Len(t, pending, 1)
|
||||
assert.Equal(t, event.ID, pending[0].EventID)
|
||||
assert.Equal(t, "POST", pending[0].Event.Method)
|
||||
|
||||
// Create a delivery result
|
||||
result := &DeliveryResult{
|
||||
DeliveryID: delivery.ID,
|
||||
AttemptNum: 1,
|
||||
Success: true,
|
||||
StatusCode: 200,
|
||||
Duration: 42,
|
||||
}
|
||||
require.NoError(t, db.Create(result).Error)
|
||||
|
||||
// Update delivery status
|
||||
require.NoError(t, db.Model(delivery).Update("status", DeliveryStatusDelivered).Error)
|
||||
|
||||
// Verify no more pending deliveries
|
||||
var stillPending []Delivery
|
||||
require.NoError(t, db.Where("status = ?", DeliveryStatusPending).Find(&stillPending).Error)
|
||||
assert.Empty(t, stillPending)
|
||||
}
|
||||
|
||||
func TestWebhookDBManager_MultipleWebhooks(t *testing.T) {
|
||||
mgr, lc := setupTestWebhookDBManager(t)
|
||||
ctx := context.Background()
|
||||
require.NoError(t, lc.Start(ctx))
|
||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
||||
|
||||
webhook1 := uuid.New().String()
|
||||
webhook2 := uuid.New().String()
|
||||
|
||||
// Create DBs for two webhooks
|
||||
require.NoError(t, mgr.CreateDB(webhook1))
|
||||
require.NoError(t, mgr.CreateDB(webhook2))
|
||||
|
||||
db1, err := mgr.GetDB(webhook1)
|
||||
require.NoError(t, err)
|
||||
db2, err := mgr.GetDB(webhook2)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Write events to each webhook's DB
|
||||
event1 := &Event{
|
||||
WebhookID: webhook1,
|
||||
EntrypointID: uuid.New().String(),
|
||||
Method: "POST",
|
||||
Body: `{"webhook": 1}`,
|
||||
ContentType: "application/json",
|
||||
}
|
||||
event2 := &Event{
|
||||
WebhookID: webhook2,
|
||||
EntrypointID: uuid.New().String(),
|
||||
Method: "PUT",
|
||||
Body: `{"webhook": 2}`,
|
||||
ContentType: "application/json",
|
||||
}
|
||||
require.NoError(t, db1.Create(event1).Error)
|
||||
require.NoError(t, db2.Create(event2).Error)
|
||||
|
||||
// Verify isolation: each DB only has its own events
|
||||
var count1 int64
|
||||
db1.Model(&Event{}).Count(&count1)
|
||||
assert.Equal(t, int64(1), count1)
|
||||
|
||||
var count2 int64
|
||||
db2.Model(&Event{}).Count(&count2)
|
||||
assert.Equal(t, int64(1), count2)
|
||||
|
||||
// Delete webhook1's DB, webhook2 should be unaffected
|
||||
require.NoError(t, mgr.DeleteDB(webhook1))
|
||||
assert.False(t, mgr.DBExists(webhook1))
|
||||
assert.True(t, mgr.DBExists(webhook2))
|
||||
|
||||
// webhook2's data should still be accessible
|
||||
var events []Event
|
||||
require.NoError(t, db2.Find(&events).Error)
|
||||
assert.Len(t, events, 1)
|
||||
assert.Equal(t, "PUT", events[0].Method)
|
||||
}
|
||||
|
||||
func TestWebhookDBManager_CloseAll(t *testing.T) {
|
||||
mgr, lc := setupTestWebhookDBManager(t)
|
||||
ctx := context.Background()
|
||||
require.NoError(t, lc.Start(ctx))
|
||||
|
||||
// Create a few DBs
|
||||
for i := 0; i < 3; i++ {
|
||||
require.NoError(t, mgr.CreateDB(uuid.New().String()))
|
||||
}
|
||||
|
||||
// CloseAll should close all connections without error
|
||||
require.NoError(t, mgr.CloseAll())
|
||||
|
||||
// Stop lifecycle (CloseAll already called, but shouldn't panic)
|
||||
require.NoError(t, lc.Stop(ctx))
|
||||
}
|
||||
Reference in New Issue
Block a user