Files
webhooker/internal/delivery/engine_integration_test.go
sneak dfd559417e
All checks were successful
check / check (push) Successful in 3m10s
Expose delivery metrics on /metrics (closes #209)
/metrics carried only the inbound HTTP surface, so a destination
failing for an hour, a growing retry backlog and a stuck-open circuit
breaker were all invisible: the receive side stays healthy in each
case because it is.

New internal/metrics registers, on the existing default registry that
the go-http-metrics recorder and the promhttp handler already share:

- webhooker_events_received_total
- webhooker_delivery_attempts_total
- webhooker_deliveries_succeeded_total
- webhooker_deliveries_failed_total
- webhooker_delivery_retries_total
- webhooker_delivery_duration_seconds
- webhooker_deliveries_pending / _retrying
- webhooker_circuit_breakers_open

The route mounting is untouched.

Every delivery metric carries one label, target_type, whose domain is
the four target-type constants; anything outside it collapses to
"unknown" so no series can be minted from a UUID. Target ids, event
ids and entrypoint ids are deliberately not labels.

An attempt is counted, and its duration observed, only where one was
actually dispatched — the target's own result path, which is also
where the DeliveryResult is written. A delivery an open circuit
breaker refuses sends nothing and records no result row; counting it
would climb the attempts counter with no traffic behind it and pull
the duration quantiles down for as long as the breaker stayed open,
moving the metric the wrong way during the outage it exists to reveal.
The log and database targets now time their own work, so their result
rows carry a real duration too.

The outcome counters move after the status row is written rather than
before, so a transition the database rejected is never reported as an
outcome that happened.

The queue-depth gauges are counted out of the per-webhook databases by
a 30s sampler rather than tracked as deltas, which would need seeding
at startup and would drift on any transition that failed to persist.
They publish an "unknown" series from registration: deliveries queued
against a target that has since been deleted resolve to the empty type
and are folded there, because a backlog behind a deleted target is
precisely the one nobody is watching. The open-breaker gauge is
recounted from the target's breaker registry on every state change.

The orphaned-retry terminal path takes the target type as an argument
rather than attaching the loaded target to the delivery. That path
loads the delivery without its target relation on purpose: a populated
Delivery.Target makes GORM's SaveBeforeAssociations upsert the whole
target row on the status UPDATE, writing the plaintext target config —
the credential, for a slack target — into the per-webhook events
database. A test asserts that path leaves the targets table empty.
2026-08-20 05:08:31 +00:00

1343 lines
25 KiB
Go

package delivery_test
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
_ "modernc.org/sqlite"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/delivery"
)
// iSetup holds common integration test dependencies.
type iSetup struct {
MainDB *gorm.DB
DBMgr *database.WebhookDBManager
WebhookID string
WebhookDB *gorm.DB
Engine *delivery.Engine
}
func newISetup(t *testing.T) iSetup {
t.Helper()
mainDB := iMainDB(t)
dbMgr := iDBManager(t)
wID := uuid.New().String()
wDB := iSeedWebhookDB(t, dbMgr, wID)
return iSetup{
MainDB: mainDB,
DBMgr: dbMgr,
WebhookID: wID,
WebhookDB: wDB,
Engine: delivery.NewTestEngineWithDB(
database.NewTestDatabase(mainDB),
dbMgr,
slog.New(slog.NewTextHandler(
os.Stderr,
&slog.HandlerOptions{
Level: slog.LevelDebug,
},
)),
&http.Client{Timeout: 5 * time.Second},
2,
),
}
}
func iMainDB(t *testing.T) *gorm.DB {
t.Helper()
dbPath := filepath.Join(
t.TempDir(), "main-test.db",
)
dsn := fmt.Sprintf(
"file:%s?cache=shared&mode=rwc", dbPath,
)
sqlDB, err := sql.Open("sqlite", dsn)
require.NoError(t, err)
t.Cleanup(func() { _ = sqlDB.Close() })
db, err := gorm.Open(
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
)
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(
&database.Webhook{},
&database.Target{},
&database.User{},
&database.Setting{},
))
return db
}
func iDBManager(
t *testing.T,
) *database.WebhookDBManager {
t.Helper()
return database.NewTestWebhookDBManager(t.TempDir())
}
func iSeedWebhookDB(
t *testing.T,
mgr *database.WebhookDBManager,
webhookID string,
) *gorm.DB {
t.Helper()
db, err := mgr.GetDB(webhookID)
require.NoError(t, err)
return db
}
func iHTTPConfig(url string) string {
cfg := delivery.HTTPTargetConfig{URL: url}
data, err := json.Marshal(cfg)
if err != nil {
panic("failed to marshal HTTPTargetConfig")
}
return string(data)
}
func iEngine(
t *testing.T, workers int,
) *delivery.Engine {
t.Helper()
return delivery.NewTestEngine(
slog.New(slog.NewTextHandler(
os.Stderr,
&slog.HandlerOptions{Level: slog.LevelDebug},
)),
&http.Client{Timeout: 5 * time.Second},
workers,
)
}
// iSeedEvent creates a test event in the database.
func iSeedEvent(
t *testing.T,
db *gorm.DB,
webhookID, body string,
) database.Event {
t.Helper()
event := database.Event{
WebhookID: webhookID,
EntrypointID: uuid.New().String(),
Method: http.MethodPost,
Headers: `{}`,
Body: body,
ContentType: testContentType,
}
require.NoError(t, db.Create(&event).Error)
return event
}
// iSeedDelivery creates a test delivery record.
func iSeedDelivery(
t *testing.T,
db *gorm.DB,
eventID, targetID string,
status database.DeliveryStatus,
) database.Delivery {
t.Helper()
d := database.Delivery{
EventID: eventID,
TargetID: targetID,
Status: status,
}
require.NoError(t, db.Create(&d).Error)
return d
}
// iTask builds a delivery.Task for integration tests.
func iTask(
d database.Delivery,
event database.Event,
webhookID, targetID, name, config string,
maxRetries, attemptNum int,
body *string,
) delivery.Task {
return delivery.Task{
DeliveryID: d.ID,
EventID: event.ID,
WebhookID: webhookID,
TargetID: targetID,
TargetName: name,
TargetType: database.TargetTypeHTTP,
TargetConfig: config,
MaxRetries: maxRetries,
Method: event.Method,
Headers: event.Headers,
ContentType: event.ContentType,
Body: body,
AttemptNum: attemptNum,
}
}
// iAssertStatus checks the delivery status.
func iAssertStatus(
t *testing.T,
db *gorm.DB,
deliveryID string,
expected database.DeliveryStatus,
) {
t.Helper()
var updated database.Delivery
require.NoError(t, db.First(
&updated, "id = ?", deliveryID,
).Error)
assert.Equal(t, expected, updated.Status)
}
// --- processNewTask Tests ---
func TestProcessNewTask_InlineBody(t *testing.T) {
t.Parallel()
s := newISetup(t)
var received atomic.Bool
ts := httptest.NewServer(
http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
received.Store(true)
w.WriteHeader(http.StatusOK)
},
),
)
defer ts.Close()
event := iSeedEvent(
t, s.WebhookDB, s.WebhookID,
`{"hello":"world"}`,
)
targetID := uuid.New().String()
d := iSeedDelivery(
t, s.WebhookDB, event.ID, targetID,
database.DeliveryStatusPending,
)
bodyStr := event.Body
cfg := iHTTPConfig(ts.URL)
task := iTask(
d, event, s.WebhookID, targetID,
"test-target", cfg, 0, 1, &bodyStr,
)
s.Engine.ExportProcessNewTask(
context.TODO(), &task,
)
assert.True(t, received.Load())
iAssertStatus(t, s.WebhookDB, d.ID,
database.DeliveryStatusDelivered,
)
}
func TestProcessNewTask_LargeBody_FetchFromDB(
t *testing.T,
) {
t.Parallel()
s := newISetup(t)
largeBody := strings.Repeat(
"x", delivery.MaxInlineBodySize+100,
)
var receivedBody string
ts := httptest.NewServer(
http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
receivedBody = string(body)
w.WriteHeader(http.StatusOK)
},
),
)
defer ts.Close()
event := iSeedEvent(
t, s.WebhookDB, s.WebhookID, largeBody,
)
targetID := uuid.New().String()
d := iSeedDelivery(
t, s.WebhookDB, event.ID, targetID,
database.DeliveryStatusPending,
)
cfg := iHTTPConfig(ts.URL)
task := iTask(
d, event, s.WebhookID, targetID,
"test-large", cfg, 0, 1, nil,
)
s.Engine.ExportProcessNewTask(
context.TODO(), &task,
)
assert.Equal(t, largeBody, receivedBody)
iAssertStatus(t, s.WebhookDB, d.ID,
database.DeliveryStatusDelivered,
)
}
func TestProcessNewTask_InvalidWebhookID(t *testing.T) {
t.Parallel()
s := newISetup(t)
task := delivery.Task{
DeliveryID: uuid.New().String(),
EventID: uuid.New().String(),
WebhookID: uuid.New().String(),
TargetID: uuid.New().String(),
TargetName: "test",
TargetType: database.TargetTypeHTTP,
TargetConfig: iHTTPConfig("http://localhost:9999"),
MaxRetries: 0,
Body: nil,
AttemptNum: 1,
}
s.Engine.ExportProcessNewTask(
context.TODO(), &task,
)
}
// --- processRetryTask Tests ---
func TestProcessRetryTask_SuccessfulRetry(t *testing.T) {
t.Parallel()
s := newISetup(t)
ts := httptest.NewServer(
http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
},
),
)
defer ts.Close()
event := iSeedEvent(
t, s.WebhookDB, s.WebhookID,
`{"retry":"test"}`,
)
targetID := uuid.New().String()
d := iSeedDelivery(
t, s.WebhookDB, event.ID, targetID,
database.DeliveryStatusRetrying,
)
bodyStr := event.Body
cfg := iHTTPConfig(ts.URL)
task := iTask(
d, event, s.WebhookID, targetID,
"retry-target", cfg, 5, 2, &bodyStr,
)
s.Engine.ExportProcessRetryTask(
context.TODO(), &task,
)
iAssertStatus(t, s.WebhookDB, d.ID,
database.DeliveryStatusDelivered,
)
}
func TestProcessRetryTask_SkipsNonRetryingDelivery(
t *testing.T,
) {
t.Parallel()
s := newISetup(t)
event := iSeedEvent(
t, s.WebhookDB, s.WebhookID,
`{"skip":"test"}`,
)
targetID := uuid.New().String()
d := iSeedDelivery(
t, s.WebhookDB, event.ID, targetID,
database.DeliveryStatusDelivered,
)
bodyStr := event.Body
cfg := iHTTPConfig("http://localhost:1")
task := iTask(
d, event, s.WebhookID, targetID,
"skip-target", cfg, 5, 2, &bodyStr,
)
s.Engine.ExportProcessRetryTask(
context.TODO(), &task,
)
iAssertStatus(t, s.WebhookDB, d.ID,
database.DeliveryStatusDelivered,
)
}
func TestProcessRetryTask_LargeBody_FetchFromDB(
t *testing.T,
) {
t.Parallel()
s := newISetup(t)
ts := httptest.NewServer(
http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
},
),
)
defer ts.Close()
largeBody := strings.Repeat(
"z", delivery.MaxInlineBodySize+50,
)
event := iSeedEvent(
t, s.WebhookDB, s.WebhookID, largeBody,
)
targetID := uuid.New().String()
d := iSeedDelivery(
t, s.WebhookDB, event.ID, targetID,
database.DeliveryStatusRetrying,
)
cfg := iHTTPConfig(ts.URL)
task := iTask(
d, event, s.WebhookID, targetID,
"retry-large", cfg, 5, 2, nil,
)
s.Engine.ExportProcessRetryTask(
context.TODO(), &task,
)
iAssertStatus(t, s.WebhookDB, d.ID,
database.DeliveryStatusDelivered,
)
}
// --- Worker Lifecycle Tests ---
func TestWorkerLifecycle_StartStop(t *testing.T) {
t.Parallel()
s := newISetup(t)
s.Engine.ExportStart()
event := iSeedEvent(
t, s.WebhookDB, s.WebhookID,
`{"lifecycle":"test"}`,
)
targetID := uuid.New().String()
d := iSeedDelivery(
t, s.WebhookDB, event.ID, targetID,
database.DeliveryStatusPending,
)
bodyStr := event.Body
task := iTask(
d, event, s.WebhookID, targetID,
"lifecycle-test", "",
0, 1, &bodyStr,
)
task.TargetType = database.TargetTypeLog
s.Engine.Notify([]delivery.Task{task})
iWaitForDelivered(t, s.WebhookDB, d.ID)
require.NoError(t, s.Engine.ExportStop(context.Background()))
}
// iWaitForDelivered polls until the delivery reaches the
// delivered status.
func iWaitForDelivered(
t *testing.T,
db *gorm.DB,
deliveryID string,
) {
t.Helper()
require.Eventually(t, func() bool {
var d database.Delivery
err := db.First(
&d, "id = ?", deliveryID,
).Error
if err != nil {
return false
}
return d.Status == database.DeliveryStatusDelivered
}, 5*time.Second, 50*time.Millisecond)
}
func TestWorkerLifecycle_ProcessesRetryChannel(
t *testing.T,
) {
t.Parallel()
s := newISetup(t)
ts := httptest.NewServer(
http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
},
),
)
defer ts.Close()
event := iSeedEvent(
t, s.WebhookDB, s.WebhookID,
`{"retry-chan":"test"}`,
)
targetID := uuid.New().String()
d := iSeedDelivery(
t, s.WebhookDB, event.ID, targetID,
database.DeliveryStatusRetrying,
)
s.Engine.ExportStart()
bodyStr := event.Body
cfg := iHTTPConfig(ts.URL)
task := iTask(
d, event, s.WebhookID, targetID,
"retry-chan-test", cfg, 5, 2, &bodyStr,
)
s.Engine.ExportRetryCh() <- task
iWaitForDelivered(t, s.WebhookDB, d.ID)
require.NoError(t, s.Engine.ExportStop(context.Background()))
}
// --- processDelivery: unknown target type ---
func TestProcessDelivery_UnknownTargetType(
t *testing.T,
) {
t.Parallel()
s := newISetup(t)
event := iSeedEvent(
t, s.WebhookDB, s.WebhookID,
`{"unknown":"type"}`,
)
del := iSeedDelivery(
t, s.WebhookDB, event.ID,
uuid.New().String(),
database.DeliveryStatusPending,
)
d := &database.Delivery{
EventID: event.ID,
TargetID: del.TargetID,
Status: database.DeliveryStatusPending,
Event: event,
Target: database.Target{
Name: "unknown",
Type: database.TargetType("unknown"),
},
}
d.ID = del.ID
task := &delivery.Task{
DeliveryID: del.ID,
TargetType: database.TargetType("unknown"),
}
s.Engine.ExportProcessDelivery(
context.TODO(), s.WebhookDB, d, task,
)
iAssertStatus(t, s.WebhookDB, del.ID,
database.DeliveryStatusFailed,
)
}
// --- Recovery Tests ---
func TestRecoverPendingDeliveries(t *testing.T) {
t.Parallel()
s := newISetup(t)
targetID := uuid.New().String()
iCreateTarget(t, s.MainDB, targetID,
s.WebhookID, "recovery-target",
database.TargetTypeLog, "", 0,
)
iSeedPendingDeliveries(
t, s.WebhookDB, s.WebhookID, targetID, 3,
)
s.Engine.ExportRecoverPendingDeliveries(
context.Background(), s.WebhookDB,
s.WebhookID,
)
for i := range 3 {
select {
case task := <-s.Engine.ExportDeliveryCh():
assert.Equal(t, targetID, task.TargetID)
assert.Equal(t,
database.TargetTypeLog,
task.TargetType,
)
case <-time.After(2 * time.Second):
t.Fatalf("expected task %d", i)
}
}
}
// iCreateTarget creates a target in the main DB.
func iCreateTarget(
t *testing.T,
mainDB *gorm.DB,
targetID, webhookID, name string,
targetType database.TargetType,
config string,
maxRetries int,
) {
t.Helper()
target := database.Target{
WebhookID: webhookID,
Name: name,
Type: targetType,
Active: true,
Config: config,
MaxRetries: maxRetries,
}
target.ID = targetID
require.NoError(t, mainDB.Create(&target).Error)
}
// iSeedPendingDeliveries creates n pending deliveries.
func iSeedPendingDeliveries(
t *testing.T,
webhookDB *gorm.DB,
webhookID, targetID string,
n int,
) {
t.Helper()
for i := range n {
event := iSeedEvent(
t, webhookDB, webhookID,
fmt.Sprintf(`{"recovery":%d}`, i),
)
iSeedDelivery(
t, webhookDB, event.ID, targetID,
database.DeliveryStatusPending,
)
}
}
func TestRecoverWebhookDeliveries_RetryingDeliveries(
t *testing.T,
) {
t.Parallel()
s := newISetup(t)
targetID := uuid.New().String()
iCreateTarget(t, s.MainDB, targetID,
s.WebhookID, "retry-recovery",
database.TargetTypeHTTP,
iHTTPConfig("http://example.com/hook"), 5,
)
event := iSeedEvent(
t, s.WebhookDB, s.WebhookID,
`{"retry-recovery":"test"}`,
)
d := iSeedDelivery(
t, s.WebhookDB, event.ID, targetID,
database.DeliveryStatusRetrying,
)
iSeedFailedResult(t, s.WebhookDB, d.ID)
iCreateWebhook(
t, s.MainDB, s.WebhookID, "test-webhook",
)
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID,
)
select {
case task := <-s.Engine.ExportRetryCh():
assert.Equal(t, d.ID, task.DeliveryID)
assert.Equal(t, targetID, task.TargetID)
assert.Equal(t, 2, task.AttemptNum)
case <-time.After(5 * time.Second):
t.Fatal("expected retry task from recovery")
}
// Regression guard: a target that still supports retries
// must be rescheduled, never terminally failed, and must
// not gain a synthetic result row.
iAssertStatus(
t, s.WebhookDB, d.ID,
database.DeliveryStatusRetrying,
)
assert.Len(t, iResults(t, s.WebhookDB, d.ID), 1)
}
// --- Retrying deliveries whose target type changed ---
// iSeedRetryingWithType seeds a retrying delivery with one
// recorded failed attempt against a target of the given type,
// standing in for a target whose type was edited in the main
// database while the delivery was still retrying.
func iSeedRetryingWithType(
t *testing.T,
s iSetup,
targetType database.TargetType,
) string {
t.Helper()
targetID := uuid.New().String()
iCreateTarget(t, s.MainDB, targetID,
s.WebhookID, "mutated-target", targetType,
iHTTPConfig("http://example.com/hook"), 5,
)
event := iSeedEvent(
t, s.WebhookDB, s.WebhookID,
`{"orphaned":"retry"}`,
)
d := iSeedDelivery(
t, s.WebhookDB, event.ID, targetID,
database.DeliveryStatusRetrying,
)
iSeedFailedResult(t, s.WebhookDB, d.ID)
return d.ID
}
// iResults loads a delivery's results in attempt order.
func iResults(
t *testing.T, db *gorm.DB, deliveryID string,
) []database.DeliveryResult {
t.Helper()
var results []database.DeliveryResult
require.NoError(t, db.
Where("delivery_id = ?", deliveryID).
Order("attempt_num").
Find(&results).Error)
return results
}
// iAssertTerminallyFailed asserts the delivery ended failed
// with a result row recording why, and was not rescheduled.
func iAssertTerminallyFailed(
t *testing.T,
s iSetup,
deliveryID string,
targetType database.TargetType,
) {
t.Helper()
iAssertStatus(
t, s.WebhookDB, deliveryID,
database.DeliveryStatusFailed,
)
results := iResults(t, s.WebhookDB, deliveryID)
require.Len(t, results, 2)
last := results[1]
assert.False(t, last.Success)
assert.Equal(t, 2, last.AttemptNum)
assert.Contains(
t, last.Error, string(targetType),
)
assert.Contains(
t, last.Error, "does not support retries",
)
assert.Empty(t, s.Engine.ExportRetryCh())
}
func TestRecoverSingleRetry_TypeNoLongerRetries(
t *testing.T,
) {
t.Parallel()
s := newISetup(t)
iCreateWebhook(
t, s.MainDB, s.WebhookID, "mutated-type",
)
deliveryID := iSeedRetryingWithType(
t, s, database.TargetTypeLog,
)
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID,
)
iAssertTerminallyFailed(
t, s, deliveryID, database.TargetTypeLog,
)
}
func TestSweepSingleRetry_TypeNoLongerRetries(
t *testing.T,
) {
t.Parallel()
s := newISetup(t)
iCreateWebhook(
t, s.MainDB, s.WebhookID, "mutated-type-sweep",
)
deliveryID := iSeedRetryingWithType(
t, s, database.TargetTypeDatabase,
)
s.Engine.ExportSweepWebhookRetries(
context.Background(), s.WebhookID,
)
iAssertTerminallyFailed(
t, s, deliveryID, database.TargetTypeDatabase,
)
}
// TestFailUnretryableRetry_WritesNoTargetRow proves the
// orphaned-retry terminal path leaves no target row — and so no
// plaintext target config — in the per-webhook event database.
//
// That path loads the delivery without its Target relation on
// purpose. Populating d.Target makes GORM's SaveBeforeAssociations
// upsert the whole target row on the status UPDATE, which for a slack
// target writes the incoming-webhook credential into events-*.db.
// See https://git.eeqj.de/sneak/webhooker/issues/206.
func TestFailUnretryableRetry_WritesNoTargetRow(
t *testing.T,
) {
t.Parallel()
s := newISetup(t)
iCreateWebhook(
t, s.MainDB, s.WebhookID, "no-target-row",
)
targetID := uuid.New().String()
// A Slack incoming-webhook URL: the target config IS the
// credential, which is what makes a leaked target row a
// disclosure rather than a curiosity.
hookURL := "https://hooks.slack.com/services/T00/B00/x"
iCreateTarget(t, s.MainDB, targetID,
s.WebhookID, "credential-bearing",
database.TargetTypeLog, iHTTPConfig(hookURL), 5,
)
event := iSeedEvent(
t, s.WebhookDB, s.WebhookID, `{"orphaned":"retry"}`,
)
d := iSeedDelivery(
t, s.WebhookDB, event.ID, targetID,
database.DeliveryStatusRetrying,
)
iSeedFailedResult(t, s.WebhookDB, d.ID)
s.Engine.ExportSweepWebhookRetries(
context.Background(), s.WebhookID,
)
iAssertStatus(t, s.WebhookDB, d.ID,
database.DeliveryStatusFailed,
)
// The table exists in the per-webhook database because GORM
// migrates the Delivery relation's model alongside it. It must
// stay empty.
var targetRows int64
require.NoError(t, s.WebhookDB.
Table("targets").
Count(&targetRows).Error)
assert.Zero(t, targetRows,
"orphaned-retry terminal failure wrote a target row "+
"into the per-webhook event database",
)
var configs []string
require.NoError(t, s.WebhookDB.
Table("targets").
Pluck("config", &configs).Error)
assert.NotContains(
t, strings.Join(configs, " "), hookURL,
)
}
func TestRecoverSingleRetry_UnknownTargetType(
t *testing.T,
) {
t.Parallel()
s := newISetup(t)
iCreateWebhook(
t, s.MainDB, s.WebhookID, "unknown-type",
)
unknown := database.TargetType("not-a-target-type")
deliveryID := iSeedRetryingWithType(t, s, unknown)
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID,
)
iAssertTerminallyFailed(t, s, deliveryID, unknown)
}
func TestSweepSingleRetry_UnknownTargetType(
t *testing.T,
) {
t.Parallel()
s := newISetup(t)
iCreateWebhook(
t, s.MainDB, s.WebhookID, "unknown-type-sweep",
)
unknown := database.TargetType("not-a-target-type")
deliveryID := iSeedRetryingWithType(t, s, unknown)
s.Engine.ExportSweepWebhookRetries(
context.Background(), s.WebhookID,
)
iAssertTerminallyFailed(t, s, deliveryID, unknown)
}
// iSeedFailedResult creates a failed delivery result.
func iSeedFailedResult(
t *testing.T,
db *gorm.DB,
deliveryID string,
) {
t.Helper()
result := database.DeliveryResult{
DeliveryID: deliveryID,
AttemptNum: 1,
Success: false,
StatusCode: 500,
Error: "server error",
}
require.NoError(t, db.Create(&result).Error)
}
// iCreateWebhook creates a webhook record in main DB.
func iCreateWebhook(
t *testing.T,
mainDB *gorm.DB,
webhookID, name string,
) {
t.Helper()
webhook := database.Webhook{
UserID: uuid.New().String(),
Name: name,
}
webhook.ID = webhookID
require.NoError(t, mainDB.Create(&webhook).Error)
}
// --- recoverInFlight Tests ---
func TestRecoverInFlight_NoWebhooks(t *testing.T) {
t.Parallel()
s := newISetup(t)
s.Engine.ExportRecoverInFlight(
context.Background(),
)
}
func TestRecoverInFlight_WithPendingDeliveries(
t *testing.T,
) {
t.Parallel()
s := newISetup(t)
targetID := uuid.New().String()
iCreateWebhook(
t, s.MainDB, s.WebhookID, "recover-test",
)
iCreateTarget(t, s.MainDB, targetID,
s.WebhookID, "recover-target",
database.TargetTypeLog, "", 0,
)
event := iSeedEvent(
t, s.WebhookDB, s.WebhookID,
`{"recover":"inflight"}`,
)
d := iSeedDelivery(
t, s.WebhookDB, event.ID, targetID,
database.DeliveryStatusPending,
)
s.Engine.ExportRecoverInFlight(
context.Background(),
)
select {
case task := <-s.Engine.ExportDeliveryCh():
assert.Equal(t, d.ID, task.DeliveryID)
assert.Equal(t,
database.TargetTypeLog, task.TargetType,
)
case <-time.After(2 * time.Second):
t.Fatal("expected task from recoverInFlight")
}
}
// --- HTTP Config with custom headers ---
func TestDeliverHTTP_CustomTargetHeaders(t *testing.T) {
t.Parallel()
s := newISetup(t)
var receivedAuth string
ts := httptest.NewServer(
http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
receivedAuth = r.Header.Get(
"Authorization",
)
w.WriteHeader(http.StatusOK)
},
),
)
defer ts.Close()
cfg := delivery.HTTPTargetConfig{
URL: ts.URL,
Headers: map[string]string{
"Authorization": "Bearer secret-token",
},
}
cfgJSON, err := json.Marshal(cfg)
require.NoError(t, err)
event := iSeedEvent(
t, s.WebhookDB, s.WebhookID,
`{"auth":"test"}`,
)
targetID := uuid.New().String()
d := iSeedDelivery(
t, s.WebhookDB, event.ID, targetID,
database.DeliveryStatusPending,
)
bodyStr := event.Body
task := iTask(
d, event, s.WebhookID, targetID,
"auth-target", string(cfgJSON),
0, 1, &bodyStr,
)
s.Engine.ExportProcessNewTask(
context.TODO(), &task,
)
assert.Equal(t,
"Bearer secret-token", receivedAuth,
)
}
// --- HTTP delivery with custom timeout ---
func TestDeliverHTTP_TargetTimeout(t *testing.T) {
t.Parallel()
db := testWebhookDB(t)
e := iEngine(t, 1)
ts := httptest.NewServer(
http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
time.Sleep(2 * time.Second)
w.WriteHeader(http.StatusOK)
},
),
)
defer ts.Close()
cfg := delivery.HTTPTargetConfig{
URL: ts.URL,
Timeout: 1,
}
cfgJSON, err := json.Marshal(cfg)
require.NoError(t, err)
event, del := iSeedEventAndDelivery(
t, db, `{"timeout":"test"}`,
string(cfgJSON),
)
task, d := iHTTPTaskAndDelivery(
event, del, "timeout-target",
string(cfgJSON), 0, 1,
)
e.ExportDeliverHTTP(context.TODO(), db, d, task)
iAssertStatus(t, db, del.ID,
database.DeliveryStatusFailed,
)
iAssertResultFailed(t, db, del.ID)
}
// iSeedEventAndDelivery creates event + delivery
// for standalone tests.
func iSeedEventAndDelivery(
t *testing.T,
db *gorm.DB,
body, _ string,
) (database.Event, database.Delivery) {
t.Helper()
event := database.Event{
WebhookID: uuid.New().String(),
EntrypointID: uuid.New().String(),
Method: http.MethodPost,
Headers: `{"Content-Type":["application/json"]}`,
Body: body,
ContentType: testContentType,
}
require.NoError(t, db.Create(&event).Error)
d := database.Delivery{
EventID: event.ID,
TargetID: uuid.New().String(),
Status: database.DeliveryStatusPending,
}
require.NoError(t, db.Create(&d).Error)
return event, d
}
// iHTTPTaskAndDelivery builds a task/delivery pair for
// standalone HTTP tests.
func iHTTPTaskAndDelivery(
event database.Event,
del database.Delivery,
name, config string,
maxRetries, attemptNum int,
) (*delivery.Task, *database.Delivery) {
task := &delivery.Task{
DeliveryID: del.ID,
EventID: event.ID,
WebhookID: event.WebhookID,
TargetID: del.TargetID,
TargetName: name,
TargetType: database.TargetTypeHTTP,
TargetConfig: config,
MaxRetries: maxRetries,
AttemptNum: attemptNum,
}
d := &database.Delivery{
EventID: event.ID,
TargetID: del.TargetID,
Status: database.DeliveryStatusPending,
Event: event,
Target: database.Target{
Name: name,
Type: database.TargetTypeHTTP,
Config: config,
},
}
d.ID = del.ID
return task, d
}
// iAssertResultFailed checks that a failed delivery
// result exists.
func iAssertResultFailed(
t *testing.T,
db *gorm.DB,
deliveryID string,
) {
t.Helper()
var result database.DeliveryResult
require.NoError(t, db.Where(
"delivery_id = ?", deliveryID,
).First(&result).Error)
assert.False(t, result.Success)
assert.NotEmpty(t, result.Error)
}
// --- HTTP request with invalid config ---
func TestDeliverHTTP_InvalidConfig(t *testing.T) {
t.Parallel()
db := testWebhookDB(t)
e := iEngine(t, 1)
event, del := iSeedEventAndDelivery(
t, db, `{"config":"invalid"}`, "",
)
task, d := iHTTPTaskAndDelivery(
event, del, "bad-config", `not-json`, 0, 1,
)
e.ExportDeliverHTTP(context.TODO(), db, d, task)
iAssertStatus(t, db, del.ID,
database.DeliveryStatusFailed,
)
}
// --- Notify batching ---
func TestNotify_MultipleTasks(t *testing.T) {
t.Parallel()
e := iEngine(t, 1)
tasks := make([]delivery.Task, 5)
for i := range tasks {
tasks[i] = delivery.Task{
DeliveryID: fmt.Sprintf("task-%d", i),
}
}
e.Notify(tasks)
for i := range 5 {
select {
case task := <-e.ExportDeliveryCh():
assert.Equal(t,
fmt.Sprintf("task-%d", i),
task.DeliveryID,
)
case <-time.After(time.Second):
t.Fatalf("expected task %d", i)
}
}
}