sneak/webhooker
1
0
Fork 0
Code Issues 13 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Compare commits

base: sneak:641a8ebd66f9083bae4564124ec835a81c298733
Branches Tags
sneak:main sneak:feat/receiver-rate-limit sneak:issue-64-receiver-rate-limit sneak:issue-70-log-target
..
compare: sneak:issue-70-log-target
Branches Tags
sneak:main sneak:feat/receiver-rate-limit sneak:issue-64-receiver-rate-limit sneak:issue-70-log-target

1 Commits
641a8ebd66 ... issue-70-l

Author SHA1 Message Date
sneak
ed81db137e Implement the log delivery target (closes #70)
All checks were successful
check / check (push) Successful in 4s
Details
2026-08-07 19:57:16 +07:00
35 changed files with 1184 additions and 2305 deletions
Expand all files Collapse all files

8
.golangci.yml
View File

@@ -1,9 +1,5 @@
version: "2" version: "2"
# Config schema uses the golangci-lint v2 layout (settings live under
# linters.settings, not top-level linters-settings) so that the
# thresholds below are actually applied by golangci-lint >= v2.
run: run:
timeout: 5m timeout: 5m
modules-download-mode: readonly modules-download-mode: readonly
@@ -18,7 +14,8 @@ linters:
- wsl # Deprecated, replaced by wsl_v5 - wsl # Deprecated, replaced by wsl_v5
- wrapcheck # Too verbose for internal packages - wrapcheck # Too verbose for internal packages
- varnamelen # Short names like db, id are idiomatic Go - varnamelen # Short names like db, id are idiomatic Go
settings:
linters-settings:
lll: lll:
line-length: 88 line-length: 88
funlen: funlen:
@@ -30,5 +27,6 @@ linters:
threshold: 100 threshold: 100
issues: issues:
exclude-use-default: false
max-issues-per-linter: 0 max-issues-per-linter: 0
max-same-issues: 0 max-same-issues: 0

4
Dockerfile
View File

@@ -1,8 +1,8 @@
# Lint stage # Lint stage
# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-07 # golangci/golangci-lint:v2.11.3 (Debian-based), 2026-03-17
# Using Debian-based image because mattn/go-sqlite3 (CGO) does not # Using Debian-based image because mattn/go-sqlite3 (CGO) does not
# compile on Alpine musl (off64_t is a glibc type). # compile on Alpine musl (off64_t is a glibc type).
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint FROM golangci/golangci-lint:v2.11.3@sha256:e838e8ab68aaefe83e2408691510867ade9329c0e0b895a3fb35eb93d1c2a4ba AS lint
RUN apt-get update && apt-get install -y --no-install-recommends make && rm -rf /var/lib/apt/lists/* RUN apt-get update && apt-get install -y --no-install-recommends make && rm -rf /var/lib/apt/lists/*

5
TODO.md
View File

@@ -28,11 +28,6 @@ databases currently grow without bound.
# Completed Steps # Completed Steps
- 2026-08-07 Update golangci-lint to v2.12.2 (Docker image digest in
`Dockerfile`, release-archive sha256 pins in `script/bootstrap`),
adopt the canonical `.golangci.yml` (v2 `linters.settings` layout so
`lll`/`funlen`/`cyclop`/`dupl` thresholds actually apply), and fix
all newly surfaced lint findings
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, - 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints,
Makefile shims, README Entrypoints section Makefile shims, README Entrypoints section
- 2026-03-25 pin golangci-lint Docker image for linting (#55) - 2026-03-25 pin golangci-lint Docker image for linting (#55)

10
cmd/webhooker/main.go
View File

@@ -34,7 +34,6 @@ func main() {
config.New, config.New,
database.New, database.New,
database.NewWebhookDBManager, database.NewWebhookDBManager,
database.NewRetentionReaper,
healthcheck.New, healthcheck.New,
session.New, session.New,
handlers.New, handlers.New,
@@ -45,13 +44,6 @@ func main() {
func(e *delivery.Engine) delivery.Notifier { return e }, func(e *delivery.Engine) delivery.Notifier { return e },
server.New, server.New,
), ),
fx.Invoke( fx.Invoke(func(*server.Server, *delivery.Engine) {}),
func(
*server.Server,
*delivery.Engine,
*database.RetentionReaper,
) {
},
),
).Run() ).Run()
} }

46
internal/config/config.go
View File

@@ -8,7 +8,6 @@ import (
"os" "os"
"strconv" "strconv"
"strings" "strings"
"time"
"go.uber.org/fx" "go.uber.org/fx"
"sneak.berlin/go/webhooker/internal/globals" "sneak.berlin/go/webhooker/internal/globals"
@@ -27,10 +26,6 @@ const (
// defaultPort is the default HTTP listen port. // defaultPort is the default HTTP listen port.
defaultPort = 8080 defaultPort = 8080
// defaultRetentionSweepInterval is how often the retention
// reaper deletes events older than each webhook's RetentionDays.
defaultRetentionSweepInterval = time.Hour
) )
// ErrInvalidEnvironment is returned when WEBHOOKER_ENVIRONMENT // ErrInvalidEnvironment is returned when WEBHOOKER_ENVIRONMENT
@@ -56,10 +51,6 @@ type Config struct {
MetricsUsername string MetricsUsername string
Port int Port int
SentryDSN string SentryDSN string
// RetentionSweepInterval is how often the retention reaper runs.
RetentionSweepInterval time.Duration
params *ConfigParams params *ConfigParams
log *slog.Logger log *slog.Logger
} }
@@ -104,30 +95,6 @@ func envInt(key string, defaultValue int) int {
return defaultValue return defaultValue
} }
// envDuration returns the value of the named environment variable
// parsed as a Go duration (e.g. "1h", "30m"). Returns defaultValue if
// not set. If the variable is set but cannot be parsed, it returns a
// wrapped error naming the key and the bad value, so startup fails
// loudly rather than silently falling back to the default.
func envDuration(
key string,
defaultValue time.Duration,
) (time.Duration, error) {
v := os.Getenv(key)
if v == "" {
return defaultValue, nil
}
d, err := time.ParseDuration(v)
if err != nil {
return 0, fmt.Errorf(
"invalid duration for %s: %q: %w", key, v, err,
)
}
return d, nil
}
// New creates a Config by reading environment variables. // New creates a Config by reading environment variables.
// //
//nolint:revive // lc parameter is required by fx even if unused. //nolint:revive // lc parameter is required by fx even if unused.
@@ -151,17 +118,6 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) {
) )
} }
// Parse the retention sweep interval; a set-but-unparseable value
// is a hard error so fx aborts startup rather than silently using
// the default.
retentionSweepInterval, err := envDuration(
"RETENTION_SWEEP_INTERVAL",
defaultRetentionSweepInterval,
)
if err != nil {
return nil, err
}
// Load configuration values from environment variables // Load configuration values from environment variables
s := &Config{ s := &Config{
DataDir: envString("DATA_DIR"), DataDir: envString("DATA_DIR"),
@@ -172,7 +128,6 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) {
MetricsPassword: envString("METRICS_PASSWORD"), MetricsPassword: envString("METRICS_PASSWORD"),
Port: envInt("PORT", defaultPort), Port: envInt("PORT", defaultPort),
SentryDSN: envString("SENTRY_DSN"), SentryDSN: envString("SENTRY_DSN"),
RetentionSweepInterval: retentionSweepInterval,
log: log, log: log,
params: &params, params: &params,
} }
@@ -196,7 +151,6 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) {
"debug", s.Debug, "debug", s.Debug,
"maintenanceMode", s.MaintenanceMode, "maintenanceMode", s.MaintenanceMode,
"dataDir", s.DataDir, "dataDir", s.DataDir,
"retentionSweepInterval", s.RetentionSweepInterval.String(),
"hasSentryDSN", s.SentryDSN != "", "hasSentryDSN", s.SentryDSN != "",
"hasMetricsAuth", "hasMetricsAuth",
s.MetricsUsername != "" && s.MetricsPassword != "", s.MetricsUsername != "" && s.MetricsPassword != "",

95
internal/config/config_test.go
View File

@@ -3,7 +3,6 @@ package config_test
import ( import (
"os" "os"
"testing" "testing"
"time"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@@ -121,100 +120,6 @@ func testEnvironmentConfigSuccess(
assert.Equal(t, isProd, cfg.IsProd()) assert.Equal(t, isProd, cfg.IsProd())
} }
func TestRetentionSweepInterval(t *testing.T) {
tests := []struct {
name string
set bool
value string
expectError bool
expected time.Duration
}{
{
name: "unset uses default",
set: false,
expected: time.Hour,
},
{
name: "valid value is parsed",
set: true,
value: "15m",
expected: 15 * time.Minute,
},
{
name: "unparseable value fails startup",
set: true,
value: "not-a-duration",
expectError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Cannot use t.Parallel() here because t.Setenv
// is incompatible with parallel subtests.
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
if tt.set {
t.Setenv("RETENTION_SWEEP_INTERVAL", tt.value)
} else {
require.NoError(t, os.Unsetenv(
"RETENTION_SWEEP_INTERVAL",
))
}
if tt.expectError {
testRetentionSweepIntervalError(t)
} else {
testRetentionSweepIntervalSuccess(t, tt.expected)
}
})
}
}
func testRetentionSweepIntervalError(t *testing.T) {
t.Helper()
var cfg *config.Config
app := fx.New(
fx.NopLogger,
fx.Provide(
globals.New,
logger.New,
config.New,
),
fx.Populate(&cfg),
)
assert.Error(t, app.Err())
}
func testRetentionSweepIntervalSuccess(
t *testing.T,
expected time.Duration,
) {
t.Helper()
var cfg *config.Config
app := fxtest.New(
t,
fx.Provide(
globals.New,
logger.New,
config.New,
),
fx.Populate(&cfg),
)
require.NoError(t, app.Err())
app.RequireStart()
defer app.RequireStop()
assert.Equal(t, expected, cfg.RetentionSweepInterval)
}
func TestDefaultDataDir(t *testing.T) { func TestDefaultDataDir(t *testing.T) {
for _, env := range []string{"", "dev", "prod"} { for _, env := range []string{"", "dev", "prod"} {
name := env name := env

13
internal/database/database_test.go
View File

@@ -11,15 +11,6 @@ import (
"sneak.berlin/go/webhooker/internal/logger" "sneak.berlin/go/webhooker/internal/logger"
) )
const (
// testAppname is the Globals.Appname used in tests.
testAppname = "webhooker-test"
// testVersion is the Globals.Version used in tests.
testVersion = "test"
// testContentType is the event content type used in tests.
testContentType = "application/json"
)
func setupTestDB( func setupTestDB(
t *testing.T, t *testing.T,
) (*database.Database, *fxtest.Lifecycle) { ) (*database.Database, *fxtest.Lifecycle) {
@@ -28,8 +19,8 @@ func setupTestDB(
lc := fxtest.NewLifecycle(t) lc := fxtest.NewLifecycle(t)
g := &globals.Globals{ g := &globals.Globals{
Appname: testAppname, Appname: "webhooker-test",
Version: testVersion, Version: "test",
} }
l, err := logger.New( l, err := logger.New(

31
internal/database/export_test.go
View File

@@ -1,31 +0,0 @@
package database
import (
"context"
"log/slog"
"os"
"time"
)
// NewTestRetentionReaper builds a RetentionReaper backed by the given
// main database and per-webhook database manager, without the fx
// lifecycle. Intended for tests.
func NewTestRetentionReaper(
db *Database,
mgr *WebhookDBManager,
) *RetentionReaper {
return &RetentionReaper{
db: db,
dbManager: mgr,
log: slog.New(slog.NewTextHandler(
os.Stderr,
&slog.HandlerOptions{Level: slog.LevelDebug},
)),
interval: time.Hour,
}
}
// ExportSweep runs a single retention sweep synchronously for tests.
func (r *RetentionReaper) ExportSweep(ctx context.Context) {
r.sweep(ctx)
}

5
internal/database/model_entrypoint.go
View File

@@ -5,10 +5,7 @@ type Entrypoint struct {
BaseModel BaseModel
WebhookID string `gorm:"type:uuid;not null" json:"webhookId"` WebhookID string `gorm:"type:uuid;not null" json:"webhookId"`
Path string `gorm:"uniqueIndex;not null" json:"path"` // URL path for this entrypoint
// Path is the URL path for this entrypoint.
Path string `gorm:"uniqueIndex;not null" json:"path"`
Description string `json:"description"` Description string `json:"description"`
Active bool `gorm:"default:true" json:"active"` Active bool `gorm:"default:true" json:"active"`

3
internal/database/model_target.go
View File

@@ -23,8 +23,7 @@ type Target struct {
// Configuration fields (JSON stored based on type) // Configuration fields (JSON stored based on type)
Config string `gorm:"type:text" json:"config"` // JSON configuration Config string `gorm:"type:text" json:"config"` // JSON configuration
// For HTTP targets (max_retries=0 means fire-and-forget, // For HTTP targets (max_retries=0 means fire-and-forget, >0 enables retries with backoff)
// >0 enables retries with backoff)
MaxRetries int `json:"maxRetries,omitempty"` MaxRetries int `json:"maxRetries,omitempty"`
MaxQueueSize int `json:"maxQueueSize,omitempty"` MaxQueueSize int `json:"maxQueueSize,omitempty"`

4
internal/database/model_webhook.go
View File

@@ -7,9 +7,7 @@ type Webhook struct {
UserID string `gorm:"type:uuid;not null" json:"userId"` UserID string `gorm:"type:uuid;not null" json:"userId"`
Name string `gorm:"not null" json:"name"` Name string `gorm:"not null" json:"name"`
Description string `json:"description"` Description string `json:"description"`
RetentionDays int `gorm:"default:30" json:"retentionDays"` // Days to retain events
// RetentionDays is the number of days to retain events.
RetentionDays int `gorm:"default:30" json:"retentionDays"`
// Relations // Relations
User User `json:"user,omitzero"` User User `json:"user,omitzero"`

252
internal/database/retention.go
View File

@@ -1,252 +0,0 @@
package database
import (
"context"
"fmt"
"log/slog"
"sync"
"time"
"go.uber.org/fx"
"gorm.io/gorm"
"sneak.berlin/go/webhooker/internal/config"
"sneak.berlin/go/webhooker/internal/logger"
)
// hoursPerDay converts a RetentionDays count into hours for cutoff
// computation.
const hoursPerDay = 24
// RetentionReaperParams holds the fx dependencies for the
// RetentionReaper.
type RetentionReaperParams struct {
fx.In
Config *config.Config
Database *Database
DBManager *WebhookDBManager
Logger *logger.Logger
}
// RetentionReaper periodically deletes expired events (and their
// dependent deliveries and delivery results) from each per-webhook
// database, enforcing every webhook's RetentionDays. Rows are removed
// permanently so that per-webhook SQLite files do not grow without
// bound.
type RetentionReaper struct {
db *Database
dbManager *WebhookDBManager
log *slog.Logger
interval time.Duration
cancel context.CancelFunc
wg sync.WaitGroup
}
// NewRetentionReaper creates the retention reaper and registers its
// fx lifecycle hooks. The background sweep loop starts on OnStart and
// stops cleanly on OnStop via context cancellation.
func NewRetentionReaper(
lc fx.Lifecycle,
params RetentionReaperParams,
) *RetentionReaper {
r := &RetentionReaper{
db: params.Database,
dbManager: params.DBManager,
log: params.Logger.Get(),
interval: params.Config.RetentionSweepInterval,
}
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
r.start(ctx)
return nil
},
OnStop: func(_ context.Context) error {
r.stop()
return nil
},
})
return r
}
func (r *RetentionReaper) start(ctx context.Context) {
ctx, cancel := context.WithCancel(ctx)
r.cancel = cancel
r.wg.Add(1)
go r.run(ctx)
r.log.Info(
"retention reaper started",
"interval", r.interval.String(),
)
}
func (r *RetentionReaper) stop() {
r.log.Info("retention reaper stopping")
if r.cancel != nil {
r.cancel()
}
r.wg.Wait()
r.log.Info("retention reaper stopped")
}
func (r *RetentionReaper) run(ctx context.Context) {
defer r.wg.Done()
ticker := time.NewTicker(r.interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
r.sweep(ctx)
}
}
}
// sweep lists every webhook from the main database and reaps expired
// rows from each per-webhook database whose RetentionDays is positive.
func (r *RetentionReaper) sweep(ctx context.Context) {
var webhooks []Webhook
err := r.db.DB().
Model(&Webhook{}).
Find(&webhooks).Error
if err != nil {
r.log.Error(
"retention sweep: failed to list webhooks",
"error", err,
)
return
}
for i := range webhooks {
select {
case <-ctx.Done():
return
default:
}
wh := webhooks[i]
// RetentionDays of zero or less means retain forever.
if wh.RetentionDays <= 0 {
continue
}
// Nothing to reap if the per-webhook database has never
// been created.
if !r.dbManager.DBExists(wh.ID) {
continue
}
r.reapWebhook(wh.ID, wh.RetentionDays)
}
}
// reapWebhook removes every expired event (and its dependents) from a
// single webhook's database.
func (r *RetentionReaper) reapWebhook(
webhookID string,
retentionDays int,
) {
db, err := r.dbManager.GetDB(webhookID)
if err != nil {
r.log.Error(
"retention sweep: failed to open webhook database",
"webhook_id", webhookID,
"error", err,
)
return
}
cutoff := time.Now().Add(
-time.Duration(retentionDays*hoursPerDay) * time.Hour,
)
deleted, err := reapExpired(db, cutoff)
if err != nil {
r.log.Error(
"retention sweep: failed to reap expired events",
"webhook_id", webhookID,
"error", err,
)
return
}
if deleted > 0 {
r.log.Info(
"retention sweep: reaped expired events",
"webhook_id", webhookID,
"retention_days", retentionDays,
"events_deleted", deleted,
)
}
}
// reapExpired hard-deletes, in foreign-key-safe order, the delivery
// results, deliveries, and events associated with events older than
// cutoff. Deletes are unscoped so rows are physically removed rather
// than soft-deleted, reclaiming disk. It returns the number of events
// deleted.
func reapExpired(db *gorm.DB, cutoff time.Time) (int64, error) {
// Fresh subqueries are built per statement to avoid reusing a
// mutated builder across executions.
expiredEventIDs := func() *gorm.DB {
return db.Model(&Event{}).
Select("id").
Where("created_at < ?", cutoff)
}
expiredDeliveryIDs := func() *gorm.DB {
return db.Model(&Delivery{}).
Select("id").
Where("event_id IN (?)", expiredEventIDs())
}
// 1. Delivery results whose delivery belongs to an expired event.
res := db.Unscoped().
Where("delivery_id IN (?)", expiredDeliveryIDs()).
Delete(&DeliveryResult{})
if res.Error != nil {
return 0, fmt.Errorf(
"deleting expired delivery results: %w",
res.Error,
)
}
// 2. Deliveries belonging to an expired event.
del := db.Unscoped().
Where("event_id IN (?)", expiredEventIDs()).
Delete(&Delivery{})
if del.Error != nil {
return 0, fmt.Errorf(
"deleting expired deliveries: %w",
del.Error,
)
}
// 3. The expired events themselves.
ev := db.Unscoped().
Where("created_at < ?", cutoff).
Delete(&Event{})
if ev.Error != nil {
return 0, fmt.Errorf(
"deleting expired events: %w",
ev.Error,
)
}
return ev.RowsAffected, nil
}

278
internal/database/retention_test.go
View File

@@ -1,278 +0,0 @@
package database_test
import (
"context"
"net/http"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/fx/fxtest"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"sneak.berlin/go/webhooker/internal/config"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/globals"
"sneak.berlin/go/webhooker/internal/logger"
)
// retentionTestEnv bundles the pieces a retention test drives.
type retentionTestEnv struct {
reaper *database.RetentionReaper
mainDB *database.Database
mgr *database.WebhookDBManager
}
func setupRetentionTest(t *testing.T) *retentionTestEnv {
t.Helper()
lc := fxtest.NewLifecycle(t)
g := &globals.Globals{
Appname: testAppname,
Version: testVersion,
}
l, err := logger.New(lc, logger.LoggerParams{Globals: g})
require.NoError(t, err)
cfg := &config.Config{
DataDir: t.TempDir(),
Environment: "dev",
}
mainDB, err := database.New(lc, database.DatabaseParams{
Config: cfg,
Logger: l,
})
require.NoError(t, err)
mgr, err := database.NewWebhookDBManager(
lc,
database.WebhookDBManagerParams{Config: cfg, Logger: l},
)
require.NoError(t, err)
ctx := context.Background()
require.NoError(t, lc.Start(ctx))
t.Cleanup(func() { require.NoError(t, lc.Stop(ctx)) })
return &retentionTestEnv{
reaper: database.NewTestRetentionReaper(mainDB, mgr),
mainDB: mainDB,
mgr: mgr,
}
}
// createWebhook inserts a webhook row into the main database with the
// given retention policy and returns its ID.
func createWebhook(
t *testing.T,
db *gorm.DB,
retentionDays int,
) string {
t.Helper()
wh := &database.Webhook{
UserID: uuid.New().String(),
Name: "test-webhook",
RetentionDays: retentionDays,
}
require.NoError(
t,
db.Omit(clause.Associations).Create(wh).Error,
)
// The RetentionDays column carries a GORM default of 30, so a
// zero (or negative) value passed to Create is replaced by that
// default. Force the requested value explicitly so the
// retain-forever (<= 0) path can be exercised.
require.NoError(
t,
db.Model(wh).
Update("retention_days", retentionDays).Error,
)
return wh.ID
}
// eventChain is the set of row IDs seeded for a single event.
type eventChain struct {
eventID string
deliveryID string
resultID string
}
// seedEventChain creates an event with one delivery and one delivery
// result, all stamped with createdAt, and returns their IDs.
func seedEventChain(
t *testing.T,
db *gorm.DB,
webhookID string,
createdAt time.Time,
) eventChain {
t.Helper()
event := &database.Event{
WebhookID: webhookID,
EntrypointID: uuid.New().String(),
Method: http.MethodPost,
Body: `{"seed": true}`,
ContentType: testContentType,
}
event.CreatedAt = createdAt
require.NoError(t, db.Create(event).Error)
delivery := &database.Delivery{
EventID: event.ID,
TargetID: uuid.New().String(),
Status: database.DeliveryStatusDelivered,
}
delivery.CreatedAt = createdAt
require.NoError(t, db.Create(delivery).Error)
result := &database.DeliveryResult{
DeliveryID: delivery.ID,
AttemptNum: 1,
Success: true,
StatusCode: 200,
Duration: 10,
}
result.CreatedAt = createdAt
require.NoError(t, db.Create(result).Error)
return eventChain{
eventID: event.ID,
deliveryID: delivery.ID,
resultID: result.ID,
}
}
// countByID returns how many rows of model match the given id,
// counting even hard-deletable rows via Unscoped.
func countByID(
t *testing.T,
db *gorm.DB,
model any,
id string,
) int64 {
t.Helper()
var n int64
require.NoError(
t,
db.Unscoped().Model(model).
Where("id = ?", id).Count(&n).Error,
)
return n
}
func assertChainGone(
t *testing.T,
db *gorm.DB,
chain eventChain,
) {
t.Helper()
assert.Zero(
t,
countByID(t, db, &database.Event{}, chain.eventID),
"expired event should be removed",
)
assert.Zero(
t,
countByID(t, db, &database.Delivery{}, chain.deliveryID),
"expired delivery should be removed",
)
assert.Zero(
t,
countByID(
t, db, &database.DeliveryResult{}, chain.resultID,
),
"expired delivery result should be removed",
)
}
func assertChainPresent(
t *testing.T,
db *gorm.DB,
chain eventChain,
) {
t.Helper()
assert.Equal(
t,
int64(1),
countByID(t, db, &database.Event{}, chain.eventID),
"recent event should be retained",
)
assert.Equal(
t,
int64(1),
countByID(t, db, &database.Delivery{}, chain.deliveryID),
"recent delivery should be retained",
)
assert.Equal(
t,
int64(1),
countByID(
t, db, &database.DeliveryResult{}, chain.resultID,
),
"recent delivery result should be retained",
)
}
func TestRetentionReaper_ReapsExpiredKeepsRecent(t *testing.T) {
t.Parallel()
env := setupRetentionTest(t)
const retentionDays = 30
webhookID := createWebhook(
t, env.mainDB.DB(), retentionDays,
)
db, err := env.mgr.GetDB(webhookID)
require.NoError(t, err)
now := time.Now()
old := seedEventChain(
t, db, webhookID,
now.Add(-40*24*time.Hour),
)
recent := seedEventChain(
t, db, webhookID,
now.Add(-1*24*time.Hour),
)
env.reaper.ExportSweep(context.Background())
assertChainGone(t, db, old)
assertChainPresent(t, db, recent)
}
func TestRetentionReaper_RetainsForeverWhenNonPositive(t *testing.T) {
t.Parallel()
env := setupRetentionTest(t)
// RetentionDays of zero means retain forever.
webhookID := createWebhook(t, env.mainDB.DB(), 0)
db, err := env.mgr.GetDB(webhookID)
require.NoError(t, err)
ancient := seedEventChain(
t, db, webhookID,
time.Now().Add(-365*24*time.Hour),
)
env.reaper.ExportSweep(context.Background())
assertChainPresent(t, db, ancient)
}

10
internal/database/testing.go
View File

@@ -14,10 +14,7 @@ import (
func NewTestDatabase(db *gorm.DB) *Database { func NewTestDatabase(db *gorm.DB) *Database {
return &Database{ return &Database{
db: db, db: db,
log: slog.New(slog.NewTextHandler( log: slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})),
os.Stderr,
&slog.HandlerOptions{Level: slog.LevelDebug},
)),
} }
} }
@@ -26,9 +23,6 @@ func NewTestDatabase(db *gorm.DB) *Database {
func NewTestWebhookDBManager(dataDir string) *WebhookDBManager { func NewTestWebhookDBManager(dataDir string) *WebhookDBManager {
return &WebhookDBManager{ return &WebhookDBManager{
dataDir: dataDir, dataDir: dataDir,
log: slog.New(slog.NewTextHandler( log: slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})),
os.Stderr,
&slog.HandlerOptions{Level: slog.LevelDebug},
)),
} }
} }

29
internal/database/webhook_db_manager_test.go
View File

@@ -2,7 +2,6 @@ package database_test
import ( import (
"context" "context"
"net/http"
"os" "os"
"path/filepath" "path/filepath"
"testing" "testing"
@@ -26,8 +25,8 @@ func setupTestWebhookDBManager(
lc := fxtest.NewLifecycle(t) lc := fxtest.NewLifecycle(t)
g := &globals.Globals{ g := &globals.Globals{
Appname: testAppname, Appname: "webhooker-test",
Version: testVersion, Version: "test",
} }
l, err := logger.New( l, err := logger.New(
@@ -84,10 +83,10 @@ func TestWebhookDBManager_CreateAndGetDB(t *testing.T) {
event := &database.Event{ event := &database.Event{
WebhookID: webhookID, WebhookID: webhookID,
EntrypointID: uuid.New().String(), EntrypointID: uuid.New().String(),
Method: http.MethodPost, Method: "POST",
Headers: `{"Content-Type":["application/json"]}`, Headers: `{"Content-Type":["application/json"]}`,
Body: `{"test": true}`, Body: `{"test": true}`,
ContentType: testContentType, ContentType: "application/json",
} }
require.NoError(t, db.Create(event).Error) require.NoError(t, db.Create(event).Error)
assert.NotEmpty(t, event.ID) assert.NotEmpty(t, event.ID)
@@ -100,7 +99,7 @@ func TestWebhookDBManager_CreateAndGetDB(t *testing.T) {
db.First(&readEvent, "id = ?", event.ID).Error, db.First(&readEvent, "id = ?", event.ID).Error,
) )
assert.Equal(t, webhookID, readEvent.WebhookID) assert.Equal(t, webhookID, readEvent.WebhookID)
assert.Equal(t, http.MethodPost, readEvent.Method) assert.Equal(t, "POST", readEvent.Method)
assert.Equal(t, `{"test": true}`, readEvent.Body) assert.Equal(t, `{"test": true}`, readEvent.Body)
} }
@@ -124,9 +123,9 @@ func TestWebhookDBManager_DeleteDB(t *testing.T) {
event := &database.Event{ event := &database.Event{
WebhookID: webhookID, WebhookID: webhookID,
EntrypointID: uuid.New().String(), EntrypointID: uuid.New().String(),
Method: http.MethodPost, Method: "POST",
Body: `{"test": true}`, Body: `{"test": true}`,
ContentType: testContentType, ContentType: "application/json",
} }
require.NoError(t, db.Create(event).Error) require.NoError(t, db.Create(event).Error)
@@ -197,10 +196,10 @@ func seedDeliveryWorkflow(
event := &database.Event{ event := &database.Event{
WebhookID: webhookID, WebhookID: webhookID,
EntrypointID: uuid.New().String(), EntrypointID: uuid.New().String(),
Method: http.MethodPost, Method: "POST",
Headers: `{"Content-Type":["application/json"]}`, Headers: `{"Content-Type":["application/json"]}`,
Body: `{"payload": "test"}`, Body: `{"payload": "test"}`,
ContentType: testContentType, ContentType: "application/json",
} }
require.NoError(t, db.Create(event).Error) require.NoError(t, db.Create(event).Error)
@@ -232,7 +231,7 @@ func verifyPendingDeliveries(
) )
require.Len(t, pending, 1) require.Len(t, pending, 1)
assert.Equal(t, event.ID, pending[0].EventID) assert.Equal(t, event.ID, pending[0].EventID)
assert.Equal(t, http.MethodPost, pending[0].Event.Method) assert.Equal(t, "POST", pending[0].Event.Method)
} }
func completeDelivery( func completeDelivery(
@@ -304,16 +303,16 @@ func TestWebhookDBManager_MultipleWebhooks(t *testing.T) {
event1 := &database.Event{ event1 := &database.Event{
WebhookID: webhook1, WebhookID: webhook1,
EntrypointID: uuid.New().String(), EntrypointID: uuid.New().String(),
Method: http.MethodPost, Method: "POST",
Body: `{"webhook": 1}`, Body: `{"webhook": 1}`,
ContentType: testContentType, ContentType: "application/json",
} }
event2 := &database.Event{ event2 := &database.Event{
WebhookID: webhook2, WebhookID: webhook2,
EntrypointID: uuid.New().String(), EntrypointID: uuid.New().String(),
Method: http.MethodPut, Method: "PUT",
Body: `{"webhook": 2}`, Body: `{"webhook": 2}`,
ContentType: testContentType, ContentType: "application/json",
} }
require.NoError(t, db1.Create(event1).Error) require.NoError(t, db1.Create(event1).Error)

922
internal/delivery/engine.go
View File

File diff suppressed because it is too large Load Diff

42
internal/delivery/engine_integration_test.go
View File

@@ -126,6 +126,36 @@ func iHTTPConfig(url string) string {
return string(data) return string(data)
} }
func iWebhookDB(t *testing.T) *gorm.DB {
t.Helper()
dbPath := filepath.Join(
t.TempDir(), "events-test.db",
)
dsn := fmt.Sprintf(
"file:%s?cache=shared&mode=rwc", dbPath,
)
sqlDB, err := sql.Open("sqlite", dsn)
require.NoError(t, err)
t.Cleanup(func() { _ = sqlDB.Close() })
db, err := gorm.Open(
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
)
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(
&database.Event{},
&database.Delivery{},
&database.DeliveryResult{},
))
return db
}
func iEngine( func iEngine(
t *testing.T, workers int, t *testing.T, workers int,
) *delivery.Engine { ) *delivery.Engine {
@@ -152,10 +182,10 @@ func iSeedEvent(
event := database.Event{ event := database.Event{
WebhookID: webhookID, WebhookID: webhookID,
EntrypointID: uuid.New().String(), EntrypointID: uuid.New().String(),
Method: http.MethodPost, Method: "POST",
Headers: `{}`, Headers: `{}`,
Body: body, Body: body,
ContentType: testContentType, ContentType: "application/json",
} }
require.NoError(t, db.Create(&event).Error) require.NoError(t, db.Create(&event).Error)
@@ -905,7 +935,7 @@ func TestDeliverHTTP_CustomTargetHeaders(t *testing.T) {
func TestDeliverHTTP_TargetTimeout(t *testing.T) { func TestDeliverHTTP_TargetTimeout(t *testing.T) {
t.Parallel() t.Parallel()
db := testWebhookDB(t) db := iWebhookDB(t)
e := iEngine(t, 1) e := iEngine(t, 1)
ts := httptest.NewServer( ts := httptest.NewServer(
@@ -957,10 +987,10 @@ func iSeedEventAndDelivery(
event := database.Event{ event := database.Event{
WebhookID: uuid.New().String(), WebhookID: uuid.New().String(),
EntrypointID: uuid.New().String(), EntrypointID: uuid.New().String(),
Method: http.MethodPost, Method: "POST",
Headers: `{"Content-Type":["application/json"]}`, Headers: `{"Content-Type":["application/json"]}`,
Body: body, Body: body,
ContentType: testContentType, ContentType: "application/json",
} }
require.NoError(t, db.Create(&event).Error) require.NoError(t, db.Create(&event).Error)
@@ -1037,7 +1067,7 @@ func iAssertResultFailed(
func TestDeliverHTTP_InvalidConfig(t *testing.T) { func TestDeliverHTTP_InvalidConfig(t *testing.T) {
t.Parallel() t.Parallel()
db := testWebhookDB(t) db := iWebhookDB(t)
e := iEngine(t, 1) e := iEngine(t, 1)
event, del := iSeedEventAndDelivery( event, del := iSeedEventAndDelivery(

285
internal/delivery/engine_test.go
View File

@@ -27,9 +27,6 @@ import (
"sneak.berlin/go/webhooker/internal/delivery" "sneak.berlin/go/webhooker/internal/delivery"
) )
// testContentType is the event content type used in tests.
const testContentType = "application/json"
func testWebhookDB(t *testing.T) *gorm.DB { func testWebhookDB(t *testing.T) *gorm.DB {
t.Helper() t.Helper()
@@ -97,10 +94,10 @@ func seedEvent(
event := database.Event{ event := database.Event{
WebhookID: uuid.New().String(), WebhookID: uuid.New().String(),
EntrypointID: uuid.New().String(), EntrypointID: uuid.New().String(),
Method: http.MethodPost, Method: "POST",
Headers: `{"Content-Type":["application/json"]}`, Headers: `{"Content-Type":["application/json"]}`,
Body: body, Body: body,
ContentType: testContentType, ContentType: "application/json",
} }
require.NoError(t, db.Create(&event).Error) require.NoError(t, db.Create(&event).Error)
@@ -439,6 +436,91 @@ func TestDeliverLog_ImmediateSuccess(t *testing.T) {
assert.True(t, result.Success) assert.True(t, result.Success)
} }
func TestDeliverLog_StructuredLogFields(t *testing.T) {
t.Parallel()
db := testWebhookDB(t)
var logBuf bytes.Buffer
e := delivery.NewTestEngine(
slog.New(slog.NewTextHandler(
&logBuf,
&slog.HandlerOptions{Level: slog.LevelDebug},
)),
&http.Client{Timeout: 5 * time.Second},
1,
)
event := seedEvent(t, db, `{"log":"structured"}`)
dlv := seedDelivery(
t, db, event.ID, uuid.New().String(),
database.DeliveryStatusPending,
)
d := &database.Delivery{
EventID: event.ID,
TargetID: dlv.TargetID,
Status: database.DeliveryStatusPending,
Event: event,
Target: database.Target{
Name: "structured-log",
Type: database.TargetTypeLog,
},
}
d.ID = dlv.ID
e.ExportDeliverLog(db, d)
// The delivery is marked delivered and a success
// DeliveryResult with no HTTP status is recorded,
// mirroring the other target types' bookkeeping.
var updated database.Delivery
require.NoError(t, db.First(
&updated, "id = ?", dlv.ID,
).Error)
assert.Equal(t,
database.DeliveryStatusDelivered, updated.Status,
"log target should immediately succeed",
)
var result database.DeliveryResult
require.NoError(t, db.Where(
"delivery_id = ?", dlv.ID,
).First(&result).Error)
assert.True(t, result.Success)
assert.Equal(t, 0, result.StatusCode,
"log target should not have an HTTP status",
)
assertLogFields(t, logBuf.String(), event, "structured-log")
}
// assertLogFields checks that a log target's structured
// log line carries the required fields: event id,
// webhook/entrypoint, target name, and outcome.
func assertLogFields(
t *testing.T,
logged string,
event database.Event,
targetName string,
) {
t.Helper()
assert.Contains(t, logged, "event_id="+event.ID)
assert.Contains(t, logged, "webhook_id="+event.WebhookID)
assert.Contains(t,
logged, "entrypoint_id="+event.EntrypointID,
)
assert.Contains(t, logged, "target_name="+targetName)
assert.Contains(t, logged, "outcome=delivered")
}
func TestDeliverHTTP_WithRetries_Success(t *testing.T) { func TestDeliverHTTP_WithRetries_Success(t *testing.T) {
t.Parallel() t.Parallel()
@@ -1120,10 +1202,10 @@ func TestDoHTTPRequest_ForwardsHeaders(t *testing.T) {
} }
event := &database.Event{ event := &database.Event{
Method: http.MethodPost, Method: "POST",
Headers: `{"X-Custom":["value1"],"Content-Type":["application/json"]}`, Headers: `{"X-Custom":["value1"],"Content-Type":["application/json"]}`,
Body: `{"test":true}`, Body: `{"test":true}`,
ContentType: testContentType, ContentType: "application/json",
} }
statusCode, _, _, err := e.ExportDoHTTPRequest( statusCode, _, _, err := e.ExportDoHTTPRequest(
@@ -1145,7 +1227,7 @@ func TestDoHTTPRequest_ForwardsHeaders(t *testing.T) {
) )
assert.Equal(t, assert.Equal(t,
testContentType, "application/json",
receivedHeaders.Get("Content-Type"), receivedHeaders.Get("Content-Type"),
) )
@@ -1292,8 +1374,8 @@ func TestFormatSlackMessage_JSONBody(t *testing.T) {
t.Parallel() t.Parallel()
event := &database.Event{ event := &database.Event{
Method: http.MethodPost, Method: "POST",
ContentType: testContentType, ContentType: "application/json",
Body: `{"action":"push",` + Body: `{"action":"push",` +
`"repo":"test/repo",` + `"repo":"test/repo",` +
`"ref":"refs/heads/main"}`, `"ref":"refs/heads/main"}`,
@@ -1318,7 +1400,7 @@ func TestFormatSlackMessage_NonJSONBody(t *testing.T) {
t.Parallel() t.Parallel()
event := &database.Event{ event := &database.Event{
Method: http.MethodPost, Method: "POST",
ContentType: "text/plain", ContentType: "text/plain",
Body: "hello world plain text", Body: "hello world plain text",
} }
@@ -1341,8 +1423,8 @@ func TestFormatSlackMessage_EmptyBody(t *testing.T) {
t.Parallel() t.Parallel()
event := &database.Event{ event := &database.Event{
Method: http.MethodPost, Method: "POST",
ContentType: testContentType, ContentType: "application/json",
Body: "", Body: "",
} }
event.CreatedAt = time.Date( event.CreatedAt = time.Date(
@@ -1370,8 +1452,8 @@ func TestFormatSlackMessage_LargeJSONTruncated(
require.NoError(t, err) require.NoError(t, err)
event := &database.Event{ event := &database.Event{
Method: http.MethodPost, Method: "POST",
ContentType: testContentType, ContentType: "application/json",
Body: string(largeJSON), Body: string(largeJSON),
} }
event.CreatedAt = time.Date( event.CreatedAt = time.Date(
@@ -1656,179 +1738,6 @@ func TestProcessDelivery_RoutesToSlack(t *testing.T) {
) )
} }
// newLogCaptureEngine builds a test engine whose logger
// writes to the returned buffer, for inspecting log output.
func newLogCaptureEngine(
t *testing.T,
) (*delivery.Engine, *bytes.Buffer) {
t.Helper()
var buf bytes.Buffer
log := slog.New(slog.NewTextHandler(
&buf,
&slog.HandlerOptions{Level: slog.LevelDebug},
))
e := delivery.NewTestEngine(
log, &http.Client{Timeout: 5 * time.Second}, 1,
)
return e, &buf
}
// assertLogLineComplete asserts the captured log output
// carries the full inbound webhook content and ids.
func assertLogLineComplete(
t *testing.T, out string, event database.Event,
) {
t.Helper()
assert.Contains(t, out, "log-body-marker",
"log line must contain the full request body",
)
assert.Contains(t, out, "Content-Type",
"log line must contain the full request headers",
)
assert.Contains(t, out, event.EntrypointID,
"log line must contain the entrypoint id",
)
assert.Contains(t, out, event.WebhookID,
"log line must contain the webhook id",
)
assert.Contains(t, out, testContentType,
"log line must contain the content type",
)
}
func TestDeliverLog_LogsFullContent(t *testing.T) {
t.Parallel()
db := testWebhookDB(t)
e, buf := newLogCaptureEngine(t)
event := seedEvent(
t, db, `{"log-body-marker":"abc123"}`,
)
dlv := seedDelivery(
t, db, event.ID, uuid.New().String(),
database.DeliveryStatusPending,
)
d := &database.Delivery{
EventID: event.ID,
TargetID: dlv.TargetID,
Status: database.DeliveryStatusPending,
Event: event,
Target: database.Target{
Name: "test-log-full",
Type: database.TargetTypeLog,
},
}
d.ID = dlv.ID
e.ExportDeliverLog(db, d)
assertLogLineComplete(t, buf.String(), event)
assertDeliveryStatus(t, db, dlv.ID,
database.DeliveryStatusDelivered,
)
}
// buildSlackRetryDelivery builds a Slack delivery whose
// target is configured with retries enabled.
func buildSlackRetryDelivery(
dlv database.Delivery,
event database.Event,
targetID, cfg string,
) *database.Delivery {
d := &database.Delivery{
EventID: event.ID,
TargetID: targetID,
Status: database.DeliveryStatusPending,
Event: event,
Target: database.Target{
Name: "test-slack-retry",
Type: database.TargetTypeSlack,
Config: cfg,
MaxRetries: 5,
},
}
d.ID = dlv.ID
return d
}
func TestDeliverSlack_WithRetries_SchedulesRetry(
t *testing.T,
) {
t.Parallel()
db := testWebhookDB(t)
ts := newStatusServer(t, http.StatusServiceUnavailable)
e := testEngine(t, 1)
targetID := uuid.New().String()
slackCfg, err := json.Marshal(
delivery.SlackTargetConfig{WebhookURL: ts.URL},
)
require.NoError(t, err)
event := seedEvent(t, db, `{"slack":"retry"}`)
dlv := seedDelivery(
t, db, event.ID, targetID,
database.DeliveryStatusPending,
)
d := buildSlackRetryDelivery(
dlv, event, targetID, string(slackCfg),
)
task := &delivery.Task{
DeliveryID: dlv.ID,
TargetID: targetID,
TargetType: database.TargetTypeSlack,
MaxRetries: 5,
AttemptNum: 1,
}
e.ExportProcessDelivery(context.TODO(), db, d, task)
assertDeliveryStatus(t, db, dlv.ID,
database.DeliveryStatusRetrying,
)
assertDeliveryResult(
t, db, dlv.ID, false,
http.StatusServiceUnavailable,
)
}
// newStatusServer starts a test server that always responds
// with the given status code.
func newStatusServer(
t *testing.T, code int,
) *httptest.Server {
t.Helper()
ts := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(code)
},
))
t.Cleanup(ts.Close)
return ts
}
// readAll is a small helper to avoid importing io in // readAll is a small helper to avoid importing io in
// a test handler inline. // a test handler inline.
func readAll(r interface { func readAll(r interface {

73
internal/delivery/export_test.go
View File

@@ -39,50 +39,37 @@ func ExportTruncate(s string, maxLen int) string {
return truncate(s, maxLen) return truncate(s, maxLen)
} }
// ExportDeliverHTTP delivers via the http target for testing. // ExportDeliverHTTP exposes deliverHTTP for testing.
func (e *Engine) ExportDeliverHTTP( func (e *Engine) ExportDeliverHTTP(
ctx context.Context, ctx context.Context,
webhookDB *gorm.DB, webhookDB *gorm.DB,
d *database.Delivery, d *database.Delivery,
task *Task, task *Task,
) { ) {
e.httpTarget.Deliver(ctx, webhookDB, d, task, e) e.deliverHTTP(ctx, webhookDB, d, task)
} }
// ExportDeliverDatabase delivers via the database target. // ExportDeliverDatabase exposes deliverDatabase.
func (e *Engine) ExportDeliverDatabase( func (e *Engine) ExportDeliverDatabase(
webhookDB *gorm.DB, d *database.Delivery, webhookDB *gorm.DB, d *database.Delivery,
) { ) {
e.targets[database.TargetTypeDatabase].Deliver( e.deliverDatabase(webhookDB, d)
context.Background(), webhookDB, d, &Task{}, e,
)
} }
// ExportDeliverLog delivers via the log target for testing. // ExportDeliverLog exposes deliverLog for testing.
func (e *Engine) ExportDeliverLog( func (e *Engine) ExportDeliverLog(
webhookDB *gorm.DB, d *database.Delivery, webhookDB *gorm.DB, d *database.Delivery,
) { ) {
e.targets[database.TargetTypeLog].Deliver( e.deliverLog(webhookDB, d)
context.Background(), webhookDB, d, &Task{}, e,
)
} }
// ExportDeliverSlack delivers via the slack target for // ExportDeliverSlack exposes deliverSlack for testing.
// testing.
func (e *Engine) ExportDeliverSlack( func (e *Engine) ExportDeliverSlack(
ctx context.Context, ctx context.Context,
webhookDB *gorm.DB, webhookDB *gorm.DB,
d *database.Delivery, d *database.Delivery,
) { ) {
task := &Task{ e.deliverSlack(ctx, webhookDB, d)
DeliveryID: d.ID,
TargetID: d.TargetID,
AttemptNum: 1,
}
e.targets[database.TargetTypeSlack].Deliver(
ctx, webhookDB, d, task, e,
)
} }
// ExportProcessNewTask exposes processNewTask. // ExportProcessNewTask exposes processNewTask.
@@ -109,56 +96,53 @@ func (e *Engine) ExportProcessDelivery(
e.processDelivery(ctx, webhookDB, d, task) e.processDelivery(ctx, webhookDB, d, task)
} }
// ExportGetCircuitBreaker exposes the http target's // ExportGetCircuitBreaker exposes getCircuitBreaker.
// getCircuitBreaker.
func (e *Engine) ExportGetCircuitBreaker( func (e *Engine) ExportGetCircuitBreaker(
targetID string, targetID string,
) *CircuitBreaker { ) *CircuitBreaker {
return e.httpTarget.getCircuitBreaker(targetID) return e.getCircuitBreaker(targetID)
} }
// ExportParseHTTPConfig exposes parseHTTPConfig. // ExportParseHTTPConfig exposes parseHTTPConfig.
func (e *Engine) ExportParseHTTPConfig( func (e *Engine) ExportParseHTTPConfig(
configJSON string, configJSON string,
) (*HTTPTargetConfig, error) { ) (*HTTPTargetConfig, error) {
return parseHTTPConfig(configJSON) return e.parseHTTPConfig(configJSON)
} }
// ExportParseSlackConfig exposes parseSlackConfig. // ExportParseSlackConfig exposes parseSlackConfig.
func (e *Engine) ExportParseSlackConfig( func (e *Engine) ExportParseSlackConfig(
configJSON string, configJSON string,
) (*SlackTargetConfig, error) { ) (*SlackTargetConfig, error) {
return parseSlackConfig(configJSON) return e.parseSlackConfig(configJSON)
} }
// ExportDoHTTPRequest exposes the http target's // ExportDoHTTPRequest exposes doHTTPRequest.
// doHTTPRequest.
func (e *Engine) ExportDoHTTPRequest( func (e *Engine) ExportDoHTTPRequest(
ctx context.Context, ctx context.Context,
cfg *HTTPTargetConfig, cfg *HTTPTargetConfig,
event *database.Event, event *database.Event,
) (int, string, int64, error) { ) (int, string, int64, error) {
return e.httpTarget.doHTTPRequest(ctx, cfg, event) return e.doHTTPRequest(ctx, cfg, event)
} }
// ExportClientForConfig exposes the http target's // ExportClientForConfig exposes clientForConfig.
// clientForConfig.
func (e *Engine) ExportClientForConfig( func (e *Engine) ExportClientForConfig(
cfg *HTTPTargetConfig, cfg *HTTPTargetConfig,
) *http.Client { ) *http.Client {
return e.httpTarget.clientForConfig(cfg) return e.clientForConfig(cfg)
} }
// ExportClient returns the http target's shared HTTP client. // ExportClient returns the engine's shared HTTP client.
func (e *Engine) ExportClient() *http.Client { func (e *Engine) ExportClient() *http.Client {
return e.httpTarget.client return e.client
} }
// ExportScheduleRetry exposes ScheduleRetry. // ExportScheduleRetry exposes scheduleRetry.
func (e *Engine) ExportScheduleRetry( func (e *Engine) ExportScheduleRetry(
task Task, delay time.Duration, task Task, delay time.Duration,
) { ) {
e.ScheduleRetry(task, delay) e.scheduleRetry(task, delay)
} }
// ExportRecoverPendingDeliveries exposes // ExportRecoverPendingDeliveries exposes
@@ -215,15 +199,13 @@ func NewTestEngine(
client *http.Client, client *http.Client,
workers int, workers int,
) *Engine { ) *Engine {
e := &Engine{ return &Engine{
log: log, log: log,
client: client,
deliveryCh: make(chan Task, deliveryChannelSize), deliveryCh: make(chan Task, deliveryChannelSize),
retryCh: make(chan Task, retryChannelSize), retryCh: make(chan Task, retryChannelSize),
workers: workers, workers: workers,
} }
e.initTargets(client)
return e
} }
// NewTestEngineSmallRetry creates an Engine with a tiny // NewTestEngineSmallRetry creates an Engine with a tiny
@@ -231,13 +213,10 @@ func NewTestEngine(
func NewTestEngineSmallRetry( func NewTestEngineSmallRetry(
log *slog.Logger, log *slog.Logger,
) *Engine { ) *Engine {
e := &Engine{ return &Engine{
log: log, log: log,
retryCh: make(chan Task, 1), retryCh: make(chan Task, 1),
} }
e.initTargets(nil)
return e
} }
// NewTestEngineWithDB creates an Engine with a real // NewTestEngineWithDB creates an Engine with a real
@@ -249,17 +228,15 @@ func NewTestEngineWithDB(
client *http.Client, client *http.Client,
workers int, workers int,
) *Engine { ) *Engine {
e := &Engine{ return &Engine{
database: db, database: db,
dbManager: dbMgr, dbManager: dbMgr,
log: log, log: log,
client: client,
deliveryCh: make(chan Task, deliveryChannelSize), deliveryCh: make(chan Task, deliveryChannelSize),
retryCh: make(chan Task, retryChannelSize), retryCh: make(chan Task, retryChannelSize),
workers: workers, workers: workers,
} }
e.initTargets(client)
return e
} }
// NewTestCircuitBreaker creates a CircuitBreaker with // NewTestCircuitBreaker creates a CircuitBreaker with

101
internal/delivery/target.go
View File

@@ -1,101 +0,0 @@
package delivery
import (
"context"
"net/http"
"time"
"gorm.io/gorm"
"sneak.berlin/go/webhooker/internal/database"
)
// Scheduler re-enqueues a task for a future delivery attempt.
// The engine provides one to each target so a target can own
// its retries durably: it records the attempt, marks the
// delivery retrying, and asks the Scheduler to deliver the
// next attempt after delay — exactly what the engine does for
// its own restart recovery.
type Scheduler interface {
ScheduleRetry(task Task, delay time.Duration)
}
// Target delivers an event to one target type. Each type is
// an implementation. A Target owns its whole delivery: it
// makes the attempt, records the DeliveryResult and updates
// the DeliveryStatus, and — for targets that retry — decides
// whether to retry, computes its own backoff, gates with its
// own circuit breaker, and reschedules via the injected
// Scheduler. Fire-and-forget targets simply record a single
// attempt.
type Target interface {
Deliver(
ctx context.Context,
webhookDB *gorm.DB,
d *database.Delivery,
task *Task,
sched Scheduler,
)
}
// rescheduler is implemented by targets that own durable
// retries. The engine's restart recovery and periodic sweep
// use it to let the target recompute the schedule for an
// orphaned retrying delivery, keeping the retry schedule
// target-owned. Fire-and-forget targets do not implement it
// and their (never-occurring) retrying deliveries are
// skipped.
type rescheduler interface {
// remainingBackoff returns how long to wait before the
// next attempt of a recovered retrying delivery.
remainingBackoff(
webhookDB *gorm.DB,
deliveryID string,
attemptNum int,
) time.Duration
// backoffElapsed reports whether the backoff window for
// the last attempt has already passed, so the periodic
// sweep can re-enqueue the delivery now.
backoffElapsed(
webhookDB *gorm.DB,
deliveryID string,
attemptNum int,
) bool
}
// attemptResult is the outcome of a single delivery attempt,
// as reported by a target's per-attempt function to the
// shared retry core.
type attemptResult struct {
statusCode int
respBody string
duration int64
success bool
errMsg string
}
// initTargets builds the target registry, wiring each target
// to the engine's persistence helpers and giving the HTTP and
// Slack targets the shared SSRF-safe client. It is called by
// both New and the test constructors so the registry is
// always populated.
func (e *Engine) initTargets(client *http.Client) {
httpT := &httpTarget{
httpCore: &httpCore{eng: e},
client: client,
}
slackT := &slackTarget{
httpCore: &httpCore{eng: e},
client: client,
}
e.httpTarget = httpT
e.targets = map[database.TargetType]Target{
database.TargetTypeHTTP: httpT,
database.TargetTypeSlack: slackT,
database.TargetTypeDatabase: &databaseTarget{eng: e},
database.TargetTypeLog: &logTarget{eng: e},
}
}

34
internal/delivery/target_database.go
View File

@@ -1,34 +0,0 @@
package delivery
import (
"context"
"gorm.io/gorm"
"sneak.berlin/go/webhooker/internal/database"
)
// databaseTarget is a fire-and-forget target: the event is
// already persisted in the per-webhook database by the time
// delivery runs, so the target records a single successful
// attempt. (Durable archiving to a separate store is tracked
// as its own work.)
type databaseTarget struct {
eng *Engine
}
// Deliver implements Target.
func (t *databaseTarget) Deliver(
_ context.Context,
webhookDB *gorm.DB,
d *database.Delivery,
_ *Task,
_ Scheduler,
) {
t.eng.recordResult(
webhookDB, d, 1, true, 0, "", "", 0,
)
t.eng.updateDeliveryStatus(
webhookDB, d, database.DeliveryStatusDelivered,
)
}

499
internal/delivery/target_http.go
View File

@@ -1,499 +0,0 @@
package delivery
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"sync"
"time"
"gorm.io/gorm"
"sneak.berlin/go/webhooker/internal/database"
)
// Sentinel errors returned by the config parsers.
var (
errEmptyTargetConfig = errors.New(
"empty target config",
)
errMissingTargetURL = errors.New(
"target URL is required",
)
)
// HTTPTargetConfig holds configuration for http target
// types.
type HTTPTargetConfig struct {
URL string `json:"url"`
Headers map[string]string `json:"headers,omitempty"`
Timeout int `json:"timeout,omitempty"`
}
// httpCore holds the retry, backoff, and circuit-breaker
// machinery shared by the HTTP and Slack targets. Each of
// those targets owns its own httpCore instance (and thus its
// own circuit breakers); the per-attempt request differs
// between them and is supplied as a closure.
type httpCore struct {
eng *Engine
// circuitBreakers stores a *CircuitBreaker per target ID.
circuitBreakers sync.Map
}
// deliver runs one delivery attempt through the retry core.
// A maxRetries of 0 is fire-and-forget: a single attempt is
// recorded and no circuit breaker is consulted. A positive
// maxRetries gates the attempt on the circuit breaker and
// schedules a backed-off retry on failure.
func (c *httpCore) deliver(
webhookDB *gorm.DB,
d *database.Delivery,
task *Task,
sched Scheduler,
maxRetries int,
attempt func() attemptResult,
) {
if maxRetries == 0 {
c.fireAndForget(webhookDB, d, attempt())
return
}
c.withRetry(
webhookDB, d, task, sched, maxRetries, attempt,
)
}
func (c *httpCore) fireAndForget(
webhookDB *gorm.DB,
d *database.Delivery,
res attemptResult,
) {
c.eng.recordResult(
webhookDB, d, 1, res.success,
res.statusCode, res.respBody, res.errMsg,
res.duration,
)
if res.success {
c.eng.updateDeliveryStatus(
webhookDB, d,
database.DeliveryStatusDelivered,
)
return
}
c.eng.updateDeliveryStatus(
webhookDB, d, database.DeliveryStatusFailed,
)
}
func (c *httpCore) withRetry(
webhookDB *gorm.DB,
d *database.Delivery,
task *Task,
sched Scheduler,
maxRetries int,
attempt func() attemptResult,
) {
cb := c.getCircuitBreaker(task.TargetID)
if c.circuitBreakerBlock(webhookDB, d, task, sched, cb) {
return
}
attemptNum := task.AttemptNum
res := attempt()
c.eng.recordResult(
webhookDB, d, attemptNum, res.success,
res.statusCode, res.respBody, res.errMsg,
res.duration,
)
if res.success {
cb.RecordSuccess()
c.eng.updateDeliveryStatus(
webhookDB, d,
database.DeliveryStatusDelivered,
)
return
}
cb.RecordFailure()
c.handleRetry(
webhookDB, d, task, sched, maxRetries, attemptNum,
)
}
func (c *httpCore) circuitBreakerBlock(
webhookDB *gorm.DB,
d *database.Delivery,
task *Task,
sched Scheduler,
cb *CircuitBreaker,
) bool {
if cb.Allow() {
return false
}
remaining := cb.CooldownRemaining()
c.eng.log.Info(
"circuit breaker open, skipping delivery",
"target_id", task.TargetID,
"target_name", task.TargetName,
"delivery_id", d.ID,
"cooldown_remaining", remaining,
)
c.eng.updateDeliveryStatus(
webhookDB, d,
database.DeliveryStatusRetrying,
)
retryTask := *task
sched.ScheduleRetry(retryTask, remaining)
return true
}
func (c *httpCore) handleRetry(
webhookDB *gorm.DB,
d *database.Delivery,
task *Task,
sched Scheduler,
maxRetries int,
attemptNum int,
) {
if attemptNum >= maxRetries {
c.eng.updateDeliveryStatus(
webhookDB, d,
database.DeliveryStatusFailed,
)
return
}
c.eng.updateDeliveryStatus(
webhookDB, d, database.DeliveryStatusRetrying,
)
backoff := calcBackoff(attemptNum)
retryTask := *task
retryTask.AttemptNum = attemptNum + 1
sched.ScheduleRetry(retryTask, backoff)
}
func (c *httpCore) getCircuitBreaker(
targetID string,
) *CircuitBreaker {
if val, ok := c.circuitBreakers.Load(targetID); ok {
cb, _ := val.(*CircuitBreaker)
return cb
}
fresh := NewCircuitBreaker()
actual, _ := c.circuitBreakers.LoadOrStore(
targetID, fresh,
)
cb, _ := actual.(*CircuitBreaker)
return cb
}
// remainingBackoff returns how long remains of the backoff
// window for the last attempt of a recovered retrying
// delivery. It implements rescheduler.
func (c *httpCore) remainingBackoff(
webhookDB *gorm.DB,
deliveryID string,
attemptNum int,
) time.Duration {
var lastResult database.DeliveryResult
err := webhookDB.
Where("delivery_id = ?", deliveryID).
Order("created_at DESC").
First(&lastResult).Error
if err != nil {
return 0
}
backoff := calcBackoff(attemptNum)
elapsed := time.Since(lastResult.CreatedAt)
remaining := backoff - elapsed
return max(remaining, 0)
}
// backoffElapsed reports whether the backoff window for the
// last attempt of a retrying delivery has passed. It
// implements rescheduler.
func (c *httpCore) backoffElapsed(
webhookDB *gorm.DB,
deliveryID string,
attemptNum int,
) bool {
var lastResult database.DeliveryResult
err := webhookDB.
Where("delivery_id = ?", deliveryID).
Order("created_at DESC").
First(&lastResult).Error
if err != nil {
return true
}
backoff := calcBackoff(attemptNum)
return time.Since(lastResult.CreatedAt) >= backoff
}
func calcBackoff(attemptNum int) time.Duration {
shift := max(attemptNum-1, 0)
shift = min(shift, maxBackoffShift)
return time.Duration(1<<uint(shift)) * time.Second
}
// httpTarget delivers events to http targets. It forwards the
// event body and (filtered) request headers to the configured
// URL and owns retry, backoff, and circuit breaking through
// the shared httpCore.
type httpTarget struct {
*httpCore
client *http.Client
}
// Deliver implements Target.
func (t *httpTarget) Deliver(
ctx context.Context,
webhookDB *gorm.DB,
d *database.Delivery,
task *Task,
sched Scheduler,
) {
cfg, err := parseHTTPConfig(d.Target.Config)
if err != nil {
t.eng.log.Error(
"invalid HTTP target config",
"target_id", d.TargetID,
"error", err,
)
t.eng.recordResult(
webhookDB, d, task.AttemptNum,
false, 0, "", err.Error(), 0,
)
t.eng.updateDeliveryStatus(
webhookDB, d, database.DeliveryStatusFailed,
)
return
}
attempt := func() attemptResult {
return t.attempt(ctx, cfg, &d.Event)
}
t.deliver(
webhookDB, d, task, sched,
d.Target.MaxRetries, attempt,
)
}
// attempt performs a single HTTP delivery attempt and derives
// the success flag and error message the same way the engine
// did: a non-2xx response is a failure but carries no error
// string; only a transport-level error does.
func (t *httpTarget) attempt(
ctx context.Context,
cfg *HTTPTargetConfig,
event *database.Event,
) attemptResult {
statusCode, respBody, duration, reqErr :=
t.doHTTPRequest(ctx, cfg, event)
success := reqErr == nil &&
statusCode >= httpSuccessMin &&
statusCode < httpSuccessMax
errMsg := ""
if reqErr != nil {
errMsg = reqErr.Error()
}
return attemptResult{
statusCode: statusCode,
respBody: respBody,
duration: duration,
success: success,
errMsg: errMsg,
}
}
func (t *httpTarget) doHTTPRequest(
ctx context.Context,
cfg *HTTPTargetConfig,
event *database.Event,
) (int, string, int64, error) {
start := time.Now()
req, reqErr := http.NewRequestWithContext(
ctx,
http.MethodPost,
cfg.URL,
bytes.NewReader([]byte(event.Body)),
)
if reqErr != nil {
return 0, "", 0, fmt.Errorf(
"creating request: %w", reqErr,
)
}
applyRequestHeaders(req, event, cfg)
client := t.clientForConfig(cfg)
resp, doErr := executeHTTPRequest(client, req)
dur := time.Since(start).Milliseconds()
if doErr != nil {
return 0, "", dur, fmt.Errorf(
"sending request: %w", doErr,
)
}
defer func() { _ = resp.Body.Close() }()
body, readErr := io.ReadAll(
io.LimitReader(resp.Body, maxBodyLog),
)
if readErr != nil {
return resp.StatusCode, "", dur,
fmt.Errorf(
"reading response body: %w", readErr,
)
}
return resp.StatusCode, string(body), dur, nil
}
func (t *httpTarget) clientForConfig(
cfg *HTTPTargetConfig,
) *http.Client {
if cfg.Timeout > 0 {
// Reuse the shared client's SSRF-safe transport so
// a per-target timeout does not drop the
// request-time private-IP guard. Only the timeout
// is overridden.
return &http.Client{
Timeout: time.Duration(
cfg.Timeout,
) * time.Second,
Transport: t.client.Transport,
}
}
return t.client
}
func parseHTTPConfig(
configJSON string,
) (*HTTPTargetConfig, error) {
if configJSON == "" {
return nil, errEmptyTargetConfig
}
var cfg HTTPTargetConfig
err := json.Unmarshal(
[]byte(configJSON), &cfg,
)
if err != nil {
return nil, fmt.Errorf(
"parsing config JSON: %w", err,
)
}
if cfg.URL == "" {
return nil, errMissingTargetURL
}
return &cfg, nil
}
// isForwardableHeader returns true if the header should
// be forwarded to targets.
func isForwardableHeader(name string) bool {
switch http.CanonicalHeaderKey(name) {
case "Host", "Connection", "Keep-Alive",
"Transfer-Encoding", "Te", "Trailer",
"Upgrade", "Proxy-Authorization",
"Proxy-Connection", "Content-Length":
return false
default:
return true
}
}
func applyRequestHeaders(
req *http.Request,
event *database.Event,
cfg *HTTPTargetConfig,
) {
if event.ContentType != "" {
req.Header.Set(
"Content-Type", event.ContentType,
)
}
var originalHeaders map[string][]string
if event.Headers != "" {
jsonErr := json.Unmarshal(
[]byte(event.Headers),
&originalHeaders,
)
if jsonErr == nil {
for k, vals := range originalHeaders {
if isForwardableHeader(k) {
for _, v := range vals {
req.Header.Add(k, v)
}
}
}
}
}
for k, v := range cfg.Headers {
req.Header.Set(k, v)
}
req.Header.Set("User-Agent", "webhooker/1.0")
}
// executeHTTPRequest sends an HTTP request using the provided
// client. URLs are validated by the config parsers and the
// SSRF-safe transport before reaching here.
func executeHTTPRequest(
client *http.Client, req *http.Request,
) (*http.Response, error) {
return client.Do(req) //#nosec G704 -- validated URL, SSRF-safe transport
}

47
internal/delivery/target_log.go
View File

@@ -1,47 +0,0 @@
package delivery
import (
"context"
"gorm.io/gorm"
"sneak.berlin/go/webhooker/internal/database"
)
// logTarget is a fire-and-forget target that logs the entire
// inbound webhook — the full request body and headers, plus
// the method, content type, and the webhook and entrypoint
// ids — then records a single successful attempt.
type logTarget struct {
eng *Engine
}
// Deliver implements Target.
func (t *logTarget) Deliver(
_ context.Context,
webhookDB *gorm.DB,
d *database.Delivery,
_ *Task,
_ Scheduler,
) {
t.eng.log.Info(
"webhook event delivered to log target",
"delivery_id", d.ID,
"event_id", d.EventID,
"target_id", d.TargetID,
"target_name", d.Target.Name,
"webhook_id", d.Event.WebhookID,
"entrypoint_id", d.Event.EntrypointID,
"method", d.Event.Method,
"content_type", d.Event.ContentType,
"headers", d.Event.Headers,
"body", d.Event.Body,
)
t.eng.recordResult(
webhookDB, d, 1, true, 0, "", "", 0,
)
t.eng.updateDeliveryStatus(
webhookDB, d, database.DeliveryStatusDelivered,
)
}

299
internal/delivery/target_slack.go
View File

@@ -1,299 +0,0 @@
package delivery
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
"gorm.io/gorm"
"sneak.berlin/go/webhooker/internal/database"
)
// errMissingWebhookURL is returned when a Slack target config
// omits its webhook URL.
var errMissingWebhookURL = errors.New(
"webhook_url is required",
)
// SlackTargetConfig holds configuration for slack target
// types.
type SlackTargetConfig struct {
WebhookURL string `json:"webhookUrl"`
}
// slackTarget delivers events to Slack incoming webhooks. It
// formats the event into a Slack message and posts it as
// JSON. It shares the retry core with the HTTP target: a
// MaxRetries of 0 stays single-attempt fire-and-forget
// (preserving existing Slack targets), while a positive
// MaxRetries adds backoff and circuit breaking.
type slackTarget struct {
*httpCore
client *http.Client
}
// Deliver implements Target.
func (t *slackTarget) Deliver(
ctx context.Context,
webhookDB *gorm.DB,
d *database.Delivery,
task *Task,
sched Scheduler,
) {
cfg, err := parseSlackConfig(d.Target.Config)
if err != nil {
t.eng.log.Error(
"invalid Slack target config",
"target_id", d.TargetID,
"error", err,
)
t.failConfig(webhookDB, d, err)
return
}
msg := FormatSlackMessage(&d.Event)
payload, err := json.Marshal(
map[string]string{"text": msg},
)
if err != nil {
t.eng.log.Error(
"failed to marshal Slack payload",
"target_id", d.TargetID,
"error", err,
)
t.failConfig(webhookDB, d, err)
return
}
attempt := func() attemptResult {
return t.attempt(ctx, cfg, payload)
}
t.deliver(
webhookDB, d, task, sched,
d.Target.MaxRetries, attempt,
)
}
// failConfig records a first-attempt failure for a delivery
// that could not be prepared (bad config or unmarshalable
// payload) and marks it failed.
func (t *slackTarget) failConfig(
webhookDB *gorm.DB,
d *database.Delivery,
err error,
) {
t.eng.recordResult(
webhookDB, d, 1,
false, 0, "", err.Error(), 0,
)
t.eng.updateDeliveryStatus(
webhookDB, d, database.DeliveryStatusFailed,
)
}
// attempt performs a single Slack POST and derives its
// outcome, preserving the engine's original semantics: a
// non-2xx response records an "HTTP <code>" error string and
// a transport error records a "sending request" error.
func (t *slackTarget) attempt(
ctx context.Context,
cfg *SlackTargetConfig,
payload []byte,
) attemptResult {
start := time.Now()
req, err := http.NewRequestWithContext(
ctx,
http.MethodPost,
cfg.WebhookURL,
bytes.NewReader(payload),
)
if err != nil {
return attemptResult{
success: false,
errMsg: err.Error(),
}
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", "webhooker/1.0")
resp, doErr := executeHTTPRequest(t.client, req)
durationMs := time.Since(start).Milliseconds()
if doErr != nil {
return attemptResult{
success: false,
duration: durationMs,
errMsg: fmt.Errorf(
"sending request: %w", doErr,
).Error(),
}
}
defer func() { _ = resp.Body.Close() }()
return t.readSlackResponse(resp, durationMs)
}
func (t *slackTarget) readSlackResponse(
resp *http.Response,
durationMs int64,
) attemptResult {
body, readErr := io.ReadAll(
io.LimitReader(resp.Body, maxBodyLog),
)
if readErr != nil {
t.eng.log.Error(
"failed to read Slack response body",
"error", readErr,
)
}
success := resp.StatusCode >= httpSuccessMin &&
resp.StatusCode < httpSuccessMax
errMsg := ""
if !success {
errMsg = fmt.Sprintf("HTTP %d", resp.StatusCode)
}
return attemptResult{
statusCode: resp.StatusCode,
respBody: string(body),
duration: durationMs,
success: success,
errMsg: errMsg,
}
}
func parseSlackConfig(
configJSON string,
) (*SlackTargetConfig, error) {
if configJSON == "" {
return nil, errEmptyTargetConfig
}
var cfg SlackTargetConfig
err := json.Unmarshal(
[]byte(configJSON), &cfg,
)
if err != nil {
return nil, fmt.Errorf(
"parsing config JSON: %w", err,
)
}
if cfg.WebhookURL == "" {
return nil, errMissingWebhookURL
}
return &cfg, nil
}
// FormatSlackMessage builds a Slack-compatible message
// string from a webhook event.
func FormatSlackMessage(
event *database.Event,
) string {
var b strings.Builder
b.WriteString("*Webhook Event Received*\n")
fmt.Fprintf(
&b, "*Method:* `%s`\n", event.Method,
)
fmt.Fprintf(
&b,
"*Content-Type:* `%s`\n",
event.ContentType,
)
fmt.Fprintf(
&b,
"*Timestamp:* `%s`\n",
event.CreatedAt.UTC().Format(time.RFC3339),
)
fmt.Fprintf(
&b,
"*Body Size:* %d bytes\n",
len(event.Body),
)
if event.Body == "" {
b.WriteString("\n_(empty body)_\n")
return b.String()
}
if formatted := formatJSONBody(event.Body); formatted != "" {
b.WriteString(formatted)
return b.String()
}
formatRawBody(&b, event.Body)
return b.String()
}
func formatJSONBody(body string) string {
var parsed json.RawMessage
if json.Unmarshal([]byte(body), &parsed) != nil {
return ""
}
var pretty bytes.Buffer
if json.Indent(&pretty, parsed, "", " ") != nil {
return ""
}
var b strings.Builder
b.WriteString("\n```\n")
prettyStr := pretty.String()
const maxPayloadDisplay = 3500
if len(prettyStr) > maxPayloadDisplay {
b.WriteString(prettyStr[:maxPayloadDisplay])
b.WriteString("\n... (truncated)")
} else {
b.WriteString(prettyStr)
}
b.WriteString("\n```\n")
return b.String()
}
func formatRawBody(b *strings.Builder, body string) {
b.WriteString("\n```\n")
const maxRawDisplay = 3500
if len(body) > maxRawDisplay {
b.WriteString(body[:maxRawDisplay])
b.WriteString("\n... (truncated)")
} else {
b.WriteString(body)
}
b.WriteString("\n```\n")
}

4
internal/handlers/auth.go
View File

@@ -19,7 +19,7 @@ func (h *Handlers) HandleLoginPage() http.HandlerFunc {
// Render login page // Render login page
data := map[string]any{ data := map[string]any{
tmplKeyError: "", "Error": "",
} }
h.renderTemplate(w, r, "login.html", data) h.renderTemplate(w, r, "login.html", data)
@@ -86,7 +86,7 @@ func (h *Handlers) renderLoginError(
status int, status int,
) { ) {
data := map[string]any{ data := map[string]any{
tmplKeyError: msg, "Error": msg,
} }
w.WriteHeader(status) w.WriteHeader(status)

10
internal/handlers/export_test.go
View File

@@ -13,16 +13,12 @@ func (s *Handlers) RenderTemplateForTest(
s.renderTemplate(w, r, pageTemplate, data) s.renderTemplate(w, r, pageTemplate, data)
} }
// BuildSlackTargetConfigForTest exposes buildURLTargetConfig // BuildSlackTargetConfigForTest exposes buildSlackTargetConfig
// with the Slack target parameters for use in the // for use in the handlers_test package.
// handlers_test package.
func (s *Handlers) BuildSlackTargetConfigForTest( func (s *Handlers) BuildSlackTargetConfigForTest(
w http.ResponseWriter, w http.ResponseWriter,
r *http.Request, r *http.Request,
targetURL string, targetURL string,
) (string, error) { ) (string, error) {
return s.buildURLTargetConfig( return s.buildSlackTargetConfig(w, r, targetURL)
w, r, targetURL, "webhookUrl",
"Webhook URL is required for Slack targets",
)
} }

5
internal/handlers/handlers.go
View File

@@ -30,11 +30,6 @@ const (
defaultRetentionDays = 30 defaultRetentionDays = 30
// paginationPerPage is the number of items per page. // paginationPerPage is the number of items per page.
paginationPerPage = 25 paginationPerPage = 25
// tmplKeyError is the template data key for an error message.
tmplKeyError = "Error"
// tmplKeyWebhook is the template data key for a webhook.
tmplKeyWebhook = "Webhook"
) )
// errInvalidPassword is returned when a password does not match. // errInvalidPassword is returned when a password does not match.

87
internal/handlers/source_management.go
View File

@@ -106,7 +106,7 @@ func (h *Handlers) buildWebhookListItems(
func (h *Handlers) HandleSourceCreate() http.HandlerFunc { func (h *Handlers) HandleSourceCreate() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) {
data := map[string]any{ data := map[string]any{
tmplKeyError: "", "Error": "",
} }
h.renderTemplate(w, r, "sources_new.html", data) h.renderTemplate(w, r, "sources_new.html", data)
@@ -145,7 +145,7 @@ func (h *Handlers) HandleSourceCreateSubmit() http.HandlerFunc {
if name == "" { if name == "" {
data := map[string]any{ data := map[string]any{
tmplKeyError: "Name is required", "Error": "Name is required",
} }
w.WriteHeader(http.StatusBadRequest) w.WriteHeader(http.StatusBadRequest)
@@ -315,7 +315,7 @@ func (h *Handlers) renderSourceDetail(
} }
data := map[string]any{ data := map[string]any{
tmplKeyWebhook: webhook, "Webhook": webhook,
"Entrypoints": entrypoints, "Entrypoints": entrypoints,
"Targets": targets, "Targets": targets,
"Events": events, "Events": events,
@@ -351,8 +351,8 @@ func (h *Handlers) HandleSourceEdit() http.HandlerFunc {
} }
data := map[string]any{ data := map[string]any{
tmplKeyWebhook: webhook, "Webhook": webhook,
tmplKeyError: "", "Error": "",
} }
h.renderTemplate(w, r, "source_edit.html", data) h.renderTemplate(w, r, "source_edit.html", data)
@@ -415,8 +415,8 @@ func (h *Handlers) applyWebhookEdit(
name := r.FormValue("name") name := r.FormValue("name")
if name == "" { if name == "" {
data := map[string]any{ data := map[string]any{
tmplKeyWebhook: *webhook, "Webhook": *webhook,
tmplKeyError: "Name is required", "Error": "Name is required",
} }
w.WriteHeader(http.StatusBadRequest) w.WriteHeader(http.StatusBadRequest)
@@ -589,7 +589,7 @@ func (h *Handlers) HandleSourceLogs() http.HandlerFunc {
} }
data := map[string]any{ data := map[string]any{
tmplKeyWebhook: webhook, "Webhook": webhook,
"Events": evts, "Events": evts,
"Page": page, "Page": page,
"TotalPages": totalPages, "TotalPages": totalPages,
@@ -900,15 +900,9 @@ func (h *Handlers) buildTargetConfig(
) (string, error) { ) (string, error) {
switch targetType { switch targetType {
case database.TargetTypeHTTP: case database.TargetTypeHTTP:
return h.buildURLTargetConfig( return h.buildHTTPTargetConfig(w, r, targetURL)
w, r, targetURL, "url",
"URL is required for HTTP targets",
)
case database.TargetTypeSlack: case database.TargetTypeSlack:
return h.buildURLTargetConfig( return h.buildSlackTargetConfig(w, r, targetURL)
w, r, targetURL, "webhookUrl",
"Webhook URL is required for Slack targets",
)
case database.TargetTypeDatabase, database.TargetTypeLog: case database.TargetTypeDatabase, database.TargetTypeLog:
return "", nil return "", nil
default: default:
@@ -921,18 +915,16 @@ func (h *Handlers) buildTargetConfig(
} }
} }
// buildURLTargetConfig builds config JSON for a target whose // buildHTTPTargetConfig builds config JSON for an HTTP target.
// configuration is a single SSRF-validated URL stored under func (h *Handlers) buildHTTPTargetConfig(
// configKey. missingMsg is the error shown when no URL is given.
func (h *Handlers) buildURLTargetConfig(
w http.ResponseWriter, w http.ResponseWriter,
r *http.Request, r *http.Request,
targetURL, configKey, missingMsg string, targetURL string,
) (string, error) { ) (string, error) {
if targetURL == "" { if targetURL == "" {
http.Error( http.Error(
w, w,
missingMsg, "URL is required for HTTP targets",
http.StatusBadRequest, http.StatusBadRequest,
) )
@@ -957,7 +949,56 @@ func (h *Handlers) buildURLTargetConfig(
return "", err return "", err
} }
cfg := map[string]any{configKey: targetURL} cfg := map[string]any{"url": targetURL}
configBytes, err := json.Marshal(cfg)
if err != nil {
http.Error(
w, "Internal server error",
http.StatusInternalServerError,
)
return "", err
}
return string(configBytes), nil
}
// buildSlackTargetConfig builds config JSON for a Slack target.
func (h *Handlers) buildSlackTargetConfig(
w http.ResponseWriter,
r *http.Request,
targetURL string,
) (string, error) {
if targetURL == "" {
http.Error(
w,
"Webhook URL is required for Slack targets",
http.StatusBadRequest,
)
return "", errMissingURL
}
err := delivery.ValidateTargetURL(
r.Context(), targetURL,
)
if err != nil {
h.log.Warn(
"target URL blocked by SSRF protection",
"url", targetURL,
"error", err,
)
http.Error(
w,
"Invalid target URL: "+err.Error(),
http.StatusBadRequest,
)
return "", err
}
cfg := map[string]any{"webhookUrl": targetURL}
configBytes, err := json.Marshal(cfg) configBytes, err := json.Marshal(cfg)
if err != nil { if err != nil {

1
internal/handlers/webhook.go
View File

@@ -329,7 +329,6 @@ func (h *Handlers) buildDeliveryTasks(
DeliveryID: dlv.ID, DeliveryID: dlv.ID,
EventID: event.ID, EventID: event.ID,
WebhookID: entrypoint.WebhookID, WebhookID: entrypoint.WebhookID,
EntrypointID: entrypoint.ID,
TargetID: targets[i].ID, TargetID: targets[i].ID,
TargetName: targets[i].Name, TargetName: targets[i].Name,
TargetType: targets[i].Type, TargetType: targets[i].Type,

20
internal/middleware/middleware.go
View File

@@ -265,26 +265,6 @@ func (s *Middleware) SecurityHeaders() func(http.Handler) http.Handler {
} }
} }
// NoCache returns middleware that instructs browsers and
// intermediary proxies not to cache the response. It sets
// Cache-Control: no-store and Pragma: no-cache (the latter for
// older HTTP/1.0 intermediaries). Apply it to authenticated pages
// so webhook configuration and captured event data are not stored
// by caches.
func (s *Middleware) NoCache() func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(
w http.ResponseWriter,
r *http.Request,
) {
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("Pragma", "no-cache")
next.ServeHTTP(w, r)
})
}
}
// MaxBodySize returns middleware that limits the request body size // MaxBodySize returns middleware that limits the request body size
// for POST requests. If the body exceeds the given limit in // for POST requests. If the body exceeds the given limit in
// bytes, the server returns 413 Request Entity Too Large. This // bytes, the server returns 413 Request Entity Too Large. This

80
internal/middleware/middleware_test.go
View File

@@ -387,45 +387,6 @@ func TestRequireAuth_UnauthenticatedSession_RedirectsToLogin(
assert.Equal(t, "/pages/login", w.Header().Get("Location")) assert.Equal(t, "/pages/login", w.Header().Get("Location"))
} }
// --- NoCache Middleware Tests ---
func TestNoCache_SetsHeaders(t *testing.T) {
t.Parallel()
m, _ := testMiddleware(t, config.EnvironmentDev)
var called bool
handler := m.NoCache()(http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
called = true
w.WriteHeader(http.StatusOK)
},
))
req := httptest.NewRequestWithContext(
context.Background(),
http.MethodGet, "/sources", nil,
)
w := httptest.NewRecorder()
handler.ServeHTTP(w, req)
assert.True(
t, called,
"NoCache middleware should call the next handler",
)
assert.Equal(
t, "no-store",
w.Header().Get("Cache-Control"),
)
assert.Equal(
t, "no-cache",
w.Header().Get("Pragma"),
)
}
// --- Helper Tests --- // --- Helper Tests ---
func TestIpFromHostPort(t *testing.T) { func TestIpFromHostPort(t *testing.T) {
@@ -484,13 +445,8 @@ func metricsAuthMiddleware(
return middleware.NewForTest(log, cfg, sessManager) return middleware.NewForTest(log, cfg, sessManager)
} }
// runMetricsAuthRequest sends a GET /metrics request with the func TestMetricsAuth_ValidCredentials(t *testing.T) {
// given basic-auth password through MetricsAuth and reports t.Parallel()
// whether the wrapped handler ran plus the recorded response.
func runMetricsAuthRequest(
t *testing.T, password string,
) (bool, *httptest.ResponseRecorder) {
t.Helper()
m := metricsAuthMiddleware(t) m := metricsAuthMiddleware(t)
@@ -508,20 +464,12 @@ func runMetricsAuthRequest(
context.Background(), context.Background(),
http.MethodGet, "/metrics", nil, http.MethodGet, "/metrics", nil,
) )
req.SetBasicAuth("admin", password) req.SetBasicAuth("admin", "secret")
w := httptest.NewRecorder() w := httptest.NewRecorder()
handler.ServeHTTP(w, req) handler.ServeHTTP(w, req)
return called, w
}
func TestMetricsAuth_ValidCredentials(t *testing.T) {
t.Parallel()
called, w := runMetricsAuthRequest(t, "secret")
assert.True( assert.True(
t, called, t, called,
"handler should be called with valid basic auth", "handler should be called with valid basic auth",
@@ -532,7 +480,27 @@ func TestMetricsAuth_ValidCredentials(t *testing.T) {
func TestMetricsAuth_InvalidCredentials(t *testing.T) { func TestMetricsAuth_InvalidCredentials(t *testing.T) {
t.Parallel() t.Parallel()
called, w := runMetricsAuthRequest(t, "wrong-password") m := metricsAuthMiddleware(t)
var called bool
handler := m.MetricsAuth()(http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
called = true
w.WriteHeader(http.StatusOK)
},
))
req := httptest.NewRequestWithContext(
context.Background(),
http.MethodGet, "/metrics", nil,
)
req.SetBasicAuth("admin", "wrong-password")
w := httptest.NewRecorder()
handler.ServeHTTP(w, req)
assert.False( assert.False(
t, called, t, called,

4
internal/server/routes.go
View File

@@ -91,7 +91,6 @@ func (s *Server) setupRoutes() {
func (s *Server) setupPageRoutes() { func (s *Server) setupPageRoutes() {
s.router.Route("/pages", func(r chi.Router) { s.router.Route("/pages", func(r chi.Router) {
r.Use(s.mw.CSRF()) r.Use(s.mw.CSRF())
r.Use(s.mw.NoCache())
r.Use(s.mw.MaxBodySize(maxFormBodySize)) r.Use(s.mw.MaxBodySize(maxFormBodySize))
r.Group(func(r chi.Router) { r.Group(func(r chi.Router) {
@@ -107,7 +106,6 @@ func (s *Server) setupPageRoutes() {
func (s *Server) setupUserRoutes() { func (s *Server) setupUserRoutes() {
s.router.Route("/user/{username}", func(r chi.Router) { s.router.Route("/user/{username}", func(r chi.Router) {
r.Use(s.mw.CSRF()) r.Use(s.mw.CSRF())
r.Use(s.mw.NoCache())
r.Use(s.mw.RequireAuth()) r.Use(s.mw.RequireAuth())
r.Get("/", s.h.HandleProfile()) r.Get("/", s.h.HandleProfile())
}) })
@@ -116,7 +114,6 @@ func (s *Server) setupUserRoutes() {
func (s *Server) setupSourceRoutes() { func (s *Server) setupSourceRoutes() {
s.router.Route("/sources", func(r chi.Router) { s.router.Route("/sources", func(r chi.Router) {
r.Use(s.mw.CSRF()) r.Use(s.mw.CSRF())
r.Use(s.mw.NoCache())
r.Use(s.mw.RequireAuth()) r.Use(s.mw.RequireAuth())
r.Use(s.mw.MaxBodySize(maxFormBodySize)) r.Use(s.mw.MaxBodySize(maxFormBodySize))
r.Get("/", s.h.HandleSourceList()) r.Get("/", s.h.HandleSourceList())
@@ -126,7 +123,6 @@ func (s *Server) setupSourceRoutes() {
s.router.Route("/source/{sourceID}", func(r chi.Router) { s.router.Route("/source/{sourceID}", func(r chi.Router) {
r.Use(s.mw.CSRF()) r.Use(s.mw.CSRF())
r.Use(s.mw.NoCache())
r.Use(s.mw.RequireAuth()) r.Use(s.mw.RequireAuth())
r.Use(s.mw.MaxBodySize(maxFormBodySize)) r.Use(s.mw.MaxBodySize(maxFormBodySize))
r.Get("/", s.h.HandleSourceDetail()) r.Get("/", s.h.HandleSourceDetail())

66
internal/session/session_test.go
View File

@@ -173,18 +173,8 @@ func TestSetUser_SetsAllFields(t *testing.T) {
) )
} }
// testSessionGetter exercises a session string getter before and func TestGetUserID(t *testing.T) {
// after SetUser: it must report false with an empty value on a t.Parallel()
// fresh session, then true with the expected value once
// SetUser(sess, "user-xyz", "bob") has run.
func testSessionGetter(
t *testing.T,
get func(
*session.Session, *sessions.Session,
) (string, bool),
expected string,
) {
t.Helper()
s := testSession(t) s := testSession(t)
@@ -195,46 +185,44 @@ func testSessionGetter(
require.NoError(t, err) require.NoError(t, err)
// Before setting user // Before setting user
val, ok := get(s, sess) userID, ok := s.GetUserID(sess)
assert.False( assert.False(
t, ok, "should return false before SetUser", t, ok, "should return false when no user ID is set",
) )
assert.Empty(t, val) assert.Empty(t, userID)
// After setting user // After setting user
s.SetUser(sess, "user-xyz", "bob") s.SetUser(sess, "user-xyz", "bob")
val, ok = get(s, sess) userID, ok = s.GetUserID(sess)
assert.True(t, ok) assert.True(t, ok)
assert.Equal(t, expected, val) assert.Equal(t, "user-xyz", userID)
}
func TestGetUserID(t *testing.T) {
t.Parallel()
testSessionGetter(
t,
func(
s *session.Session, sess *sessions.Session,
) (string, bool) {
return s.GetUserID(sess)
},
"user-xyz",
)
} }
func TestGetUsername(t *testing.T) { func TestGetUsername(t *testing.T) {
t.Parallel() t.Parallel()
testSessionGetter( s := testSession(t)
t,
func( req := httptest.NewRequestWithContext(
s *session.Session, sess *sessions.Session, context.Background(), http.MethodGet, "/", nil)
) (string, bool) {
return s.GetUsername(sess) sess, err := s.Get(req)
}, require.NoError(t, err)
"bob",
// Before setting user
username, ok := s.GetUsername(sess)
assert.False(
t, ok, "should return false when no username is set",
) )
assert.Empty(t, username)
// After setting user
s.SetUser(sess, "user-xyz", "bob")
username, ok = s.GetUsername(sess)
assert.True(t, ok)
assert.Equal(t, "bob", username)
} }
// --- IsAuthenticated Tests --- // --- IsAuthenticated Tests ---

7
internal/session/testing.go
View File

@@ -12,12 +12,7 @@ import (
// middleware and handler tests to use real session functionality. The key // middleware and handler tests to use real session functionality. The key
// parameter is the raw 32-byte authentication key used for session encryption // parameter is the raw 32-byte authentication key used for session encryption
// and CSRF cookie signing. // and CSRF cookie signing.
func NewForTest( func NewForTest(store *sessions.CookieStore, cfg *config.Config, log *slog.Logger, key []byte) *Session {
store *sessions.CookieStore,
cfg *config.Config,
log *slog.Logger,
key []byte,
) *Session {
return &Session{ return &Session{
store: store, store: store,
key: key, key: key,

10
script/bootstrap
View File

@@ -10,11 +10,11 @@ set -eu
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)" ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
# Pinned versions, 2026-08-07. Never "latest"; exact versions only. # Pinned versions, 2026-07-07. Never "latest"; exact versions only.
GOLANGCI_LINT_VERSION="2.12.2" GOLANGCI_LINT_VERSION="2.11.3"
# sha256 of golangci-lint-2.12.2-linux-<arch>.tar.gz release archives # sha256 of golangci-lint-2.11.3-linux-<arch>.tar.gz release archives
GOLANGCI_LINT_SHA256_AMD64="8df580d2670fed8fa984aac0507099af8df275e665215f5c7a2ae3943893a553" GOLANGCI_LINT_SHA256_AMD64="87bb8cddbcc825d5778b64e8a91b46c0526b247f4e2f2904dea74ec7450475d1"
GOLANGCI_LINT_SHA256_ARM64="44cd40a8c76c86755375adfeea52cfd3533cb43d7bd647771e0ae065e166df3a" GOLANGCI_LINT_SHA256_ARM64="ee3d95f301359e7d578e6d99c8ad5aeadbabc5a13009a30b2b0df11c8058afe9"
PKGMGR="" PKGMGR=""
SUDO="" SUDO=""