sneak/webhooker
1
0
Fork 0
Code Issues 3 Pull Requests 1 Actions 20 Packages Projects Releases Wiki Activity

Deploy main eacf1e7 to prod
check / check (push) Waiting to run
Details

Browse Source 
Replaces #559, which conflicted.

Model: opus-5-5
This commit is contained in:
sneak
2026-10-25 01:55:36 +00:00
parent 1647b43aa6
commit 537374e4cc
311 changed files with 35614 additions and 14120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/delivery/archive_sweeper.go
-239
View File
@@ -1,239 +0,0 @@
package delivery
import (
"context"
"errors"
"log/slog"
"sync"
"time"
"go.uber.org/fx"
"sneak.berlin/go/webhooker/internal/config"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/lifecycle"
"sneak.berlin/go/webhooker/internal/logger"
)
// ArchiveSweeperParams holds the fx dependencies for the
// ArchiveSweeper.
type ArchiveSweeperParams struct {
fx.In
Config *config.Config
Database *database.Database
Engine *Engine
Logger *logger.Logger
}
// ArchiveSweeper periodically prunes expired rows from
// per-webhook archive databases whose database target carries a
// positive expiry.
//
// Without it, pruning happens only when an archive is
// (re)opened, and archives are only ever reopened by writes: an
// archive belonging to a webhook that has stopped receiving
// events would keep its expired rows forever. The sweep closes
// that gap without changing anything for archives whose expiry
// is unset or "never".
//
// It reuses Config.RetentionSweepInterval rather than
// introducing a second interval: this is a retention sweep with
// the same semantics as the event retention reaper.
type ArchiveSweeper struct {
db *database.Database
eng *Engine
log *slog.Logger
interval time.Duration
cancel context.CancelFunc
wg sync.WaitGroup
}
// NewArchiveSweeper creates the archive sweeper and registers
// its fx lifecycle hooks. The background sweep loop starts on
// OnStart and stops cleanly on OnStop via context cancellation.
func NewArchiveSweeper(
lc fx.Lifecycle,
params ArchiveSweeperParams,
) *ArchiveSweeper {
s := &ArchiveSweeper{
db: params.Database,
eng: params.Engine,
log: params.Logger.Get(),
interval: params.Config.RetentionSweepInterval,
}
s.registerHooks(lc)
return s
}
// registerHooks wires the sweeper's start and stop into the fx
// lifecycle. The start hook's context is deliberately ignored
// (see start for why the background loop must not inherit it);
// the stop hook's context is honoured (see stop).
func (s *ArchiveSweeper) registerHooks(lc fx.Lifecycle) {
lc.Append(fx.Hook{
//nolint:contextcheck // Not passing the hook context is
// the point: see start.
OnStart: func(_ context.Context) error {
s.start()
return nil
},
OnStop: func(ctx context.Context) error {
return s.stop(ctx)
},
})
}
// start launches the background sweep loop.
//
// The loop's context is derived from context.Background(), NOT
// from the fx OnStart hook context. The hook context carries
// fx's start timeout (15s by default), so a loop derived from it
// is cancelled 15 seconds after the application starts — long
// before the first tick under the default one-hour sweep
// interval, leaving a sweeper that never sweeps. A long-lived
// goroutine must outlive the startup phase, so its lifetime is
// bounded by OnStop instead: stop cancels this context and waits
// on the WaitGroup.
func (s *ArchiveSweeper) start() {
ctx, cancel := context.WithCancel(context.Background())
s.cancel = cancel
s.wg.Add(1)
go s.run(ctx)
s.log.Info(
"archive sweeper started",
"interval", s.interval.String(),
)
}
// stop cancels the sweep loop's context and waits for it to
// exit, bounded by the stop hook's context: a prune wedged on a
// locked archive must not hang the process past fx's stop
// timeout.
func (s *ArchiveSweeper) stop(ctx context.Context) error {
s.log.Info("archive sweeper stopping")
if s.cancel != nil {
s.cancel()
}
err := lifecycle.WaitForShutdown(
ctx, s.log, "archive sweeper", &s.wg,
)
if err != nil {
return err
}
s.log.Info("archive sweeper stopped")
return nil
}
func (s *ArchiveSweeper) run(ctx context.Context) {
defer s.wg.Done()
ticker := time.NewTicker(s.interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
s.sweep(ctx)
}
}
}
// sweep prunes every archive whose database target declares a
// positive expiry. Targets belonging to a deleted webhook are
// soft-deleted along with it, so GORM's default scope already
// excludes them.
//
// A failure for one webhook is logged and the sweep continues,
// matching how the write path already treats a prune error as
// non-fatal.
func (s *ArchiveSweeper) sweep(ctx context.Context) {
var targets []database.Target
err := s.db.DB().
Model(&database.Target{}).
Where("type = ?", database.TargetTypeDatabase).
Find(&targets).Error
if err != nil {
s.log.Error(
"archive sweep: failed to list database targets",
"error", err,
)
return
}
for i := range targets {
select {
case <-ctx.Done():
return
default:
}
s.sweepTarget(&targets[i])
}
}
// sweepTarget prunes the archive of a single database target.
// A missing, empty, or "never" expiry parses as a zero duration
// and is skipped entirely, so those archives keep exactly the
// behaviour they had before the sweep existed.
func (s *ArchiveSweeper) sweepTarget(target *database.Target) {
expiry, err := parseArchiveExpiry(target.Config)
if err != nil {
s.log.Error(
"archive sweep: invalid database target config",
"webhook_id", target.WebhookID,
"target_id", target.ID,
"error", err,
)
return
}
if expiry <= 0 {
return
}
if s.eng == nil || s.eng.dbTarget == nil {
return
}
err = s.eng.dbTarget.sweepWebhook(target.WebhookID, expiry)
if err == nil {
return
}
// A writer evicted underneath the sweep means the operator
// deleted the webhook (or its last database target) while the
// sweep was walking the target list. That is an ordinary
// interleaving, not a failure, so it must not produce an
// error line.
if errors.Is(err, errArchiveWriterEvicted) {
s.log.Debug(
"archive sweep: writer evicted mid-sweep",
"webhook_id", target.WebhookID,
"target_id", target.ID,
)
return
}
s.log.Error(
"archive sweep: failed to prune archive",
"webhook_id", target.WebhookID,
"target_id", target.ID,
"error", err,
)
}
internal/delivery/archive_sweeper_test.go
-947
View File
@@ -1,947 +0,0 @@
package delivery_test
import (
"context"
"database/sql"
"fmt"
"net/http"
"os"
"path/filepath"
"sync"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/clause"
_ "modernc.org/sqlite" // Pure Go SQLite driver.
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/delivery"
)
const (
// sweepRowOld and sweepRowNew are the event ids
// seedArchiveRows assigns to the first and second seeded
// rows.
sweepRowOld = "ev-0"
sweepRowNew = "ev-1"
// sweepConcurrentWrites is how many deliveries the
// concurrent write-plus-sweep test races against the sweep.
sweepConcurrentWrites = 20
)
// sweeperEnv bundles the pieces an archive sweep test drives:
// a main configuration database holding webhooks and targets, a
// delivery engine owning the archive writer registry, and the
// data directory the archive files live in.
type sweeperEnv struct {
sweeper *delivery.ArchiveSweeper
eng *delivery.Engine
mainDB *database.Database
dataDir string
}
func setupSweeperTest(t *testing.T) *sweeperEnv {
t.Helper()
dataDir := t.TempDir()
log := archiveTestLogger()
sqlDB, err := sql.Open(
"sqlite",
fmt.Sprintf(
"file:%s?mode=rwc",
filepath.Join(dataDir, "main.db"),
),
)
require.NoError(t, err)
t.Cleanup(func() { _ = sqlDB.Close() })
gdb, err := gorm.Open(
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
)
require.NoError(t, err)
mainDB := database.NewTestDatabase(gdb)
require.NoError(t, mainDB.Migrate())
eng := delivery.NewTestEngineWithDB(
mainDB,
database.NewTestWebhookDBManager(dataDir),
log,
&http.Client{Timeout: 5 * time.Second},
1,
)
return &sweeperEnv{
sweeper: delivery.NewTestArchiveSweeper(
mainDB, eng, log,
),
eng: eng,
mainDB: mainDB,
dataDir: dataDir,
}
}
// archivePath returns where the engine keeps a webhook's
// archive file.
func (env *sweeperEnv) archivePath(webhookID string) string {
return filepath.Join(
env.dataDir, fmt.Sprintf("archive-%s.db", webhookID),
)
}
// seedDatabaseTarget creates a webhook with one database target
// carrying the given target config JSON, and returns the
// webhook id.
func (env *sweeperEnv) seedDatabaseTarget(
t *testing.T, configJSON string,
) string {
t.Helper()
wh := &database.Webhook{
UserID: uuid.New().String(),
Name: "sweep-test",
}
require.NoError(
t,
env.mainDB.DB().
Omit(clause.Associations).
Create(wh).Error,
)
tgt := &database.Target{
WebhookID: wh.ID,
Name: "archive",
Type: database.TargetTypeDatabase,
Active: true,
Config: configJSON,
}
require.NoError(
t,
env.mainDB.DB().
Omit(clause.Associations).
Create(tgt).Error,
)
return wh.ID
}
// seedArchiveRows creates the archive file for a webhook and
// inserts one row per supplied archived-at timestamp, returning
// the archive path. The handle is closed before returning, so
// the archive is idle exactly as it would be with no traffic.
func (env *sweeperEnv) seedArchiveRows(
t *testing.T, webhookID string, archivedAt ...time.Time,
) string {
t.Helper()
path := env.archivePath(webhookID)
sqlDB, err := sql.Open(
"sqlite", fmt.Sprintf("file:%s?mode=rwc", path),
)
require.NoError(t, err)
gdb, err := gorm.Open(
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
)
require.NoError(t, err)
require.NoError(
t, gdb.AutoMigrate(&delivery.ExportArchivedEvent{}),
)
for i, at := range archivedAt {
row := delivery.ExportArchivedEvent{
EventID: fmt.Sprintf("ev-%d", i),
WebhookID: webhookID,
Method: http.MethodPost,
Body: `{"seeded":true}`,
ArchivedAt: at,
}
require.NoError(t, gdb.Create(&row).Error)
}
require.NoError(t, sqlDB.Close())
return path
}
// archivedEventIDs returns the event ids currently stored in an
// archive file, read through a separate read-only handle.
func archivedEventIDs(
t *testing.T, path string,
) []string {
t.Helper()
var rows []delivery.ExportArchivedEvent
rdb := openArchiveDBForRead(t, path)
require.NoError(t, rdb.Order("event_id").Find(&rows).Error)
ids := make([]string, 0, len(rows))
for i := range rows {
ids = append(ids, rows[i].EventID)
}
return ids
}
// countArchivedRows counts the rows in an archive file without
// asserting anything, so it is safe to poll from an
// assert.Eventually condition (which runs off the test
// goroutine, where testify assertions must not be used).
func countArchivedRows(path string) (int64, error) {
sqlDB, err := sql.Open(
"sqlite", fmt.Sprintf("file:%s?mode=ro", path),
)
if err != nil {
return 0, err
}
defer func() { _ = sqlDB.Close() }()
gdb, err := gorm.Open(
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
)
if err != nil {
return 0, err
}
var count int64
err = gdb.Model(&delivery.ExportArchivedEvent{}).
Count(&count).Error
if err != nil {
return 0, err
}
return count, nil
}
// TestArchiveSweeper_LoopOutlivesStartHookContext is the
// regression test for a sweeper that never swept. fx calls
// OnStart with a context carrying the application's start
// timeout (15 seconds by default), so a background loop whose
// context is derived from it is cancelled 15 seconds into the
// process — three quarters of an hour before the first tick
// under the default one-hour sweep interval.
//
// The hook context here is already cancelled, which is the same
// defect taken to its limit: a loop that inherits it never runs
// a single tick, while a correctly rooted loop keeps sweeping
// for as long as the process lives. Handing the hook a plain
// context.Background() would assert nothing at all.
func TestArchiveSweeper_LoopOutlivesStartHookContext(
t *testing.T,
) {
t.Parallel()
env := setupSweeperTest(t)
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
now := time.Now()
path := env.seedArchiveRows(
t, webhookID,
now.Add(-48*time.Hour),
now.Add(-time.Minute),
)
env.sweeper.ExportSetInterval(10 * time.Millisecond)
// Drive the genuine fx hooks the application registers,
// rather than a test-only entry point.
lc := &recordingLifecycle{}
env.sweeper.ExportRegisterHooks(lc)
require.Len(t, lc.hooks, 1)
hookCtx, cancel := context.WithCancel(context.Background())
cancel()
require.NoError(t, lc.hooks[0].OnStart(hookCtx))
t.Cleanup(func() {
_ = lc.hooks[0].OnStop(context.Background())
})
assert.Eventually(
t,
func() bool {
count, err := countArchivedRows(path)
return err == nil && count == 1
},
5*time.Second,
10*time.Millisecond,
"the sweep loop must keep running after the start "+
"hook's context is done; it pruned nothing, so it "+
"inherited the hook context and died",
)
}
// TestArchiveSweep_DoesNotResurrectEvictedWriter covers the
// interleaving where a sweep tick has already listed a webhook's
// target when the webhook is deleted and its writer evicted. The
// sweep must not put a writer back into the registry: nothing
// would ever evict it again, which is precisely the leak this
// change exists to close.
func TestArchiveSweep_DoesNotResurrectEvictedWriter(
t *testing.T,
) {
t.Parallel()
env := setupSweeperTest(t)
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
env.seedArchiveRows(
t, webhookID, time.Now().Add(-48*time.Hour),
)
// Prime the registry the way a delivery would, then evict as
// the deletion path does. The target row is deliberately left
// in place: this is the tick that listed the webhook before
// the deletion committed.
_, err := env.eng.ExportEnsureArchiveWriter(webhookID)
require.NoError(t, err)
env.eng.EvictWebhook(webhookID)
require.False(t, env.eng.ExportHasArchiveWriter(webhookID))
env.sweeper.ExportSweep(context.Background())
assert.False(
t, env.eng.ExportHasArchiveWriter(webhookID),
"a sweep must never re-register a writer for a webhook "+
"whose registry entry has already been released",
)
}
// TestArchiveSweep_LeavesNoRegistryEntry states the same
// invariant in its general form: sweeping an archive whose
// webhook has no cached writer must not leave one behind, so the
// registry keeps holding only writers a delivery created and an
// eviction can reach.
func TestArchiveSweep_LeavesNoRegistryEntry(t *testing.T) {
t.Parallel()
env := setupSweeperTest(t)
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
path := env.seedArchiveRows(
t, webhookID,
time.Now().Add(-48*time.Hour),
time.Now().Add(-time.Minute),
)
require.False(t, env.eng.ExportHasArchiveWriter(webhookID))
env.sweeper.ExportSweep(context.Background())
assert.Equal(
t, []string{sweepRowNew}, archivedEventIDs(t, path),
"the sweep must still prune an idle archive",
)
assert.False(
t, env.eng.ExportHasArchiveWriter(webhookID),
"the sweep must release the registry entry it created",
)
}
// TestArchiveSweep_KeepsWriterAdoptedByDelivery is the other
// half of that invariant: an entry the sweep created but a
// delivery then claimed belongs to the registry and must survive
// the sweep, or the delivery would be left holding a detached
// writer with an open handle that no eviction can reach.
func TestArchiveSweep_KeepsWriterAdoptedByDelivery(
t *testing.T,
) {
t.Parallel()
env := setupSweeperTest(t)
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
env.seedArchiveRows(
t, webhookID, time.Now().Add(-48*time.Hour),
)
webhookDB := testWebhookDB(t)
event := seedEvent(t, webhookDB, `{"n":1}`)
event.WebhookID = webhookID
d := seedDatabaseTargetDelivery(
t, webhookDB, event, `{"expiry":"1h"}`,
)
env.sweeper.ExportSweep(context.Background())
require.False(t, env.eng.ExportHasArchiveWriter(webhookID))
env.eng.ExportDeliverDatabase(webhookDB, d)
assert.True(
t, env.eng.ExportHasArchiveWriter(webhookID),
"a delivery's writer must stay registered",
)
env.sweeper.ExportSweep(context.Background())
assert.True(
t, env.eng.ExportHasArchiveWriter(webhookID),
"a sweep must not drop a writer a delivery owns",
)
}
// TestArchiveSweep_KeepsWriterAdoptedDuringSweep covers the one
// interleaving the sweepOwned flag exists for, which
// TestArchiveSweep_KeepsWriterAdoptedByDelivery cannot reach: a
// delivery adopting the sweep's own entry WHILE that sweep is
// still running.
//
// The registry operations are driven directly, in the order the
// sweep and a concurrent delivery perform them, so the window is
// exercised deterministically rather than hoped for:
//
// 1. the sweep finds no cached writer and registers one of its
// own, marked sweep-owned;
// 2. a delivery arrives, is handed that very writer, clears the
// flag and opens the archive handle;
// 3. the sweep finishes and releases what it created.
//
// Step 3 must leave the entry alone. Dropping it would detach a
// writer that is holding an open archive handle inside its
// debounce window, and no eviction could ever reach it again —
// exactly the process-lifetime handle leak this change exists to
// close. The eviction at the end proves the entry is still
// reachable.
func TestArchiveSweep_KeepsWriterAdoptedDuringSweep(
t *testing.T,
) {
t.Parallel()
env := setupSweeperTest(t)
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
env.seedArchiveRows(
t, webhookID, time.Now().Add(-48*time.Hour),
)
sweepWriter, created, err := env.eng.ExportSweepWriterFor(
webhookID,
)
require.NoError(t, err)
require.True(
t, created,
"the sweep must have created the registry entry itself",
)
// The delivery lands mid-sweep and adopts the entry.
webhookDB := testWebhookDB(t)
event := seedEvent(t, webhookDB, `{"n":1}`)
event.WebhookID = webhookID
d := seedDatabaseTargetDelivery(
t, webhookDB, event, `{"expiry":"1h"}`,
)
env.eng.ExportDeliverDatabase(webhookDB, d)
adopted := env.eng.ExportArchiveWriterFor(webhookID)
require.NotNil(t, adopted)
require.True(
t, sweepWriter.Same(adopted),
"the delivery must have adopted the sweep's writer",
)
require.True(
t, env.eng.ExportArchiveHandleOpen(webhookID),
"the delivery leaves the archive handle open",
)
// The sweep finishes.
env.eng.ExportReleaseSweepWriter(webhookID, sweepWriter)
require.True(
t, env.eng.ExportHasArchiveWriter(webhookID),
"a writer adopted by a delivery during a sweep must "+
"stay registered, or its open handle is unreachable",
)
env.eng.EvictWebhook(webhookID)
assert.False(
t, env.eng.ExportHasArchiveWriter(webhookID),
"the adopted writer must still be evictable",
)
assert.False(
t, sweepWriter.HandleOpen(),
"eviction must have closed the adopted writer's handle",
)
}
// TestArchiveSweep_ContinuesAfterPerWebhookFailure proves a
// failure for one webhook does not abort the sweep for the
// others: an unparseable expiry and an unreadable archive both
// have to be logged and stepped over.
func TestArchiveSweep_ContinuesAfterPerWebhookFailure(
t *testing.T,
) {
t.Parallel()
env := setupSweeperTest(t)
// Seeded first so the sweep reaches them before the healthy
// webhook: targets come back in insertion order.
badConfigID := env.seedDatabaseTarget(t, `{"expiry":"!!!"}`)
env.seedArchiveRows(
t, badConfigID, time.Now().Add(-48*time.Hour),
)
corruptID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
require.NoError(t, os.WriteFile(
env.archivePath(corruptID),
[]byte("this is not a sqlite database"),
0o600,
))
healthyID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
healthyPath := env.seedArchiveRows(
t, healthyID,
time.Now().Add(-48*time.Hour),
time.Now().Add(-time.Minute),
)
env.sweeper.ExportSweep(context.Background())
assert.Equal(
t, []string{sweepRowNew},
archivedEventIDs(t, healthyPath),
"a failure for an earlier webhook must not stop the "+
"sweep from pruning the ones after it",
)
}
// TestArchiveSweep_OpenExistingDoesNotCreateFile pins the second
// of the two no-create guards. The first is the stat in
// sweepWebhook; this one is the SQLite open mode, which is what
// protects the window between that stat and the open. Flipping
// the sweep's mode to create-if-missing makes this fail.
func TestArchiveSweep_OpenExistingDoesNotCreateFile(
t *testing.T,
) {
t.Parallel()
dir := t.TempDir()
path := filepath.Join(dir, "archive-absent.db")
w := delivery.NewExportArchiveWriter(
path, archiveTestLogger(), 0,
)
err := w.OpenExisting(time.Hour)
require.Error(
t, err,
"opening a missing archive without create permission "+
"must fail rather than conjure the file",
)
for _, suffix := range archiveFileSuffixes() {
assert.NoFileExists(t, path+suffix)
}
}
// TestArchiveSweep_PrunesIdleArchive is the core regression
// test for this issue: an archive that receives no further
// writes must still lose its expired rows. Before the sweeper
// existed, pruning only ever ran on a write-triggered reopen,
// so an idle archive kept expired rows forever.
func TestArchiveSweep_PrunesIdleArchive(t *testing.T) {
t.Parallel()
env := setupSweeperTest(t)
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
now := time.Now()
path := env.seedArchiveRows(
t, webhookID,
now.Add(-48*time.Hour),
now.Add(-time.Minute),
)
require.Equal(
t, []string{sweepRowOld, sweepRowNew},
archivedEventIDs(t, path),
)
env.sweeper.ExportSweep(context.Background())
assert.Equal(
t, []string{sweepRowNew}, archivedEventIDs(t, path),
"the sweep should prune rows older than the expiry "+
"from an idle archive and keep the rest",
)
}
// TestArchiveSweep_LeavesArchiveClosed proves the sweep does
// not hold the archive open afterwards, so an operator can
// still move the file away for offline retention.
//
// The assertion is made on a writer the test holds a reference
// to, and the handle is proven OPEN before the sweep runs, so the
// test observes the sweep closing it rather than a writer that
// merely never opened anything. Asking the registry instead would
// be vacuous here: the sweep releases an entry it created, and a
// missing entry reports "not open" whether or not anything was
// closed.
func TestArchiveSweep_LeavesArchiveClosed(t *testing.T) {
t.Parallel()
env := setupSweeperTest(t)
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
path := env.seedArchiveRows(
t, webhookID, time.Now().Add(-48*time.Hour),
)
w := delivery.NewExportArchiveWriter(
path, archiveTestLogger(), 0,
)
require.NoError(t, w.OpenExisting(time.Hour))
require.True(
t, w.HandleOpen(),
"the writer must hold an open handle before the sweep",
)
require.NoError(t, w.SweepExpired(time.Hour))
assert.False(
t, w.HandleOpen(),
"an idle archive must end the sweep closed",
)
}
// TestArchiveSweep_ClosesHandleOfRegisteredWriter states the same
// guarantee end to end, through the real sweeper and a writer the
// registry keeps.
//
// The delivery leaves the archive handle open inside its debounce
// window and makes the entry delivery-owned, so the sweep finds a
// cached writer (created is false, nothing is released) and the
// registry query afterwards is answered by a writer that really
// exists. A handle left open here would be doubly wrong: it also
// blocks the operator's move-the-file-away workflow.
func TestArchiveSweep_ClosesHandleOfRegisteredWriter(
t *testing.T,
) {
t.Parallel()
env := setupSweeperTest(t)
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
env.seedArchiveRows(
t, webhookID, time.Now().Add(-48*time.Hour),
)
webhookDB := testWebhookDB(t)
event := seedEvent(t, webhookDB, `{"n":1}`)
event.WebhookID = webhookID
d := seedDatabaseTargetDelivery(
t, webhookDB, event, `{"expiry":"1h"}`,
)
env.eng.ExportDeliverDatabase(webhookDB, d)
require.True(
t, env.eng.ExportArchiveHandleOpen(webhookID),
"the delivery must leave the archive handle open",
)
env.sweeper.ExportSweep(context.Background())
require.True(
t, env.eng.ExportHasArchiveWriter(webhookID),
"the delivery's registry entry must survive the sweep",
)
assert.False(
t, env.eng.ExportArchiveHandleOpen(webhookID),
"the sweep must leave the archive closed",
)
}
// TestArchiveSweep_NeverExpiryUntouched proves the sweep is a
// no-op for the default retention policy, so archives with no
// expiry (or the literal "never") behave exactly as before.
func TestArchiveSweep_NeverExpiryUntouched(t *testing.T) {
t.Parallel()
for _, configJSON := range []string{
`{"expiry":"never"}`,
`{"expiry":""}`,
"",
} {
env := setupSweeperTest(t)
webhookID := env.seedDatabaseTarget(t, configJSON)
path := env.seedArchiveRows(
t, webhookID,
time.Now().Add(-10000*time.Hour),
)
env.sweeper.ExportSweep(context.Background())
assert.Equal(
t, []string{sweepRowOld}, archivedEventIDs(t, path),
"config %q must keep rows forever", configJSON,
)
assert.False(
t, env.eng.ExportHasArchiveWriter(webhookID),
"config %q must leave no registry entry behind",
configJSON,
)
}
}
// TestArchiveSweep_NeverExpirySkipsBeforeOpening pins the
// expiry <= 0 boundary in sweepTarget, which the row assertions
// above cannot reach: pruning is separately gated on a positive
// expiry, so a "never" archive keeps its rows even if the sweep
// does open it.
//
// The spec is stronger than that — a "never" archive is skipped
// before any file is touched — so the archive here exists but has
// never been migrated. Opening it at all would run AutoMigrate
// and create the archive table, which is exactly what must not
// happen.
func TestArchiveSweep_NeverExpirySkipsBeforeOpening(
t *testing.T,
) {
t.Parallel()
env := setupSweeperTest(t)
webhookID := env.seedDatabaseTarget(t, `{"expiry":"never"}`)
path := env.archivePath(webhookID)
seedUnmigratedArchive(t, path)
require.False(t, archiveTableExists(t, path))
env.sweeper.ExportSweep(context.Background())
assert.False(
t, archiveTableExists(t, path),
"a never-expiry archive must not be opened at all",
)
}
// seedUnmigratedArchive creates an archive file that exists but
// carries no archive schema, so any open of it is observable: the
// archive table appears only if something ran AutoMigrate.
func seedUnmigratedArchive(t *testing.T, path string) {
t.Helper()
sqlDB, err := sql.Open(
"sqlite", fmt.Sprintf("file:%s?mode=rwc", path),
)
require.NoError(t, err)
_, err = sqlDB.ExecContext(
t.Context(), "CREATE TABLE placeholder (id INTEGER)",
)
require.NoError(t, err)
require.NoError(t, sqlDB.Close())
}
// archiveTableExists reports whether an archive file has had the
// archive schema migrated into it.
func archiveTableExists(t *testing.T, path string) bool {
t.Helper()
return openArchiveDBForRead(t, path).
Migrator().
HasTable(&delivery.ExportArchivedEvent{})
}
// TestArchiveSweep_DoesNotCreateArchiveFile proves the sweep
// never conjures an archive: a webhook with a database target
// that has never received an event must still have no archive
// file (nor SQLite sidecar) after a sweep.
func TestArchiveSweep_DoesNotCreateArchiveFile(t *testing.T) {
t.Parallel()
env := setupSweeperTest(t)
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
path := env.archivePath(webhookID)
require.NoFileExists(t, path)
env.sweeper.ExportSweep(context.Background())
for _, suffix := range archiveFileSuffixes() {
assert.NoFileExists(
t, path+suffix,
"the sweep must not create an archive file",
)
}
}
// TestArchiveSweep_DoesNotCreateAfterWriterExists covers the
// same guarantee once a writer is cached in the registry but
// the file itself is still absent (for instance because the
// operator moved the archive away).
func TestArchiveSweep_DoesNotCreateAfterWriterExists(
t *testing.T,
) {
t.Parallel()
env := setupSweeperTest(t)
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
path, err := env.eng.ExportEnsureArchiveWriter(webhookID)
require.NoError(t, err)
require.NoFileExists(t, path)
env.sweeper.ExportSweep(context.Background())
assert.NoFileExists(t, path)
}
// TestArchiveSweep_SkipsDeletedWebhookTargets proves that the
// sweep ignores targets soft-deleted along with their webhook,
// so a deleted webhook's archive is never reopened.
func TestArchiveSweep_SkipsDeletedWebhookTargets(t *testing.T) {
t.Parallel()
env := setupSweeperTest(t)
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
path := env.seedArchiveRows(
t, webhookID, time.Now().Add(-48*time.Hour),
)
require.NoError(
t,
env.mainDB.DB().
Where("webhook_id = ?", webhookID).
Delete(&database.Target{}).Error,
)
env.sweeper.ExportSweep(context.Background())
assert.Equal(
t, []string{sweepRowOld}, archivedEventIDs(t, path),
"a deleted target's archive must be left alone",
)
}
// TestArchiveSweep_ConcurrentWrites proves the sweep serialises
// against writes through the per-webhook writer mutex. Run
// under -race, an unsynchronised sweep would be caught here.
func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
t.Parallel()
env := setupSweeperTest(t)
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
webhookDB := testWebhookDB(t)
// The deliveries are seeded up front, on the test's own
// goroutine: the seed helpers assert, and testify assertions
// must not run off the test goroutine.
deliveries := make(
[]*database.Delivery, 0, sweepConcurrentWrites,
)
for range sweepConcurrentWrites {
event := seedEvent(t, webhookDB, `{"n":1}`)
event.WebhookID = webhookID
deliveries = append(
deliveries,
seedDatabaseTargetDelivery(
t, webhookDB, event, `{"expiry":"1h"}`,
),
)
}
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
for _, d := range deliveries {
env.eng.ExportDeliverDatabase(webhookDB, d)
}
}()
go func() {
defer wg.Done()
for range sweepConcurrentWrites {
env.sweeper.ExportSweep(context.Background())
}
}()
wg.Wait()
assert.FileExists(t, env.archivePath(webhookID))
}
// TestArchiveSweeper_StopsCleanly proves the background loop
// exits on OnStop rather than leaking a goroutine.
func TestArchiveSweeper_StopsCleanly(t *testing.T) {
t.Parallel()
env := setupSweeperTest(t)
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
env.seedArchiveRows(
t, webhookID, time.Now().Add(-48*time.Hour),
)
env.sweeper.ExportSetInterval(time.Millisecond)
env.sweeper.ExportStart()
// stop blocks on the loop's WaitGroup, so returning without
// error proves the loop observed the cancellation and exited
// well inside the stop context.
require.NoError(
t, env.sweeper.ExportStop(context.Background()),
)
}
// TestArchiveSweeper_StopHookHonoursStopTimeout is the sweeper's
// half of the same shutdown defect the engine and the retention
// reaper carried: an OnStop that discards its context and waits
// on the WaitGroup bare hangs the process forever on a prune
// wedged inside a locked archive.
func TestArchiveSweeper_StopHookHonoursStopTimeout(
t *testing.T,
) {
t.Parallel()
env := setupSweeperTest(t)
lc := &recordingLifecycle{}
env.sweeper.ExportRegisterHooks(lc)
require.Len(t, lc.hooks, 1)
require.NoError(t, lc.hooks[0].OnStart(context.Background()))
release := make(chan struct{})
t.Cleanup(func() { close(release) })
env.sweeper.ExportWedgeLoop(release)
requireStopHookExpires(t, lc.hooks[0], "archive sweeper")
}
internal/delivery/circuit_breaker.go
+14
View File
@@ -102,6 +102,20 @@ func (cb *CircuitBreaker) CooldownRemaining() time.Duration {
return remaining
}
// StateAndCooldown returns the circuit state and, while the circuit is
// open, what is left of the cooldown, or zero once that has passed.
// Both are read under one lock, so they always agree.
func (cb *CircuitBreaker) StateAndCooldown() (CircuitState, time.Duration) {
cb.mu.Lock()
defer cb.mu.Unlock()
if cb.state != CircuitOpen {
return cb.state, 0
}
return cb.state, max(cb.cooldown-time.Since(cb.lastFailure), 0)
}
// RecordSuccess records a successful delivery and resets
// the circuit breaker to closed state.
func (cb *CircuitBreaker) RecordSuccess() {
internal/delivery/client_ssrf_test.go
+3 -19
View File
@@ -46,12 +46,7 @@ func TestClientForRequest_TimeoutKeepsSSRFGuard(t *testing.T) {
t.Run(target, func(t *testing.T) {
t.Parallel()
cfg := &delivery.HTTPTargetConfig{
URL: target,
Timeout: 5,
}
client := engine.ExportClientForRequest(cfg, nil)
client := engine.ExportClientForRequest(5, nil)
require.NotSame(t, engine.ExportClient(), client,
"a per-target timeout must yield a "+
@@ -101,11 +96,7 @@ func TestClientForRequest_NoTimeoutUnchanged(t *testing.T) {
engine := newSSRFTestEngine()
cfg := &delivery.HTTPTargetConfig{
URL: "https://example.com/hook",
}
client := engine.ExportClientForRequest(cfg, nil)
client := engine.ExportClientForRequest(0, nil)
assert.Same(t, engine.ExportClient(), client,
"without a per-target timeout the shared client "+
@@ -123,15 +114,8 @@ func TestClientForRequest_HeadersKeepSSRFGuard(t *testing.T) {
engine := newSSRFTestEngine()
cfg := &delivery.HTTPTargetConfig{
URL: "https://example.com/with-headers",
Headers: map[string]string{
"X-Api-Key": "configured",
},
}
client := engine.ExportClientForRequest(
cfg, []string{"X-Api-Key"},
0, []string{probeHeaderName},
)
require.NotNil(t, client.CheckRedirect,
internal/delivery/delivery_totals_test.go
+188
View File
@@ -0,0 +1,188 @@
package delivery_test
import (
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
"sneak.berlin/go/webhooker/internal/database"
)
// targetTotals reads one target's totals from a webhook database, all
// zero when it has no row.
func targetTotals(
t *testing.T, db *gorm.DB, targetID string,
) database.TargetTotals {
t.Helper()
var rows []database.TargetTotals
require.NoError(t, db.Where("target_id = ?", targetID).
Find(&rows).Error)
if len(rows) == 0 {
return database.TargetTotals{TargetID: targetID}
}
return rows[0]
}
// TestUpdateDeliveryStatus_FinishTimeAndTargetTotals pins what a status
// write records for the webhook page's statistics: the time a delivery
// finished, set only when it becomes delivered or failed and stored in
// the same form as created_at, and one more on its target's delivered
// or failed total.
func TestUpdateDeliveryStatus_FinishTimeAndTargetTotals(t *testing.T) {
t.Parallel()
tests := []struct {
status database.DeliveryStatus
finished bool
delivered int64
failed int64
}{
{database.DeliveryStatusRetrying, false, 0, 0},
{database.DeliveryStatusDelivered, true, 1, 0},
{database.DeliveryStatusFailed, true, 0, 1},
}
for _, tt := range tests {
t.Run(string(tt.status), func(t *testing.T) {
t.Parallel()
db := testWebhookDB(t)
e := testEngine(t, 1)
event := seedEvent(t, db, `{}`)
targetID := uuid.New().String()
d := seedDelivery(
t, db, event.ID, targetID,
database.DeliveryStatusPending,
)
before := time.Now()
require.NoError(t, e.ExportUpdateDeliveryStatus(
db, &d, tt.status,
))
var stored database.Delivery
require.NoError(t, db.First(&stored, "id = ?", d.ID).Error)
assert.Equal(t, tt.status, stored.Status)
if tt.finished {
require.NotNil(t, stored.FinishedAt)
assert.False(t, stored.FinishedAt.Before(before))
var createdAt, finishedAt string
require.NoError(t, db.Raw(
`SELECT CAST(created_at AS TEXT),
CAST(finished_at AS TEXT)
FROM deliveries WHERE id = ?`, d.ID,
).Row().Scan(&createdAt, &finishedAt))
assertStoredTime(t, createdAt)
assertStoredTime(t, finishedAt)
} else {
assert.Nil(t, stored.FinishedAt)
}
assert.Equal(t, database.TargetTotals{
TargetID: targetID,
Delivered: tt.delivered,
Failed: tt.failed,
}, targetTotals(t, db, targetID))
})
}
}
// TestUpdateDeliveryStatus_DeletedDeliveryIsNotCounted covers a
// delivery retention deleted while the engine still held it. Failing
// it afterwards writes no row, so it adds no failure either: retention
// has already counted what it removed.
func TestUpdateDeliveryStatus_DeletedDeliveryIsNotCounted(t *testing.T) {
t.Parallel()
db := testWebhookDB(t)
e := testEngine(t, 1)
event := seedEvent(t, db, `{}`)
targetID := uuid.New().String()
d := seedDelivery(
t, db, event.ID, targetID,
database.DeliveryStatusRetrying,
)
require.NoError(t, db.Unscoped().
Delete(&database.Delivery{}, "id = ?", d.ID).Error)
require.NoError(t, e.ExportUpdateDeliveryStatus(
db, &d, database.DeliveryStatusFailed,
))
assert.Equal(t, database.TargetTotals{TargetID: targetID},
targetTotals(t, db, targetID))
}
// TestUpdateDeliveryStatus_FinishedDeliveryIsNotSettledAgain covers a
// delivery settled a second time, as recovery can do when a worker has
// settled it since recovery read it. Neither status writes over the
// first, and the totals do not move.
func TestUpdateDeliveryStatus_FinishedDeliveryIsNotSettledAgain(
t *testing.T,
) {
t.Parallel()
finished := []database.DeliveryStatus{
database.DeliveryStatusDelivered,
database.DeliveryStatusFailed,
}
for _, first := range finished {
t.Run(string(first), func(t *testing.T) {
t.Parallel()
db := testWebhookDB(t)
e := testEngine(t, 1)
event := seedEvent(t, db, `{}`)
targetID := uuid.New().String()
d := seedDelivery(
t, db, event.ID, targetID,
database.DeliveryStatusRetrying,
)
// The delivery as recovery read it, before the worker
// settled it.
readBefore := d
require.NoError(t, e.ExportUpdateDeliveryStatus(
db, &d, first,
))
var settled database.Delivery
require.NoError(t, db.First(&settled, "id = ?", d.ID).Error)
require.NotNil(t, settled.FinishedAt)
totals := targetTotals(t, db, targetID)
for _, again := range finished {
stale := readBefore
require.NoError(t, e.ExportUpdateDeliveryStatus(
db, &stale, again,
))
}
var stored database.Delivery
require.NoError(t, db.First(&stored, "id = ?", d.ID).Error)
assert.Equal(t, first, stored.Status)
require.NotNil(t, stored.FinishedAt)
assert.True(t, settled.FinishedAt.Equal(*stored.FinishedAt))
assert.Equal(t, totals, targetTotals(t, db, targetID))
})
}
}
internal/delivery/engine.go
+383 -491
View File
File diff suppressed because it is too large Load Diff
internal/delivery/engine_integration_test.go
+240 -199
View File
@@ -1,15 +1,14 @@
package delivery_test
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync/atomic"
"testing"
@@ -18,10 +17,10 @@ import (
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
_ "modernc.org/sqlite"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/database/databasetest"
"sneak.berlin/go/webhooker/internal/delivery"
)
@@ -43,12 +42,12 @@ func newISetup(t *testing.T) iSetup {
wDB := iSeedWebhookDB(t, dbMgr, wID)
return iSetup{
MainDB: mainDB,
MainDB: mainDB.DB(),
DBMgr: dbMgr,
WebhookID: wID,
WebhookDB: wDB,
Engine: delivery.NewTestEngineWithDB(
database.NewTestDatabase(mainDB),
mainDB,
dbMgr,
slog.New(slog.NewTextHandler(
os.Stderr,
@@ -62,34 +61,16 @@ func newISetup(t *testing.T) iSetup {
}
}
func iMainDB(t *testing.T) *gorm.DB {
// iMainDB opens a main database through database.Open, the way the
// service opens it, so these tests cannot pass against journal and
// locking settings production does not use.
func iMainDB(t *testing.T) *database.Database {
t.Helper()
dbPath := filepath.Join(
t.TempDir(), "main-test.db",
)
// Opened the way the service opens the main database, so these
// tests cannot pass against journal and locking settings
// production does not use.
sqlDB, err := database.OpenSQLite(
dbPath, database.SQLiteModeCreate,
)
db, err := database.Open(t.TempDir(), slog.New(slog.DiscardHandler))
require.NoError(t, err)
t.Cleanup(func() { _ = sqlDB.Close() })
db, err := gorm.Open(
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
)
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(
&database.Webhook{},
&database.Target{},
&database.User{},
&database.Setting{},
))
t.Cleanup(func() { _ = db.Close() })
return db
}
@@ -99,7 +80,7 @@ func iDBManager(
) *database.WebhookDBManager {
t.Helper()
return database.NewTestWebhookDBManager(t.TempDir())
return databasetest.NewWebhookDBManager(t, t.TempDir())
}
func iSeedWebhookDB(
@@ -115,17 +96,6 @@ func iSeedWebhookDB(
return db
}
func iHTTPConfig(url string) string {
cfg := delivery.HTTPTargetConfig{URL: url}
data, err := json.Marshal(cfg)
if err != nil {
panic("failed to marshal HTTPTargetConfig")
}
return string(data)
}
func iEngine(
t *testing.T, workers int,
) *delivery.Engine {
@@ -187,24 +157,24 @@ func iSeedDelivery(
func iTask(
d database.Delivery,
event database.Event,
webhookID, targetID, name, config string,
maxRetries, attemptNum int,
webhookID, targetID, name, targetURL string,
maxAttempts, attemptNum int,
body *string,
) delivery.Task {
return delivery.Task{
DeliveryID: d.ID,
EventID: event.ID,
WebhookID: webhookID,
TargetID: targetID,
TargetName: name,
TargetType: database.TargetTypeHTTP,
TargetConfig: config,
MaxRetries: maxRetries,
Method: event.Method,
Headers: event.Headers,
ContentType: event.ContentType,
Body: body,
AttemptNum: attemptNum,
DeliveryID: d.ID,
EventID: event.ID,
WebhookID: webhookID,
TargetID: targetID,
TargetName: name,
TargetType: database.TargetTypeHTTP,
TargetURL: targetURL,
MaxAttempts: maxAttempts,
Method: event.Method,
Headers: event.Headers,
ContentType: event.ContentType,
Body: body,
AttemptNum: attemptNum,
}
}
@@ -257,10 +227,9 @@ func TestProcessNewTask_InlineBody(t *testing.T) {
)
bodyStr := event.Body
cfg := iHTTPConfig(ts.URL)
task := iTask(
d, event, s.WebhookID, targetID,
"test-target", cfg, 0, 1, &bodyStr,
"test-target", ts.URL, 1, 1, &bodyStr,
)
s.Engine.ExportProcessNewTask(
@@ -308,10 +277,9 @@ func TestProcessNewTask_LargeBody_FetchFromDB(
database.DeliveryStatusPending,
)
cfg := iHTTPConfig(ts.URL)
task := iTask(
d, event, s.WebhookID, targetID,
"test-large", cfg, 0, 1, nil,
"test-large", ts.URL, 1, 1, nil,
)
s.Engine.ExportProcessNewTask(
@@ -331,16 +299,16 @@ func TestProcessNewTask_InvalidWebhookID(t *testing.T) {
s := newISetup(t)
task := delivery.Task{
DeliveryID: uuid.New().String(),
EventID: uuid.New().String(),
WebhookID: uuid.New().String(),
TargetID: uuid.New().String(),
TargetName: "test",
TargetType: database.TargetTypeHTTP,
TargetConfig: iHTTPConfig("http://localhost:9999"),
MaxRetries: 0,
Body: nil,
AttemptNum: 1,
DeliveryID: uuid.New().String(),
EventID: uuid.New().String(),
WebhookID: uuid.New().String(),
TargetID: uuid.New().String(),
TargetName: "test",
TargetType: database.TargetTypeHTTP,
TargetURL: "http://localhost:9999",
MaxAttempts: 1,
Body: nil,
AttemptNum: 1,
}
s.Engine.ExportProcessNewTask(
@@ -355,9 +323,14 @@ func TestProcessRetryTask_SuccessfulRetry(t *testing.T) {
s := newISetup(t)
var receivedBody string
ts := httptest.NewServer(
http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
receivedBody = string(body)
w.WriteHeader(http.StatusOK)
},
),
@@ -376,7 +349,6 @@ func TestProcessRetryTask_SuccessfulRetry(t *testing.T) {
)
bodyStr := event.Body
cfg := iHTTPConfig(ts.URL)
// The target row exists because the engine confirms a scheduled
// retry's target has not been deleted before it runs it. A retry
@@ -385,18 +357,20 @@ func TestProcessRetryTask_SuccessfulRetry(t *testing.T) {
// task. See https://git.eeqj.de/sneak/webhooker/issues/107.
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "retry-target",
database.TargetTypeHTTP, cfg, 5,
database.TargetTypeHTTP, ts.URL, 5,
)
task := iTask(
d, event, s.WebhookID, targetID,
"retry-target", cfg, 5, 2, &bodyStr,
"retry-target", ts.URL, 5, 2, &bodyStr,
)
s.Engine.ExportProcessRetryTask(
context.TODO(), &task,
)
assert.Equal(t, event.Body, receivedBody)
iAssertStatus(t, s.WebhookDB, d.ID,
database.DeliveryStatusDelivered,
)
@@ -421,10 +395,9 @@ func TestProcessRetryTask_SkipsNonRetryingDelivery(
)
bodyStr := event.Body
cfg := iHTTPConfig("http://localhost:1")
task := iTask(
d, event, s.WebhookID, targetID,
"skip-target", cfg, 5, 2, &bodyStr,
"skip-target", "http://localhost:1", 5, 2, &bodyStr,
)
s.Engine.ExportProcessRetryTask(
@@ -443,9 +416,14 @@ func TestProcessRetryTask_LargeBody_FetchFromDB(
s := newISetup(t)
var receivedBody string
ts := httptest.NewServer(
http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
receivedBody = string(body)
w.WriteHeader(http.StatusOK)
},
),
@@ -466,22 +444,22 @@ func TestProcessRetryTask_LargeBody_FetchFromDB(
database.DeliveryStatusRetrying,
)
cfg := iHTTPConfig(ts.URL)
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "retry-large",
database.TargetTypeHTTP, cfg, 5,
database.TargetTypeHTTP, ts.URL, 5,
)
task := iTask(
d, event, s.WebhookID, targetID,
"retry-large", cfg, 5, 2, nil,
"retry-large", ts.URL, 5, 2, nil,
)
s.Engine.ExportProcessRetryTask(
context.TODO(), &task,
)
assert.Equal(t, largeBody, receivedBody)
iAssertStatus(t, s.WebhookDB, d.ID,
database.DeliveryStatusDelivered,
)
@@ -510,11 +488,11 @@ func TestWorkerLifecycle_StartStop(t *testing.T) {
task := iTask(
d, event, s.WebhookID, targetID,
"lifecycle-test", "",
0, 1, &bodyStr,
1, 1, &bodyStr,
)
task.TargetType = database.TargetTypeLog
s.Engine.Notify([]delivery.Task{task})
s.Engine.ExportNotify([]delivery.Task{task})
iWaitForDelivered(t, s.WebhookDB, d.ID)
@@ -574,16 +552,15 @@ func TestWorkerLifecycle_ProcessesRetryChannel(
s.Engine.ExportStart()
bodyStr := event.Body
cfg := iHTTPConfig(ts.URL)
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "retry-chan-test",
database.TargetTypeHTTP, cfg, 5,
database.TargetTypeHTTP, ts.URL, 5,
)
task := iTask(
d, event, s.WebhookID, targetID,
"retry-chan-test", cfg, 5, 2, &bodyStr,
"retry-chan-test", ts.URL, 5, 2, &bodyStr,
)
s.Engine.ExportRetryCh() <- task
@@ -649,16 +626,21 @@ func TestRecoverPendingDeliveries(t *testing.T) {
iCreateTarget(t, s.MainDB, targetID,
s.WebhookID, "recovery-target",
database.TargetTypeLog, "", 0,
database.TargetTypeLog, "", 1,
)
iSeedPendingDeliveries(
t, s.WebhookDB, s.WebhookID, targetID, 3,
)
// A recovered delivery still carries its event's query string.
require.NoError(t, s.WebhookDB.Model(&database.Event{}).
Where("webhook_id = ?", s.WebhookID).
Update("raw_query", eventQuery).Error)
s.Engine.ExportRecoverPendingDeliveries(
context.Background(), s.WebhookDB,
s.WebhookID,
s.WebhookID, atStart,
)
for i := range 3 {
@@ -670,6 +652,8 @@ func TestRecoverPendingDeliveries(t *testing.T) {
database.TargetTypeLog,
task.TargetType,
)
assert.Equal(t, eventQuery, task.RawQuery)
case <-time.After(2 * time.Second):
t.Fatalf("expected task %d", i)
}
@@ -682,18 +666,18 @@ func iCreateTarget(
mainDB *gorm.DB,
targetID, webhookID, name string,
targetType database.TargetType,
config string,
maxRetries int,
targetURL string,
maxAttempts int,
) {
t.Helper()
target := database.Target{
WebhookID: webhookID,
Name: name,
Type: targetType,
Active: true,
Config: config,
MaxRetries: maxRetries,
WebhookID: webhookID,
Name: name,
Type: targetType,
Active: true,
URL: targetURL,
MaxAttempts: maxAttempts,
}
target.ID = targetID
@@ -733,7 +717,7 @@ func TestRecoverWebhookDeliveries_RetryingDeliveries(
iCreateTarget(t, s.MainDB, targetID,
s.WebhookID, "retry-recovery",
database.TargetTypeHTTP,
iHTTPConfig("http://example.com/hook"), 5,
"http://example.com/hook", 5,
)
event := iSeedEvent(
@@ -753,7 +737,7 @@ func TestRecoverWebhookDeliveries_RetryingDeliveries(
)
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID,
context.Background(), s.WebhookID, atStart,
)
select {
@@ -793,7 +777,7 @@ func iSeedRetryingWithType(
iCreateTarget(t, s.MainDB, targetID,
s.WebhookID, "mutated-target", targetType,
iHTTPConfig("http://example.com/hook"), 5,
"http://example.com/hook", 5,
)
event := iSeedEvent(
@@ -877,7 +861,7 @@ func TestRecoverSingleRetry_TypeNoLongerRetries(
)
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID,
context.Background(), s.WebhookID, atStart,
)
iAssertTerminallyFailed(
@@ -885,7 +869,7 @@ func TestRecoverSingleRetry_TypeNoLongerRetries(
)
}
func TestSweepSingleRetry_TypeNoLongerRetries(
func TestRecoverSingleRetry_OnTick_TypeNoLongerRetries(
t *testing.T,
) {
t.Parallel()
@@ -900,8 +884,8 @@ func TestSweepSingleRetry_TypeNoLongerRetries(
t, s, database.TargetTypeDatabase,
)
s.Engine.ExportSweepWebhookRetries(
context.Background(), s.WebhookID,
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID, onTick,
)
iAssertTerminallyFailed(
@@ -931,14 +915,14 @@ func TestFailUnretryableRetry_WritesNoTargetRow(
targetID := uuid.New().String()
// A Slack incoming-webhook URL: the target config IS the
// A Slack incoming-webhook URL: the target's URL IS the
// credential, which is what makes a leaked target row a
// disclosure rather than a curiosity.
hookURL := "https://hooks.slack.com/services/T00/B00/x"
iCreateTarget(t, s.MainDB, targetID,
s.WebhookID, "credential-bearing",
database.TargetTypeLog, iHTTPConfig(hookURL), 5,
database.TargetTypeLog, hookURL, 5,
)
event := iSeedEvent(
@@ -952,37 +936,20 @@ func TestFailUnretryableRetry_WritesNoTargetRow(
iSeedFailedResult(t, s.WebhookDB, d.ID)
s.Engine.ExportSweepWebhookRetries(
context.Background(), s.WebhookID,
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID, onTick,
)
iAssertStatus(t, s.WebhookDB, d.ID,
database.DeliveryStatusFailed,
)
// The table exists in the per-webhook database because GORM
// migrates the Delivery relation's model alongside it. It must
// stay empty.
var targetRows int64
require.NoError(t, s.WebhookDB.
Table("targets").
Count(&targetRows).Error)
assert.Zero(t, targetRows,
// The per-webhook database has no targets table, so writing a
// target row there would have failed the status write above.
assert.False(t, s.WebhookDB.Migrator().HasTable("targets"),
"orphaned-retry terminal failure wrote a target row "+
"into the per-webhook event database",
)
var configs []string
require.NoError(t, s.WebhookDB.
Table("targets").
Pluck("config", &configs).Error)
assert.NotContains(
t, strings.Join(configs, " "), hookURL,
)
}
func TestRecoverSingleRetry_UnknownTargetType(
@@ -1001,13 +968,13 @@ func TestRecoverSingleRetry_UnknownTargetType(
deliveryID := iSeedRetryingWithType(t, s, unknown)
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID,
context.Background(), s.WebhookID, atStart,
)
iAssertTerminallyFailed(t, s, deliveryID, unknown)
}
func TestSweepSingleRetry_UnknownTargetType(
func TestRecoverSingleRetry_OnTick_UnknownTargetType(
t *testing.T,
) {
t.Parallel()
@@ -1022,8 +989,8 @@ func TestSweepSingleRetry_UnknownTargetType(
deliveryID := iSeedRetryingWithType(t, s, unknown)
s.Engine.ExportSweepWebhookRetries(
context.Background(), s.WebhookID,
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID, onTick,
)
iAssertTerminallyFailed(t, s, deliveryID, unknown)
@@ -1065,19 +1032,19 @@ func iCreateWebhook(
require.NoError(t, mainDB.Create(&webhook).Error)
}
// --- recoverInFlight Tests ---
// --- recoverDeliveries Tests ---
func TestRecoverInFlight_NoWebhooks(t *testing.T) {
func TestRecoverDeliveries_NoWebhooks(t *testing.T) {
t.Parallel()
s := newISetup(t)
s.Engine.ExportRecoverInFlight(
context.Background(),
s.Engine.ExportRecoverDeliveries(
context.Background(), atStart,
)
}
func TestRecoverInFlight_WithPendingDeliveries(
func TestRecoverDeliveries_WithPendingDeliveries(
t *testing.T,
) {
t.Parallel()
@@ -1091,7 +1058,7 @@ func TestRecoverInFlight_WithPendingDeliveries(
iCreateTarget(t, s.MainDB, targetID,
s.WebhookID, "recover-target",
database.TargetTypeLog, "", 0,
database.TargetTypeLog, "", 1,
)
event := iSeedEvent(
@@ -1104,8 +1071,8 @@ func TestRecoverInFlight_WithPendingDeliveries(
database.DeliveryStatusPending,
)
s.Engine.ExportRecoverInFlight(
context.Background(),
s.Engine.ExportRecoverDeliveries(
context.Background(), atStart,
)
select {
@@ -1116,11 +1083,91 @@ func TestRecoverInFlight_WithPendingDeliveries(
database.TargetTypeLog, task.TargetType,
)
case <-time.After(2 * time.Second):
t.Fatal("expected task from recoverInFlight")
t.Fatal("expected task from recovery at start")
}
}
// --- HTTP Config with custom headers ---
// TestRecoverDeliveries_ReportsAMissingWebhookDatabase covers a webhook
// whose database file is gone, after a partial restore say. Recovery at
// start opens every webhook's database, so the empty one made in its
// place is reported at start, naming the file
// (https://git.eeqj.de/sneak/webhooker/issues/290).
func TestRecoverDeliveries_ReportsAMissingWebhookDatabase(t *testing.T) {
t.Parallel()
mainDB := iMainDB(t)
webhookID := uuid.New().String()
iCreateWebhook(t, mainDB.DB(), webhookID, "lost-database")
var logs bytes.Buffer
dbMgr := databasetest.NewWebhookDBManagerWithLogger(
t, t.TempDir(), slog.New(slog.NewTextHandler(&logs, nil)),
)
t.Cleanup(func() { _ = dbMgr.CloseAll() })
engine := delivery.NewTestEngineWithDB(
mainDB, dbMgr,
slog.New(slog.DiscardHandler),
&http.Client{Timeout: 5 * time.Second}, 1,
)
engine.ExportRecoverDeliveries(context.Background(), atStart)
assert.Contains(
t, logs.String(),
`level=WARN msg="created a new, empty database" webhook_id=`+
webhookID+" path="+dbMgr.DBPath(webhookID),
)
}
// TestRecoverDeliveries_SkipsAWebhookDeletedAfterTheListIsRead covers a
// webhook deleted from the web interface while the recovery routine
// runs. Its database file is gone, and recovery must not create it
// again.
func TestRecoverDeliveries_SkipsAWebhookDeletedAfterTheListIsRead(
t *testing.T,
) {
t.Parallel()
mainDB := iMainDB(t)
webhookID := uuid.New().String()
iCreateWebhook(t, mainDB.DB(), webhookID, "deleted-during-recovery")
// The first query to return is recovery's read of the list of
// webhooks. Deleting the webhook right after it puts the delete
// between that read and the opening of the webhook's database.
deleted := false
require.NoError(t, mainDB.DB().Callback().Query().After("gorm:query").
Register("delete-after-list", func(*gorm.DB) {
if deleted {
return
}
deleted = true
require.NoError(t, mainDB.DB().Delete(
&database.Webhook{}, "id = ?", webhookID,
).Error)
}))
dbMgr := databasetest.NewWebhookDBManager(t, t.TempDir())
t.Cleanup(func() { _ = dbMgr.CloseAll() })
engine := delivery.NewTestEngineWithDB(
mainDB, dbMgr,
slog.New(slog.DiscardHandler),
&http.Client{Timeout: 5 * time.Second}, 1,
)
engine.ExportRecoverDeliveries(context.Background(), atStart)
require.True(t, deleted)
assert.False(t, dbMgr.DBExists(webhookID))
}
// --- HTTP target with custom headers ---
func TestDeliverHTTP_CustomTargetHeaders(t *testing.T) {
t.Parallel()
@@ -1142,16 +1189,6 @@ func TestDeliverHTTP_CustomTargetHeaders(t *testing.T) {
)
defer ts.Close()
cfg := delivery.HTTPTargetConfig{
URL: ts.URL,
Headers: map[string]string{
"Authorization": "Bearer secret-token",
},
}
cfgJSON, err := json.Marshal(cfg)
require.NoError(t, err)
event := iSeedEvent(
t, s.WebhookDB, s.WebhookID,
`{"auth":"test"}`,
@@ -1166,9 +1203,12 @@ func TestDeliverHTTP_CustomTargetHeaders(t *testing.T) {
bodyStr := event.Body
task := iTask(
d, event, s.WebhookID, targetID,
"auth-target", string(cfgJSON),
0, 1, &bodyStr,
"auth-target", ts.URL,
1, 1, &bodyStr,
)
task.TargetHeaders = map[string]string{
headerAuthorization: "Bearer secret-token",
}
s.Engine.ExportProcessNewTask(
context.TODO(), &task,
@@ -1181,11 +1221,14 @@ func TestDeliverHTTP_CustomTargetHeaders(t *testing.T) {
// --- HTTP delivery with custom timeout ---
// TestDeliverHTTP_TargetTimeout proves a task's TimeoutSeconds reaches
// the delivery through the engine. The destination answers after two
// seconds, inside the engine's own five-second client timeout, so only
// the task's one second can fail the delivery.
func TestDeliverHTTP_TargetTimeout(t *testing.T) {
t.Parallel()
db := testWebhookDB(t)
e := iEngine(t, 1)
s := newISetup(t)
ts := httptest.NewServer(
http.HandlerFunc(
@@ -1197,31 +1240,30 @@ func TestDeliverHTTP_TargetTimeout(t *testing.T) {
)
defer ts.Close()
cfg := delivery.HTTPTargetConfig{
URL: ts.URL,
Timeout: 1,
}
event := iSeedEvent(
t, s.WebhookDB, s.WebhookID, `{"timeout":"test"}`,
)
targetID := uuid.New().String()
cfgJSON, err := json.Marshal(cfg)
require.NoError(t, err)
event, del := iSeedEventAndDelivery(
t, db, `{"timeout":"test"}`,
string(cfgJSON),
d := iSeedDelivery(
t, s.WebhookDB, event.ID, targetID,
database.DeliveryStatusPending,
)
task, d := iHTTPTaskAndDelivery(
event, del, "timeout-target",
string(cfgJSON), 0, 1,
bodyStr := event.Body
task := iTask(
d, event, s.WebhookID, targetID,
"timeout-target", ts.URL, 1, 1, &bodyStr,
)
task.TimeoutSeconds = 1
e.ExportDeliverHTTP(context.TODO(), db, d, task)
s.Engine.ExportProcessNewTask(context.TODO(), &task)
iAssertStatus(t, db, del.ID,
iAssertStatus(t, s.WebhookDB, d.ID,
database.DeliveryStatusFailed,
)
iAssertResultFailed(t, db, del.ID)
iAssertResultFailed(t, s.WebhookDB, d.ID)
}
// TestDeliverHTTP_CutsStoredResponseAtMaxBodyLog pins the size
@@ -1259,8 +1301,6 @@ func TestDeliverHTTP_CutsStoredResponseAtMaxBodyLog(
))
defer ts.Close()
cfgJSON := iHTTPConfig(ts.URL)
event := iSeedEvent(
t, s.WebhookDB, s.WebhookID, `{"cut":"test"}`,
)
@@ -1274,7 +1314,7 @@ func TestDeliverHTTP_CutsStoredResponseAtMaxBodyLog(
bodyStr := event.Body
task := iTask(
del, event, s.WebhookID, targetID,
"cut-target", cfgJSON, 0, 1, &bodyStr,
"cut-target", ts.URL, 1, 1, &bodyStr,
)
s.Engine.ExportProcessNewTask(context.TODO(), &task)
@@ -1306,7 +1346,7 @@ func TestDeliverHTTP_CutsStoredResponseAtMaxBodyLog(
func iSeedEventAndDelivery(
t *testing.T,
db *gorm.DB,
body, _ string,
body string,
) (database.Event, database.Delivery) {
t.Helper()
@@ -1337,19 +1377,19 @@ func iSeedEventAndDelivery(
func iHTTPTaskAndDelivery(
event database.Event,
del database.Delivery,
name, config string,
maxRetries, attemptNum int,
name, targetURL string,
maxAttempts, attemptNum int,
) (*delivery.Task, *database.Delivery) {
task := &delivery.Task{
DeliveryID: del.ID,
EventID: event.ID,
WebhookID: event.WebhookID,
TargetID: del.TargetID,
TargetName: name,
TargetType: database.TargetTypeHTTP,
TargetConfig: config,
MaxRetries: maxRetries,
AttemptNum: attemptNum,
DeliveryID: del.ID,
EventID: event.ID,
WebhookID: event.WebhookID,
TargetID: del.TargetID,
TargetName: name,
TargetType: database.TargetTypeHTTP,
TargetURL: targetURL,
MaxAttempts: maxAttempts,
AttemptNum: attemptNum,
}
d := &database.Delivery{
@@ -1358,9 +1398,10 @@ func iHTTPTaskAndDelivery(
Status: database.DeliveryStatusPending,
Event: event,
Target: database.Target{
Name: name,
Type: database.TargetTypeHTTP,
Config: config,
Name: name,
Type: database.TargetTypeHTTP,
URL: targetURL,
MaxAttempts: maxAttempts,
},
}
d.ID = del.ID
@@ -1388,30 +1429,30 @@ func iAssertResultFailed(
assert.NotEmpty(t, result.Error)
}
// --- HTTP request with invalid config ---
// --- HTTP request with no URL ---
func TestDeliverHTTP_InvalidConfig(t *testing.T) {
// TestDeliverHTTP_NoURL proves an http target without a URL, which
// the forms never store, fails its delivery and records why.
func TestDeliverHTTP_NoURL(t *testing.T) {
t.Parallel()
db := testWebhookDB(t)
e := iEngine(t, 1)
event, del := iSeedEventAndDelivery(
t, db, `{"config":"invalid"}`, "",
)
event, del := iSeedEventAndDelivery(t, db, `{"url":"none"}`)
task, d := iHTTPTaskAndDelivery(
event, del, "bad-config", `not-json`, 0, 1,
)
task, d := iHTTPTaskAndDelivery(event, del, "no-url", "", 1, 1)
e.ExportDeliverHTTP(context.TODO(), db, d, task)
iAssertStatus(t, db, del.ID,
database.DeliveryStatusFailed,
)
iAssertResultFailed(t, db, del.ID)
}
// --- Notify batching ---
// --- notify batching ---
func TestNotify_MultipleTasks(t *testing.T) {
t.Parallel()
@@ -1426,7 +1467,7 @@ func TestNotify_MultipleTasks(t *testing.T) {
}
}
e.Notify(tasks)
e.ExportNotify(tasks)
for i := range 5 {
select {
internal/delivery/engine_lifecycle_test.go
+187 -18
View File
@@ -2,14 +2,19 @@ package delivery_test
import (
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"path/filepath"
"sync"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/fx"
"gorm.io/gorm/clause"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/delivery"
)
@@ -24,7 +29,7 @@ const (
// OnStart before the caller may enqueue work. A worker pool
// wrongly rooted in the already-done hook context has nothing
// but ctx.Done() ready in its select, so it is deterministically
// gone by the end of this window. Without the wait, Notify would
// gone by the end of this window. Without the wait, notify would
// race the pool's very first select, in which a ready ctx.Done()
// and a ready deliveryCh are chosen between at random and a
// doomed pool still delivers.
@@ -94,7 +99,7 @@ func requireStopHookExpires(
// the caller can drive OnStop too.
//
// Callers must not seed pending or retrying deliveries before
// calling this: restart recovery enqueues those during startup,
// calling this: the recovery routine's run at start enqueues those,
// which would put work in the queue while the pool is still
// racing its first select.
func startEngineViaHook(
@@ -144,7 +149,7 @@ func seedLogTask(
bodyStr := event.Body
task := iTask(
d, event, s.WebhookID, targetID,
"hook-context-test", "", 0, 1, &bodyStr,
"hook-context-test", "", 1, 1, &bodyStr,
)
task.TargetType = database.TargetTypeLog
@@ -173,11 +178,11 @@ func TestEngine_WorkersOutliveStartHookContext(t *testing.T) {
_ = lc.hooks[0].OnStop(context.Background())
})
// Seeded only after the pool has settled, so restart recovery
// cannot enqueue it during startup.
// Seeded only after the pool has settled, so the recovery
// routine's run at start cannot enqueue it.
deliveryID, task := seedLogTask(t, s)
s.Engine.Notify([]delivery.Task{task})
s.Engine.ExportNotify([]delivery.Task{task})
iWaitForDelivered(t, s.WebhookDB, deliveryID)
}
@@ -197,7 +202,7 @@ func TestEngine_StopHookStopsWorkers(t *testing.T) {
// Let the pool prove it is running before stopping it, so a
// fast OnStop cannot pass by stopping something already dead.
firstID, firstTask := seedLogTask(t, s)
s.Engine.Notify([]delivery.Task{firstTask})
s.Engine.ExportNotify([]delivery.Task{firstTask})
iWaitForDelivered(t, s.WebhookDB, firstID)
var stopErr error
@@ -227,7 +232,7 @@ func TestEngine_StopHookStopsWorkers(t *testing.T) {
// With every worker gone, a freshly notified task must sit
// untouched in the queue rather than being delivered.
secondID, secondTask := seedLogTask(t, s)
s.Engine.Notify([]delivery.Task{secondTask})
s.Engine.ExportNotify([]delivery.Task{secondTask})
time.Sleep(200 * time.Millisecond)
@@ -272,22 +277,184 @@ func TestEngine_StopHookHonoursStopTimeout(t *testing.T) {
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
}
// deliverToArchive runs one delivery to a database target through
// the running engine and returns the webhook's archive file path.
// The archive writer holds the file open afterwards.
// seedHeldDelivery gives the setup's webhook an http target with
// maxAttempts, pointed at a receiver that keeps every request open until
// the client gives up on it, and seeds one delivery to that target at
// status. It returns the task that sends attempt attemptNum of the
// delivery, and a channel that is closed when a request reaches the
// receiver.
func seedHeldDelivery(
t *testing.T,
s iSetup,
maxAttempts int,
status database.DeliveryStatus,
attemptNum int,
) (delivery.Task, <-chan struct{}) {
t.Helper()
arrived := make(chan struct{})
var once sync.Once
ts := httptest.NewServer(http.HandlerFunc(
func(_ http.ResponseWriter, r *http.Request) {
// The server notices the client hanging up only once the
// body has been read; until then r.Context() never ends.
_, _ = io.Copy(io.Discard, r.Body)
once.Do(func() { close(arrived) })
<-r.Context().Done()
},
))
t.Cleanup(ts.Close)
targetID := uuid.New().String()
iCreateWebhook(t, s.MainDB, s.WebhookID, "hook")
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "held",
database.TargetTypeHTTP, ts.URL, maxAttempts,
)
event := iSeedEvent(t, s.WebhookDB, s.WebhookID, `{"held":true}`)
d := iSeedDelivery(t, s.WebhookDB, event.ID, targetID, status)
bodyStr := event.Body
return iTask(
d, event, s.WebhookID, targetID, "held", ts.URL,
maxAttempts, attemptNum, &bodyStr,
), arrived
}
// stopDuringAttempt waits until the receiver holds an attempt open, then
// stops the engine through its real stop hook.
func stopDuringAttempt(
t *testing.T, lc *recordingLifecycle, arrived <-chan struct{},
) {
t.Helper()
select {
case <-arrived:
case <-time.After(5 * time.Second):
t.Fatal("the delivery attempt never reached the receiver")
}
require.NoError(t, lc.hooks[0].OnStop(context.Background()))
}
// requireRequeued runs the recovery routine as at start on the stopped
// engine and requires it to queue task's attempt again on ch.
func requireRequeued(
t *testing.T, s iSetup, ch <-chan delivery.Task, task delivery.Task,
) {
t.Helper()
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID, atStart,
)
select {
case got := <-ch:
assert.Equal(t, task.DeliveryID, got.DeliveryID)
assert.Equal(t, task.AttemptNum, got.AttemptNum)
case <-time.After(5 * time.Second):
t.Fatal("recovery at start did not queue the delivery again")
}
}
// TestEngine_StopLeavesFireAndForgetAttemptPending is the regression
// test for a stop that turned an attempt in flight into a terminal
// failure. Stopping the engine cancels the request; for a target with
// no retries that was the delivery's only attempt, recorded as failed
// with "context canceled", and nothing ever sent it again. The
// delivery must instead stay pending, with no result and no count in
// the attempt metrics, for the recovery routine to send at the next
// start.
func TestEngine_StopLeavesFireAndForgetAttemptPending(t *testing.T) {
t.Parallel()
s := newISetup(t)
reg := mIsolate(t, s)
lc := startEngineViaHook(t, s.Engine)
task, arrived := seedHeldDelivery(
t, s, 1, database.DeliveryStatusPending, 1,
)
s.Engine.ExportNotify([]delivery.Task{task})
stopDuringAttempt(t, lc, arrived)
iAssertStatus(
t, s.WebhookDB, task.DeliveryID, database.DeliveryStatusPending,
)
assert.Empty(t, iResults(t, s.WebhookDB, task.DeliveryID))
assert.Zero(t, mCounter(t, reg, mAttempts, mTypeHTTP))
assert.Zero(t, mHTTPDurations(t, reg))
requireRequeued(t, s, s.Engine.ExportDeliveryCh(), task)
}
// TestEngine_StopLeavesLastRetryAttemptRetrying is the same regression
// for a target with retries, on its last attempt: the cancelled attempt
// used to exhaust the retries and end the delivery failed. It must stay
// retrying with only its earlier attempt recorded, and the recovery
// routine at the next start must send that last attempt again.
func TestEngine_StopLeavesLastRetryAttemptRetrying(t *testing.T) {
t.Parallel()
const maxAttempts = 2
s := newISetup(t)
lc := startEngineViaHook(t, s.Engine)
task, arrived := seedHeldDelivery(
t, s, maxAttempts, database.DeliveryStatusRetrying, maxAttempts,
)
iSeedFailedResult(t, s.WebhookDB, task.DeliveryID)
s.Engine.ExportScheduleRetry(task, 0)
stopDuringAttempt(t, lc, arrived)
iAssertStatus(
t, s.WebhookDB, task.DeliveryID, database.DeliveryStatusRetrying,
)
assert.Len(t, iResults(t, s.WebhookDB, task.DeliveryID), 1)
requireRequeued(t, s, s.Engine.ExportRetryCh(), task)
}
// deliverToArchive gives the setup's webhook a database target,
// runs one delivery to it through the running engine, and returns
// the target's archive file path. The archive writer holds the file
// open afterwards.
func deliverToArchive(t *testing.T, s iSetup) string {
t.Helper()
iCreateWebhook(t, s.MainDB, s.WebhookID, "hook")
tgt := &database.Target{
WebhookID: s.WebhookID,
Name: "archive",
Type: database.TargetTypeDatabase,
MaxAttempts: 1,
}
require.NoError(
t, s.MainDB.Omit(clause.Associations).Create(tgt).Error,
)
deliveryID, task := seedLogTask(t, s)
task.TargetID = tgt.ID
task.TargetType = database.TargetTypeDatabase
s.Engine.Notify([]delivery.Task{task})
s.Engine.ExportNotify([]delivery.Task{task})
iWaitForDelivered(t, s.WebhookDB, deliveryID)
return filepath.Join(
filepath.Dir(s.DBMgr.DBPath(s.WebhookID)),
fmt.Sprintf("archive-%s.db", s.WebhookID),
"archive-hook-archive-"+tgt.ID+".db",
)
}
@@ -338,21 +505,23 @@ func TestEngine_StopHookTimeoutLeavesArchivesOpen(t *testing.T) {
lc := startEngineViaHook(t, s.Engine)
deliverToArchive(t, s)
path := deliverToArchive(t, s)
release := make(chan struct{})
t.Cleanup(func() {
close(release)
s.Engine.EvictWebhook(s.WebhookID)
s.Engine.ExportArchives().EvictWebhook(s.WebhookID)
})
s.Engine.ExportWedgeWorker(release)
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
require.True(
t, s.Engine.ExportArchiveHandleOpen(s.WebhookID),
// Closing the last handle on an archive removes its -wal, so a
// -wal still there is a writer still open.
require.FileExists(
t, path+"-wal",
"a stop that timed out must not close archive writers",
)
}
internal/delivery/engine_test.go
+310 -258
View File
@@ -5,11 +5,12 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"sync/atomic"
@@ -20,10 +21,10 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
_ "modernc.org/sqlite"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/database/databasetest"
"sneak.berlin/go/webhooker/internal/delivery"
"sneak.berlin/go/webhooker/internal/metrics"
)
@@ -34,31 +35,18 @@ const testContentType = "application/json"
func testWebhookDB(t *testing.T) *gorm.DB {
t.Helper()
dbPath := filepath.Join(
t.TempDir(), "events-test.db",
// Opened by the manager the service uses, so these tests cannot
// pass against a schema, journal and locking settings or write
// guard production does not use.
mgr := databasetest.NewWebhookDBManagerWithLogger(
t, t.TempDir(), slog.New(slog.DiscardHandler),
)
// Opened the way the service opens a per-webhook database, so
// these tests cannot pass against journal and locking settings
// production does not use.
sqlDB, err := database.OpenSQLite(
dbPath, database.SQLiteModeCreate,
)
t.Cleanup(func() { _ = mgr.CloseAll() })
db, err := mgr.GetDB(uuid.New().String())
require.NoError(t, err)
t.Cleanup(func() { _ = sqlDB.Close() })
db, err := gorm.Open(
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
)
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(
&database.Event{},
&database.Delivery{},
&database.DeliveryResult{},
))
return db
}
@@ -77,20 +65,6 @@ func testEngine(
)
}
func newHTTPTargetConfig(url string) string {
cfg := delivery.HTTPTargetConfig{URL: url}
data, err := json.Marshal(cfg)
if err != nil {
panic(
"failed to marshal HTTPTargetConfig: " +
err.Error(),
)
}
return string(data)
}
func seedEvent(
t *testing.T, db *gorm.DB, body string,
) database.Event {
@@ -129,6 +103,18 @@ func seedDelivery(
return d
}
// assertStoredTime asserts that text, a timestamp as SQLite holds it,
// is in the form GORM stores created_at in: the layout of
// time.Time.String, with no " m=+…" monotonic clock reading after it.
func assertStoredTime(t *testing.T, text string) {
t.Helper()
_, err := time.Parse(
"2006-01-02 15:04:05.999999999 -0700 MST", text,
)
assert.NoError(t, err, "stored as %q", text)
}
// httpDeliveryFixture holds the task and delivery needed
// for HTTP delivery tests.
type httpDeliveryFixture struct {
@@ -141,19 +127,19 @@ type httpDeliveryFixture struct {
func buildHTTPFixture(
dlv database.Delivery,
event database.Event,
targetID, name, config string,
maxRetries, attemptNum int,
targetID, name, targetURL string,
maxAttempts, attemptNum int,
) httpDeliveryFixture {
task := &delivery.Task{
DeliveryID: dlv.ID,
EventID: event.ID,
WebhookID: event.WebhookID,
TargetID: targetID,
TargetName: name,
TargetType: database.TargetTypeHTTP,
TargetConfig: config,
MaxRetries: maxRetries,
AttemptNum: attemptNum,
DeliveryID: dlv.ID,
EventID: event.ID,
WebhookID: event.WebhookID,
TargetID: targetID,
TargetName: name,
TargetType: database.TargetTypeHTTP,
TargetURL: targetURL,
MaxAttempts: maxAttempts,
AttemptNum: attemptNum,
}
d := &database.Delivery{
@@ -162,10 +148,10 @@ func buildHTTPFixture(
Status: dlv.Status,
Event: event,
Target: database.Target{
Name: name,
Type: database.TargetTypeHTTP,
Config: config,
MaxRetries: maxRetries,
Name: name,
Type: database.TargetTypeHTTP,
URL: targetURL,
MaxAttempts: maxAttempts,
},
}
d.ID = dlv.ID
@@ -232,7 +218,7 @@ func TestNotify_NonBlocking(t *testing.T) {
done := make(chan struct{})
go func() {
e.Notify([]delivery.Task{
e.ExportNotify([]delivery.Task{
{DeliveryID: "overflow-1"},
{DeliveryID: "overflow-2"},
})
@@ -243,7 +229,7 @@ func TestNotify_NonBlocking(t *testing.T) {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal(
"Notify blocked when delivery channel was full",
"notify blocked when delivery channel was full",
)
}
}
@@ -275,9 +261,8 @@ func TestDeliverHTTP_Success(t *testing.T) {
database.DeliveryStatusPending,
)
cfg := newHTTPTargetConfig(ts.URL)
fix := buildHTTPFixture(
dlv, event, targetID, "test-http", cfg, 0, 1,
dlv, event, targetID, "test-http", ts.URL, 1, 1,
)
e.ExportDeliverHTTP(
@@ -321,10 +306,9 @@ func TestDeliverHTTP_Failure(t *testing.T) {
database.DeliveryStatusPending,
)
cfg := newHTTPTargetConfig(ts.URL)
fix := buildHTTPFixture(
dlv, event, targetID,
"test-http-fail", cfg, 0, 1,
"test-http-fail", ts.URL, 1, 1,
)
e.ExportDeliverHTTP(
@@ -348,23 +332,15 @@ func TestDeliverDatabase_ImmediateSuccess(
db := testWebhookDB(t)
// The database target archives for real now, so the engine
// needs a webhook DB manager to locate the data directory.
e := delivery.NewTestEngineWithDB(
nil,
database.NewTestWebhookDBManager(t.TempDir()),
slog.New(slog.NewTextHandler(
os.Stderr,
&slog.HandlerOptions{Level: slog.LevelDebug},
)),
&http.Client{Timeout: 5 * time.Second},
1,
)
// The database target archives for real, so the engine needs
// the target in the main database and a data directory.
env := setupArchiveTest(t)
tgt := env.seedDatabaseTarget(t)
event := seedEvent(t, db, `{"db":"target"}`)
d := seedDatabaseTargetDelivery(t, db, event, "")
d := seedDatabaseTargetDelivery(t, db, event, tgt)
e.ExportDeliverDatabase(db, d)
env.eng.ExportDeliverDatabase(db, d)
var updated database.Delivery
@@ -460,10 +436,9 @@ func TestDeliverHTTP_WithRetries_Success(t *testing.T) {
database.DeliveryStatusPending,
)
cfg := newHTTPTargetConfig(ts.URL)
fix := buildHTTPFixture(
dlv, event, targetID,
"test-http-retry", cfg, 5, 1,
"test-http-retry", ts.URL, 5, 1,
)
e.ExportDeliverHTTP(
@@ -481,14 +456,23 @@ func TestDeliverHTTP_WithRetries_Success(t *testing.T) {
)
}
func TestDeliverHTTP_MaxRetriesExhausted(t *testing.T) {
// TestDeliverHTTP_MaxAttemptsExhausted proves a target with three
// delivery attempts makes at most three: the first two failures are
// retried, the third is final, and each attempt goes through the
// target's circuit breaker.
func TestDeliverHTTP_MaxAttemptsExhausted(t *testing.T) {
t.Parallel()
const maxAttempts = 3
db := testWebhookDB(t)
var requests atomic.Int32
ts := httptest.NewServer(
http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
requests.Add(1)
w.WriteHeader(http.StatusBadGateway)
},
),
@@ -497,26 +481,101 @@ func TestDeliverHTTP_MaxRetriesExhausted(t *testing.T) {
e := testEngine(t, 1)
targetID := uuid.New().String()
// It opens on the last failure only if every attempt reached it.
cb := delivery.NewTestCircuitBreaker(maxAttempts, time.Minute)
e.ExportSetCircuitBreaker(targetID, cb)
event := seedEvent(t, db, `{"retry":"exhaust"}`)
dlv := seedDelivery(
t, db, event.ID, targetID,
database.DeliveryStatusRetrying,
database.DeliveryStatusPending,
)
cfg := newHTTPTargetConfig(ts.URL)
fix := buildHTTPFixture(
dlv, event, targetID,
"test-http-exhaust", cfg, 3, 3,
)
sched := &recordingScheduler{}
e.ExportDeliverHTTP(
context.TODO(), db, fix.Delivery, fix.Task,
)
for attempt := 1; attempt <= maxAttempts; attempt++ {
fix := buildHTTPFixture(
dlv, event, targetID,
"test-http-exhaust", ts.URL, maxAttempts, attempt,
)
e.ExportDeliverHTTPWithScheduler(
context.TODO(), db, fix.Delivery, fix.Task, sched,
)
}
assert.Equal(t, int32(maxAttempts), requests.Load())
assert.Len(t, sched.delays, maxAttempts-1,
"every failure but the last should be retried",
)
assertDeliveryStatus(t, db, dlv.ID,
database.DeliveryStatusFailed,
)
assert.Equal(t, delivery.CircuitOpen, cb.State(),
"every attempt should be counted by the circuit breaker",
)
}
// TestDeliverHTTP_SingleAttemptSkipsCircuitBreaker proves a target
// with one delivery attempt makes exactly that attempt, even while
// its circuit breaker is open, and that a failure is final.
func TestDeliverHTTP_SingleAttemptSkipsCircuitBreaker(t *testing.T) {
t.Parallel()
db := testWebhookDB(t)
var requests atomic.Int32
ts := httptest.NewServer(
http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
requests.Add(1)
w.WriteHeader(http.StatusBadGateway)
},
),
)
defer ts.Close()
e := testEngine(t, 1)
targetID := uuid.New().String()
cb := e.ExportGetCircuitBreaker(targetID)
for range delivery.ExportDefaultFailureThreshold {
cb.RecordFailure()
}
require.Equal(t, delivery.CircuitOpen, cb.State())
event := seedEvent(t, db, `{"retry":"none"}`)
dlv := seedDelivery(
t, db, event.ID, targetID,
database.DeliveryStatusPending,
)
fix := buildHTTPFixture(
dlv, event, targetID,
"test-http-single", ts.URL, 1, 1,
)
sched := &recordingScheduler{}
e.ExportDeliverHTTPWithScheduler(
context.TODO(), db, fix.Delivery, fix.Task, sched,
)
assert.Equal(t, int32(1), requests.Load(),
"the open circuit breaker should not hold the attempt back",
)
assert.Empty(t, sched.delays, "no retry should be scheduled")
assertDeliveryStatus(t, db, dlv.ID,
database.DeliveryStatusFailed,
)
assertDeliveryResult(
t, db, dlv.ID, false, http.StatusBadGateway,
)
}
func TestDeliverHTTP_SchedulesRetryOnFailure(
@@ -546,10 +605,9 @@ func TestDeliverHTTP_SchedulesRetryOnFailure(
database.DeliveryStatusPending,
)
cfg := newHTTPTargetConfig(ts.URL)
fix := buildHTTPFixture(
dlv, event, targetID,
"test-http-schedule", cfg, 5, 1,
"test-http-schedule", ts.URL, 5, 1,
)
e.ExportDeliverHTTP(
@@ -711,7 +769,7 @@ func seedConcurrencyTasks(
t *testing.T,
db *gorm.DB,
n int,
targetCfg string,
targetURL string,
) ([]database.Delivery, []delivery.Task) {
t.Helper()
@@ -735,22 +793,23 @@ func seedConcurrencyTasks(
Status: database.DeliveryStatusPending,
Event: event,
Target: database.Target{
Name: fmt.Sprintf("task-%d", i),
Type: database.TargetTypeHTTP,
Config: targetCfg,
Name: fmt.Sprintf("task-%d", i),
Type: database.TargetTypeHTTP,
URL: targetURL,
MaxAttempts: 1,
},
}
tasks[i].ID = dlv.ID
dtasks[i] = delivery.Task{
DeliveryID: tasks[i].ID,
EventID: tasks[i].EventID,
TargetID: tasks[i].TargetID,
TargetName: tasks[i].Target.Name,
TargetType: tasks[i].Target.Type,
TargetConfig: tasks[i].Target.Config,
MaxRetries: 0,
AttemptNum: 1,
DeliveryID: tasks[i].ID,
EventID: tasks[i].EventID,
TargetID: tasks[i].TargetID,
TargetName: tasks[i].Target.Name,
TargetType: tasks[i].Target.Type,
TargetURL: tasks[i].Target.URL,
MaxAttempts: 1,
AttemptNum: 1,
}
}
@@ -786,9 +845,8 @@ func TestWorkerPool_BoundedConcurrency(t *testing.T) {
defer ts.Close()
e := testEngine(t, numWorkers)
cfg := newHTTPTargetConfig(ts.URL)
tasks, dtasks := seedConcurrencyTasks(
t, db, numTasks, cfg,
t, db, numTasks, ts.URL,
)
runConcurrencyWorkers(
@@ -869,12 +927,9 @@ func TestDeliverHTTP_CircuitBreakerBlocks(t *testing.T) {
database.DeliveryStatusPending,
)
cfg := newHTTPTargetConfig(
"http://will-not-be-called.invalid",
)
fix := buildHTTPFixture(
dlv, event, targetID,
"test-cb-block", cfg, 5, 1,
"test-cb-block", "http://will-not-be-called.invalid", 5, 1,
)
e.ExportDeliverHTTP(
@@ -937,9 +992,8 @@ func TestDeliverHTTP_HalfOpenBreakerDelaysQueuedTasks(t *testing.T) {
require.True(t, cb.Allow(), "the probe delivery should go through")
require.Equal(t, delivery.CircuitHalfOpen, cb.State())
cfg := newHTTPTargetConfig(
"http://will-not-be-called.invalid",
)
const targetURL = "http://will-not-be-called.invalid"
sched := &recordingScheduler{}
const queued, passes = 3, 4
@@ -961,7 +1015,7 @@ func TestDeliverHTTP_HalfOpenBreakerDelaysQueuedTasks(t *testing.T) {
fix := buildHTTPFixture(
row, event, targetID,
"test-cb-half-open", cfg, 5, 1,
"test-cb-half-open", targetURL, 5, 1,
)
e.ExportDeliverHTTPWithScheduler(
@@ -1020,45 +1074,60 @@ func TestGetCircuitBreaker_CreatesOnDemand(t *testing.T) {
)
}
func TestParseHTTPConfig_Valid(t *testing.T) {
// TestStateAndCooldown_ReadsHTTPAndSlackBreakers proves the engine
// reads the state of an http or a slack target's circuit breaker, with
// what is left of its cooldown while it is open, and no cooldown while
// it is half-open, once it closes, or for a target with no breaker.
func TestStateAndCooldown_ReadsHTTPAndSlackBreakers(t *testing.T) {
t.Parallel()
e := testEngine(t, 1)
cfg, err := e.ExportParseHTTPConfig(
`{"url":"https://example.com/hook",` +
`"headers":{"X-Token":"secret"}}`,
)
httpID := uuid.New().String()
slackID := uuid.New().String()
require.NoError(t, err)
assert.Equal(t, "https://example.com/hook", cfg.URL)
assert.Equal(t, "secret", cfg.Headers["X-Token"])
}
state, cooldown := e.StateAndCooldown(httpID)
assert.Equal(t, delivery.CircuitClosed, state, "no breaker")
assert.Zero(t, cooldown, "no breaker")
func TestParseHTTPConfig_Empty(t *testing.T) {
t.Parallel()
httpCB := delivery.NewTestCircuitBreaker(1, time.Hour)
e.ExportSetCircuitBreaker(httpID, httpCB)
e := testEngine(t, 1)
slackCB := delivery.NewTestCircuitBreaker(1, time.Hour)
e.ExportSetSlackCircuitBreaker(slackID, slackCB)
_, err := e.ExportParseHTTPConfig("")
httpCB.RecordFailure()
slackCB.RecordFailure()
assert.Error(t, err,
"empty config should return error",
)
}
for _, id := range []string{httpID, slackID} {
state, cooldown := e.StateAndCooldown(id)
assert.Equal(t, delivery.CircuitOpen, state)
assert.Greater(t, cooldown, 59*time.Minute)
assert.LessOrEqual(t, cooldown, time.Hour)
}
func TestParseHTTPConfig_MissingURL(t *testing.T) {
t.Parallel()
httpCB.RecordSuccess()
slackCB.RecordSuccess()
e := testEngine(t, 1)
for _, id := range []string{httpID, slackID} {
state, cooldown := e.StateAndCooldown(id)
assert.Equal(t, delivery.CircuitClosed, state, "closed")
assert.Zero(t, cooldown, "closed")
}
_, err := e.ExportParseHTTPConfig(
`{"headers":{"X-Token":"secret"}}`,
)
// A breaker with no cooldown goes half-open on the first Allow
// after it trips, letting that one delivery through to test the
// target.
halfOpenID := uuid.New().String()
halfOpenCB := delivery.NewTestCircuitBreaker(1, 0)
e.ExportSetCircuitBreaker(halfOpenID, halfOpenCB)
assert.Error(t, err,
"config without URL should return error",
)
halfOpenCB.RecordFailure()
require.True(t, halfOpenCB.Allow())
state, cooldown = e.StateAndCooldown(halfOpenID)
assert.Equal(t, delivery.CircuitHalfOpen, state)
assert.Zero(t, cooldown, "half-open")
}
func TestScheduleRetry_SendsToRetryChannel(
@@ -1208,7 +1277,7 @@ func TestDoHTTPRequest_ForwardsHeaders(t *testing.T) {
e := testEngine(t, 1)
cfg := &delivery.HTTPTargetConfig{
target := &database.Target{
URL: ts.URL,
Headers: map[string]string{
"X-Target-Auth": "bearer xyz",
@@ -1223,7 +1292,7 @@ func TestDoHTTPRequest_ForwardsHeaders(t *testing.T) {
}
statusCode, _, _, err := e.ExportDoHTTPRequest(
context.TODO(), cfg, event,
context.TODO(), target, event,
)
require.NoError(t, err)
@@ -1244,11 +1313,33 @@ func TestDoHTTPRequest_ForwardsHeaders(t *testing.T) {
testContentType,
receivedHeaders.Get("Content-Type"),
)
}
assert.Equal(t,
"webhooker/1.0",
receivedHeaders.Get("User-Agent"),
// A response that ends before the length it announced is an error, not
// a short body.
func TestDoHTTPRequest_CutShortResponseIsAnError(t *testing.T) {
t.Parallel()
ts := httptest.NewServer(
http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Length", "100")
_, _ = w.Write([]byte("cut short"))
},
),
)
defer ts.Close()
e := testEngine(t, 1)
_, body, _, err := e.ExportDoHTTPRequest(
context.TODO(),
&database.Target{URL: ts.URL},
&database.Event{},
)
require.ErrorIs(t, err, io.ErrUnexpectedEOF)
assert.Empty(t, body)
}
// The event's stored inbound headers carry the same Content-Type the
@@ -1295,9 +1386,9 @@ func TestApplyRequestHeaders_SendsOneContentType(t *testing.T) {
})
require.NoError(t, err)
cfg := &delivery.HTTPTargetConfig{}
var configured map[string]string
if tc.configured != "" {
cfg.Headers = map[string]string{
configured = map[string]string{
headerContentType: tc.configured,
}
}
@@ -1316,7 +1407,8 @@ func TestApplyRequestHeaders_SendsOneContentType(t *testing.T) {
Headers: string(inbound),
ContentType: tc.event,
},
cfg,
configured,
"webhooker/dev",
)
assert.Equal(t,
@@ -1326,6 +1418,34 @@ func TestApplyRequestHeaders_SendsOneContentType(t *testing.T) {
}
}
// Stored inbound headers that do not decode forward nothing, not the
// part of them that happened to decode.
func TestApplyRequestHeaders_UndecodableInboundForwardsNothing(
t *testing.T,
) {
t.Parallel()
req, err := http.NewRequestWithContext(
context.Background(),
http.MethodPost,
"https://target.example.com/hook",
http.NoBody,
)
require.NoError(t, err)
names := delivery.ExportApplyRequestHeaders(
req,
&database.Event{
Headers: `{"X-Custom":["value1"],"X-Broken":"not a list"}`,
},
nil,
"webhooker/dev",
)
assert.Empty(t, names)
assert.Empty(t, req.Header.Get("X-Custom"))
}
func TestProcessDelivery_RoutesToCorrectHandler(
t *testing.T,
) {
@@ -1333,32 +1453,27 @@ func TestProcessDelivery_RoutesToCorrectHandler(
db := testWebhookDB(t)
// The database target archives for real now, so the engine
// needs a webhook DB manager to locate the data directory.
e := delivery.NewTestEngineWithDB(
nil,
database.NewTestWebhookDBManager(t.TempDir()),
slog.New(slog.NewTextHandler(
os.Stderr,
&slog.HandlerOptions{Level: slog.LevelDebug},
)),
&http.Client{Timeout: 5 * time.Second},
1,
)
// The database target archives for real, so the engine needs
// the target in the main database and a data directory.
env := setupArchiveTest(t)
archive := env.seedDatabaseTarget(t)
tests := []struct {
name string
targetType database.TargetType
targetID string
wantStatus database.DeliveryStatus
}{
{
"database target",
database.TargetTypeDatabase,
archive.ID,
database.DeliveryStatusDelivered,
},
{
"log target",
database.TargetTypeLog,
uuid.New().String(),
database.DeliveryStatusDelivered,
},
}
@@ -1368,7 +1483,7 @@ func TestProcessDelivery_RoutesToCorrectHandler(
t.Parallel()
runRoutingSubtest(
t, db, e, tt.targetType,
t, db, env.eng, tt.targetType, tt.targetID,
tt.wantStatus,
)
})
@@ -1380,6 +1495,7 @@ func runRoutingSubtest(
db *gorm.DB,
e *delivery.Engine,
targetType database.TargetType,
targetID string,
wantStatus database.DeliveryStatus,
) {
t.Helper()
@@ -1387,8 +1503,7 @@ func runRoutingSubtest(
event := seedEvent(t, db, `{"routing":"test"}`)
dlv := seedDelivery(
t, db, event.ID,
uuid.New().String(),
t, db, event.ID, targetID,
database.DeliveryStatusPending,
)
@@ -1426,51 +1541,6 @@ func TestMaxInlineBodySize_Constant(t *testing.T) {
)
}
func TestParseSlackConfig_Valid(t *testing.T) {
t.Parallel()
e := testEngine(t, 1)
cfg, err := e.ExportParseSlackConfig(
`{"webhookUrl":"https://hooks.slack.com/services/T00/B00/xxx"}`,
)
require.NoError(t, err)
assert.Equal(t,
"https://hooks.slack.com/services/T00/B00/xxx",
cfg.WebhookURL,
)
}
func TestParseSlackConfig_Empty(t *testing.T) {
t.Parallel()
e := testEngine(t, 1)
_, err := e.ExportParseSlackConfig("")
assert.Error(t, err,
"empty config should return error",
)
}
func TestParseSlackConfig_MissingWebhookURL(
t *testing.T,
) {
t.Parallel()
e := testEngine(t, 1)
_, err := e.ExportParseSlackConfig(
`{"other":"field"}`,
)
assert.Error(t, err,
"config without webhook_url should return error",
)
}
func TestFormatSlackMessage_JSONBody(t *testing.T) {
t.Parallel()
@@ -1567,11 +1637,11 @@ func TestFormatSlackMessage_LargeJSONTruncated(
}
// buildSlackDelivery creates a database.Delivery
// with a Slack target config for testing.
// to a Slack target with the given webhook URL for testing.
func buildSlackDelivery(
dlv database.Delivery,
event database.Event,
targetID, name, slackCfg string,
targetID, name, webhookURL string,
) *database.Delivery {
d := &database.Delivery{
EventID: event.ID,
@@ -1579,9 +1649,10 @@ func buildSlackDelivery(
Status: dlv.Status,
Event: event,
Target: database.Target{
Name: name,
Type: database.TargetTypeSlack,
Config: slackCfg,
Name: name,
Type: database.TargetTypeSlack,
URL: webhookURL,
MaxAttempts: 1,
},
}
d.ID = dlv.ID
@@ -1618,11 +1689,10 @@ func TestDeliverSlack_Success(t *testing.T) {
)
defer ts.Close()
e, targetID, slackCfg, event, dlv :=
setupSlackTest(t, db, ts.URL)
e, targetID, event, dlv := setupSlackTest(t, db)
d := buildSlackDelivery(
dlv, event, targetID, "test-slack", slackCfg,
dlv, event, targetID, "test-slack", ts.URL,
)
e.ExportDeliverSlack(context.TODO(), db, d)
@@ -1641,9 +1711,8 @@ func TestDeliverSlack_Success(t *testing.T) {
func setupSlackTest(
t *testing.T,
db *gorm.DB,
serverURL string,
) (
*delivery.Engine, string, string,
*delivery.Engine, string,
database.Event, database.Delivery,
) {
t.Helper()
@@ -1651,13 +1720,6 @@ func setupSlackTest(
e := testEngine(t, 1)
targetID := uuid.New().String()
cfgBytes, err := json.Marshal(
delivery.SlackTargetConfig{
WebhookURL: serverURL,
},
)
require.NoError(t, err)
event := seedEvent(
t, db, `{"action":"test","data":"value"}`,
)
@@ -1667,7 +1729,7 @@ func setupSlackTest(
database.DeliveryStatusPending,
)
return e, targetID, string(cfgBytes), event, dlv
return e, targetID, event, dlv
}
func assertSlackPayload(
@@ -1716,13 +1778,6 @@ func TestDeliverSlack_Failure(t *testing.T) {
e := testEngine(t, 1)
targetID := uuid.New().String()
slackCfg, err := json.Marshal(
delivery.SlackTargetConfig{
WebhookURL: ts.URL,
},
)
require.NoError(t, err)
event := seedEvent(t, db, `{"test":true}`)
dlv := seedDelivery(
@@ -1732,7 +1787,7 @@ func TestDeliverSlack_Failure(t *testing.T) {
d := buildSlackDelivery(
dlv, event, targetID,
"test-slack-fail", string(slackCfg),
"test-slack-fail", ts.URL,
)
e.ExportDeliverSlack(context.TODO(), db, d)
@@ -1746,7 +1801,10 @@ func TestDeliverSlack_Failure(t *testing.T) {
)
}
func TestDeliverSlack_InvalidConfig(t *testing.T) {
// TestDeliverSlack_NoURL proves a Slack target without a URL, which
// the forms never store, fails its delivery rather than reporting it
// delivered.
func TestDeliverSlack_NoURL(t *testing.T) {
t.Parallel()
db := testWebhookDB(t)
@@ -1765,9 +1823,9 @@ func TestDeliverSlack_InvalidConfig(t *testing.T) {
Status: database.DeliveryStatusPending,
Event: event,
Target: database.Target{
Name: "test-slack-bad",
Type: database.TargetTypeSlack,
Config: `{"not_webhook_url":"missing"}`,
Name: "test-slack-bad",
Type: database.TargetTypeSlack,
MaxAttempts: 1,
},
}
d.ID = dlv.ID
@@ -1804,13 +1862,6 @@ func TestProcessDelivery_RoutesToSlack(t *testing.T) {
e := testEngine(t, 1)
slackCfg, err := json.Marshal(
delivery.SlackTargetConfig{
WebhookURL: ts.URL,
},
)
require.NoError(t, err)
event := seedEvent(t, db, `{"route":"slack"}`)
dlv := seedDelivery(
@@ -1820,12 +1871,13 @@ func TestProcessDelivery_RoutesToSlack(t *testing.T) {
d := buildSlackDelivery(
dlv, event, dlv.TargetID,
"test-slack-route", string(slackCfg),
"test-slack-route", ts.URL,
)
task := &delivery.Task{
DeliveryID: dlv.ID,
TargetType: database.TargetTypeSlack,
DeliveryID: dlv.ID,
TargetType: database.TargetTypeSlack,
MaxAttempts: 1,
}
e.ExportProcessDelivery(
@@ -1875,6 +1927,10 @@ func assertLogLineComplete(
"log line must contain the full request headers",
)
assert.Contains(t, out, "raw_query="+strconv.Quote(event.RawQuery),
"log line must contain the query string",
)
assert.Contains(t, out, event.EntrypointID,
"log line must contain the entrypoint id",
)
@@ -1897,6 +1953,7 @@ func TestDeliverLog_LogsFullContent(t *testing.T) {
event := seedEvent(
t, db, `{"log-body-marker":"abc123"}`,
)
event.RawQuery = eventQuery
dlv := seedDelivery(
t, db, event.ID, uuid.New().String(),
@@ -1929,7 +1986,7 @@ func TestDeliverLog_LogsFullContent(t *testing.T) {
func buildSlackRetryDelivery(
dlv database.Delivery,
event database.Event,
targetID, cfg string,
targetID, webhookURL string,
) *database.Delivery {
d := &database.Delivery{
EventID: event.ID,
@@ -1937,10 +1994,10 @@ func buildSlackRetryDelivery(
Status: database.DeliveryStatusPending,
Event: event,
Target: database.Target{
Name: "test-slack-retry",
Type: database.TargetTypeSlack,
Config: cfg,
MaxRetries: 5,
Name: "test-slack-retry",
Type: database.TargetTypeSlack,
URL: webhookURL,
MaxAttempts: 5,
},
}
d.ID = dlv.ID
@@ -1958,11 +2015,6 @@ func TestDeliverSlack_WithRetries_SchedulesRetry(
e := testEngine(t, 1)
targetID := uuid.New().String()
slackCfg, err := json.Marshal(
delivery.SlackTargetConfig{WebhookURL: ts.URL},
)
require.NoError(t, err)
event := seedEvent(t, db, `{"slack":"retry"}`)
dlv := seedDelivery(
@@ -1971,15 +2023,15 @@ func TestDeliverSlack_WithRetries_SchedulesRetry(
)
d := buildSlackRetryDelivery(
dlv, event, targetID, string(slackCfg),
dlv, event, targetID, ts.URL,
)
task := &delivery.Task{
DeliveryID: dlv.ID,
TargetID: targetID,
TargetType: database.TargetTypeSlack,
MaxRetries: 5,
AttemptNum: 1,
DeliveryID: dlv.ID,
TargetID: targetID,
TargetType: database.TargetTypeSlack,
MaxAttempts: 5,
AttemptNum: 1,
}
e.ExportProcessDelivery(context.TODO(), db, d, task)
internal/delivery/event_db_isolation_test.go
+4 -6
View File
@@ -69,7 +69,6 @@ func TestEventDBHoldsNoTargetRows(t *testing.T) {
))
defer ts.Close()
cfg := iHTTPConfig(ts.URL)
targetID := uuid.New().String()
dbPath := s.DBMgr.DBPath(s.WebhookID)
@@ -85,7 +84,7 @@ func TestEventDBHoldsNoTargetRows(t *testing.T) {
)
task := iTask(
d, event, s.WebhookID, targetID,
"leaky-target", cfg, 5, 1, &body,
"leaky-target", ts.URL, 5, 1, &body,
)
s.Engine.ExportProcessNewTask(context.TODO(), &task)
@@ -102,7 +101,7 @@ func TestEventDBHoldsNoTargetRows(t *testing.T) {
// https://git.eeqj.de/sneak/webhooker/issues/107.
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "leaky-target",
database.TargetTypeHTTP, cfg, 5,
database.TargetTypeHTTP, ts.URL, 5,
)
rd := iSeedDelivery(
@@ -111,7 +110,7 @@ func TestEventDBHoldsNoTargetRows(t *testing.T) {
)
rTask := iTask(
rd, event, s.WebhookID, targetID,
"leaky-target", cfg, 5, 2, &body,
"leaky-target", ts.URL, 5, 2, &body,
)
s.Engine.ExportProcessRetryTask(context.TODO(), &rTask)
@@ -138,7 +137,6 @@ func TestEventDBHoldsNoTargetRowsOnFailedDelivery(t *testing.T) {
))
defer ts.Close()
cfg := iHTTPConfig(ts.URL)
targetID := uuid.New().String()
event := iSeedEvent(
@@ -152,7 +150,7 @@ func TestEventDBHoldsNoTargetRowsOnFailedDelivery(t *testing.T) {
)
task := iTask(
d, event, s.WebhookID, targetID,
"leaky-target", cfg, 0, 1, &body,
"leaky-target", ts.URL, 1, 1, &body,
)
s.Engine.ExportProcessNewTask(context.TODO(), &task)
internal/delivery/event_timestamp_test.go
+12 -25
View File
@@ -93,17 +93,6 @@ func (s *tsSink) text(t *testing.T) string {
}
}
func tsSlackConfig(t *testing.T, url string) string {
t.Helper()
data, err := json.Marshal(
delivery.SlackTargetConfig{WebhookURL: url},
)
require.NoError(t, err)
return string(data)
}
// tsSeedEvent writes an event whose CreatedAt is tsEventCreatedAt
// rather than the write time, so an assertion on the rendered
// timestamp cannot pass by accident against "roughly now".
@@ -142,16 +131,17 @@ func tsSeedEvent(
// tsSeedTarget writes the slack target row into the main database.
// The retry path confirms the target still exists before sending.
func tsSeedTarget(
t *testing.T, mainDB *gorm.DB, webhookID, config string,
t *testing.T, mainDB *gorm.DB, webhookID, webhookURL string,
) database.Target {
t.Helper()
target := database.Target{
WebhookID: webhookID,
Name: "slack-sink",
Type: database.TargetTypeSlack,
Config: config,
Active: true,
WebhookID: webhookID,
Name: "slack-sink",
Type: database.TargetTypeSlack,
URL: webhookURL,
Active: true,
MaxAttempts: 1,
}
require.NoError(t, mainDB.Create(&target).Error)
@@ -175,8 +165,8 @@ func tsTask(
TargetID: target.ID,
TargetName: target.Name,
TargetType: database.TargetTypeSlack,
TargetConfig: target.Config,
MaxRetries: 0,
TargetURL: target.URL,
MaxAttempts: 1,
Method: event.Method,
Headers: event.Headers,
ContentType: event.ContentType,
@@ -225,8 +215,7 @@ func (c tsCase) run(t *testing.T) (iSetup, database.Delivery, string) {
s := newISetup(t)
sink := newTSSink(t)
cfg := tsSlackConfig(t, sink.URL)
target := tsSeedTarget(t, s.MainDB, s.WebhookID, cfg)
target := tsSeedTarget(t, s.MainDB, s.WebhookID, sink.URL)
event := tsSeedEvent(t, s.WebhookDB, s.WebhookID)
d := iSeedDelivery(
@@ -337,8 +326,7 @@ func TestFormatSlackMessageOverTaskReconstructedEvent(
s := newISetup(t)
cfg := tsSlackConfig(t, tsUndeliverableHook)
target := tsSeedTarget(t, s.MainDB, s.WebhookID, cfg)
target := tsSeedTarget(t, s.MainDB, s.WebhookID, tsUndeliverableHook)
event := tsSeedEvent(t, s.WebhookDB, s.WebhookID)
d := iSeedDelivery(
@@ -409,8 +397,7 @@ func TestEventReconstructionSurvivesAReapedRow(t *testing.T) {
s := newISetup(t)
cfg := tsSlackConfig(t, tsUndeliverableHook)
target := tsSeedTarget(t, s.MainDB, s.WebhookID, cfg)
target := tsSeedTarget(t, s.MainDB, s.WebhookID, tsUndeliverableHook)
event := tsSeedEvent(t, s.WebhookDB, s.WebhookID)
d := iSeedDelivery(
internal/delivery/export_test.go
+72 -306
View File
@@ -9,18 +9,14 @@ import (
"net/url"
"time"
"github.com/prometheus/client_golang/prometheus"
"go.uber.org/fx"
"gorm.io/gorm"
"sneak.berlin/go/webhooker/internal/archive"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/metrics"
)
// ErrExportArchiveWriterEvicted exposes the sentinel returned by
// an evicted archive writer. It carries the Err prefix rather
// than this file's usual Export one because it is a sentinel
// error.
var ErrExportArchiveWriterEvicted = errArchiveWriterEvicted
// Exported constants for test access.
const (
ExportDeliveryChannelSize = deliveryChannelSize
@@ -40,11 +36,6 @@ const (
ExportPendingSweepMinAge = pendingSweepMinAge
)
// ExportIsBlockedIP exposes isBlockedIP for testing.
func ExportIsBlockedIP(ip net.IP) bool {
return isBlockedIP(ip)
}
// NewTestGuard builds an SSRF Guard from an explicit egress
// allowlist, without going through config. Passing no prefixes
// yields the default guard, which blocks every private/reserved
@@ -70,6 +61,11 @@ func ExportBlockedNetworks() []*net.IPNet {
return blockedNetworks
}
// ExportBlockedPublicNetworks exposes blockedPublicNetworks.
func ExportBlockedPublicNetworks() []*net.IPNet {
return blockedPublicNetworks
}
// ExportIsForwardableHeader exposes isForwardableHeader.
func ExportIsForwardableHeader(name string) bool {
return isForwardableHeader(name)
@@ -81,9 +77,10 @@ func ExportIsForwardableHeader(name string) bool {
func ExportApplyRequestHeaders(
req *http.Request,
event *database.Event,
cfg *HTTPTargetConfig,
configured map[string]string,
userAgent string,
) []string {
return applyRequestHeaders(req, event, cfg)
return applyRequestHeaders(req, event, configured, userAgent)
}
// ExportTruncate exposes truncate for testing.
@@ -140,9 +137,10 @@ func (e *Engine) ExportDeliverSlack(
d *database.Delivery,
) {
task := &Task{
DeliveryID: d.ID,
TargetID: d.TargetID,
AttemptNum: 1,
DeliveryID: d.ID,
TargetID: d.TargetID,
MaxAttempts: d.Target.MaxAttempts,
AttemptNum: 1,
}
e.targets[database.TargetTypeSlack].Deliver(
@@ -150,6 +148,21 @@ func (e *Engine) ExportDeliverSlack(
)
}
// ExportUpdateDeliveryStatus exposes updateDeliveryStatus. It passes no
// target type, so no metric moves.
func (e *Engine) ExportUpdateDeliveryStatus(
webhookDB *gorm.DB,
d *database.Delivery,
status database.DeliveryStatus,
) error {
return e.updateDeliveryStatus(webhookDB, d, "", status)
}
// ExportNotify exposes notify.
func (e *Engine) ExportNotify(tasks []Task) {
e.notify(tasks)
}
// ExportProcessNewTask exposes processNewTask.
func (e *Engine) ExportProcessNewTask(
ctx context.Context, task *Task,
@@ -200,37 +213,31 @@ func (e *Engine) ExportSetCircuitBreaker(
e.httpTarget.circuitBreakers.Store(targetID, cb)
}
// ExportParseHTTPConfig exposes parseHTTPConfig.
func (e *Engine) ExportParseHTTPConfig(
configJSON string,
) (*HTTPTargetConfig, error) {
return parseHTTPConfig(configJSON)
}
// ExportParseSlackConfig exposes parseSlackConfig.
func (e *Engine) ExportParseSlackConfig(
configJSON string,
) (*SlackTargetConfig, error) {
return parseSlackConfig(configJSON)
// ExportSetSlackCircuitBreaker is ExportSetCircuitBreaker for the
// slack target.
func (e *Engine) ExportSetSlackCircuitBreaker(
targetID string, cb *CircuitBreaker,
) {
e.slackTarget.circuitBreakers.Store(targetID, cb)
}
// ExportDoHTTPRequest exposes the http target's
// doHTTPRequest.
func (e *Engine) ExportDoHTTPRequest(
ctx context.Context,
cfg *HTTPTargetConfig,
target *database.Target,
event *database.Event,
) (int, string, int64, error) {
return e.httpTarget.doHTTPRequest(ctx, cfg, event)
return e.httpTarget.doHTTPRequest(ctx, target, event)
}
// ExportClientForRequest exposes the http target's
// clientForRequest.
func (e *Engine) ExportClientForRequest(
cfg *HTTPTargetConfig,
timeoutSeconds int,
originScoped []string,
) *http.Client {
return e.httpTarget.clientForRequest(cfg, originScoped)
return e.httpTarget.clientForRequest(timeoutSeconds, originScoped)
}
// ErrExportTooManyRedirects exposes the sentinel the redirect
@@ -265,32 +272,27 @@ func (e *Engine) ExportRecoverPendingDeliveries(
ctx context.Context,
webhookDB *gorm.DB,
webhookID string,
atStart bool,
) {
e.recoverPendingDeliveries(
ctx, webhookDB, webhookID,
ctx, webhookDB, webhookID, atStart,
)
}
// ExportRecoverWebhookDeliveries exposes
// recoverWebhookDeliveries.
func (e *Engine) ExportRecoverWebhookDeliveries(
ctx context.Context, webhookID string,
ctx context.Context, webhookID string, atStart bool,
) {
e.recoverWebhookDeliveries(ctx, webhookID)
e.recoverWebhookDeliveries(ctx, webhookID, atStart)
}
// ExportRecoverInFlight exposes recoverInFlight.
func (e *Engine) ExportRecoverInFlight(
ctx context.Context,
// ExportRecoverDeliveries exposes recoverDeliveries, the recovery
// routine.
func (e *Engine) ExportRecoverDeliveries(
ctx context.Context, atStart bool,
) {
e.recoverInFlight(ctx)
}
// ExportSweepWebhookRetries exposes sweepWebhookRetries.
func (e *Engine) ExportSweepWebhookRetries(
ctx context.Context, webhookID string,
) {
e.sweepWebhookRetries(ctx, webhookID)
e.recoverDeliveries(ctx, atStart)
}
// ExportStart exposes start for testing.
@@ -337,9 +339,9 @@ func (e *Engine) ExportRetainDelivery(deliveryID string) bool {
// ExportRecoverRetryingDeliveries exposes recoverRetryingDeliveries.
func (e *Engine) ExportRecoverRetryingDeliveries(
webhookDB *gorm.DB, webhookID string,
ctx context.Context, webhookDB *gorm.DB, webhookID string,
) {
e.recoverRetryingDeliveries(webhookDB, webhookID)
e.recoverRetryingDeliveries(ctx, webhookDB, webhookID)
}
// ExportFailMissingTarget exposes failMissingTarget, so a test can hand
@@ -378,7 +380,8 @@ func (e *Engine) ExportRetryCh() chan Task {
}
// NewTestEngine creates an Engine for unit tests without
// database dependencies.
// database dependencies. Its archive writers have no data
// directory, so a delivery to a database target fails.
func NewTestEngine(
log *slog.Logger,
client *http.Client,
@@ -389,7 +392,8 @@ func NewTestEngine(
deliveryCh: make(chan Task, deliveryChannelSize),
retryCh: make(chan Task, retryChannelSize),
workers: workers,
mtr: metrics.Default(),
mtr: metrics.New(prometheus.NewRegistry()),
archives: archive.NewWriters(archive.WritersParams{Logger: log}),
}
e.initTargets(client)
@@ -402,9 +406,10 @@ func NewTestEngineSmallRetry(
log *slog.Logger,
) *Engine {
e := &Engine{
log: log,
retryCh: make(chan Task, 1),
mtr: metrics.Default(),
log: log,
retryCh: make(chan Task, 1),
mtr: metrics.New(prometheus.NewRegistry()),
archives: archive.NewWriters(archive.WritersParams{Logger: log}),
}
e.initTargets(nil)
@@ -412,7 +417,8 @@ func NewTestEngineSmallRetry(
}
// NewTestEngineWithDB creates an Engine with a real
// database and dbManager for integration tests.
// database and dbManager for integration tests, and archive
// writers of its own over the same two.
func NewTestEngineWithDB(
db *database.Database,
dbMgr *database.WebhookDBManager,
@@ -427,16 +433,25 @@ func NewTestEngineWithDB(
deliveryCh: make(chan Task, deliveryChannelSize),
retryCh: make(chan Task, retryChannelSize),
workers: workers,
mtr: metrics.Default(),
mtr: metrics.New(prometheus.NewRegistry()),
archives: archive.NewWriters(archive.WritersParams{
Database: db,
DBManager: dbMgr,
Logger: log,
}),
}
e.initTargets(client)
return e
}
// ExportArchives returns the engine's archive writers.
func (e *Engine) ExportArchives() *archive.Writers {
return e.archives
}
// ExportSetMetrics substitutes the engine's metric set, so a test can
// assert on collectors registered on a private registry instead of
// the process-wide ones every other test is also moving.
// assert on collectors registered on a registry it holds.
func (e *Engine) ExportSetMetrics(mtr *metrics.Set) {
e.mtr = mtr
}
@@ -457,252 +472,3 @@ func NewTestCircuitBreaker(
cooldown: cooldown,
}
}
// ExportArchivedEvent aliases the archive row type so black-box
// tests can construct and read archive rows.
type ExportArchivedEvent = archivedEvent
// ExportArchiveWriter wraps an archiveWriter so black-box tests
// can exercise the per-webhook archive file mechanics.
type ExportArchiveWriter struct {
w *archiveWriter
}
// NewExportArchiveWriter builds an archive writer for tests,
// optionally overriding the reopen debounce (a non-positive
// debounce keeps the production default).
func NewExportArchiveWriter(
path string, log *slog.Logger, debounce time.Duration,
) *ExportArchiveWriter {
w := newArchiveWriter(path, log)
if debounce > 0 {
w.debounce = debounce
}
return &ExportArchiveWriter{w: w}
}
// Write archives a row through the writer.
func (e *ExportArchiveWriter) Write(
row ExportArchivedEvent, expiry time.Duration,
) error {
return e.w.write(row, expiry)
}
// Open opens the archive file, pruning when expiry is positive.
func (e *ExportArchiveWriter) Open(expiry time.Duration) error {
return e.w.open(expiry)
}
// Reopen closes and reopens the archive file.
func (e *ExportArchiveWriter) Reopen(
expiry time.Duration,
) error {
return e.w.reopen(expiry)
}
// Reopens reports how many times the file has been opened.
func (e *ExportArchiveWriter) Reopens() int {
return e.w.reopens
}
// DB returns the writer's current open handle for row
// inspection in tests.
func (e *ExportArchiveWriter) DB() *gorm.DB {
return e.w.db
}
// Path returns the archive file the writer owns.
func (e *ExportArchiveWriter) Path() string {
return e.w.path
}
// OpenExisting opens the archive without permitting creation,
// the way the idle sweep does.
func (e *ExportArchiveWriter) OpenExisting(
expiry time.Duration,
) error {
return e.w.openMode(archiveModeExisting, expiry)
}
// SweepExpired runs an idle sweep of the archive.
func (e *ExportArchiveWriter) SweepExpired(
expiry time.Duration,
) error {
return e.w.sweepExpired(expiry)
}
// Evict marks the writer evicted and closes its handle, exactly
// as leaving the registry does.
func (e *ExportArchiveWriter) Evict() {
e.w.evict()
}
// HandleOpen reports whether the writer currently holds an open
// archive handle.
func (e *ExportArchiveWriter) HandleOpen() bool {
e.w.mu.Lock()
defer e.w.mu.Unlock()
return e.w.db != nil
}
// Same reports whether both wrappers refer to the very same
// underlying archive writer, so a test can prove a registry entry
// is the writer it was handed rather than a replacement.
func (e *ExportArchiveWriter) Same(
other *ExportArchiveWriter,
) bool {
return other != nil && e.w == other.w
}
// ExportArchiveWriterFor returns the archive writer the registry
// currently caches for a webhook, or nil when none is cached. It
// never creates one, so a test can hold a reference to the very
// writer an eviction is about to detach.
func (e *Engine) ExportArchiveWriterFor(
webhookID string,
) *ExportArchiveWriter {
e.dbTarget.mu.Lock()
defer e.dbTarget.mu.Unlock()
w, ok := e.dbTarget.writers[webhookID]
if !ok {
return nil
}
return &ExportArchiveWriter{w: w}
}
// ExportHasArchiveWriter reports whether the database target
// currently caches an archive writer for a webhook.
func (e *Engine) ExportHasArchiveWriter(
webhookID string,
) bool {
e.dbTarget.mu.Lock()
defer e.dbTarget.mu.Unlock()
_, ok := e.dbTarget.writers[webhookID]
return ok
}
// ExportArchiveHandleOpen reports whether the cached archive
// writer for a webhook holds an open database handle. It
// returns false when no writer is cached.
func (e *Engine) ExportArchiveHandleOpen(
webhookID string,
) bool {
e.dbTarget.mu.Lock()
w, ok := e.dbTarget.writers[webhookID]
e.dbTarget.mu.Unlock()
if !ok {
return false
}
w.mu.Lock()
defer w.mu.Unlock()
return w.db != nil
}
// ExportEnsureArchiveWriter creates (if needed) and returns the
// archive file path of the cached writer for a webhook, so a
// test can prime the registry the way a delivery would.
func (e *Engine) ExportEnsureArchiveWriter(
webhookID string,
) (string, error) {
w, err := e.dbTarget.writerFor(webhookID)
if err != nil {
return "", err
}
return w.path, nil
}
// ExportSweepWriterFor takes a webhook's registry writer exactly
// as the idle sweep does, reporting whether the sweep had to
// create the entry. It lets a test drive the registry through the
// sweep's own entry point instead of choreographing goroutines.
func (e *Engine) ExportSweepWriterFor(
webhookID string,
) (*ExportArchiveWriter, bool, error) {
w, created, err := e.dbTarget.sweepWriterFor(webhookID)
if err != nil {
return nil, false, err
}
return &ExportArchiveWriter{w: w}, created, nil
}
// ExportReleaseSweepWriter releases a sweep-created registry entry
// exactly as a finished sweep does.
func (e *Engine) ExportReleaseSweepWriter(
webhookID string, w *ExportArchiveWriter,
) {
e.dbTarget.releaseSweepWriter(webhookID, w.w)
}
// NewTestArchiveSweeper builds an ArchiveSweeper backed by the
// given main database and engine, without the fx lifecycle.
// Intended for tests.
func NewTestArchiveSweeper(
db *database.Database,
eng *Engine,
log *slog.Logger,
) *ArchiveSweeper {
return &ArchiveSweeper{
db: db,
eng: eng,
log: log,
interval: time.Hour,
}
}
// ExportSweep runs a single archive sweep synchronously for
// tests.
func (s *ArchiveSweeper) ExportSweep(ctx context.Context) {
s.sweep(ctx)
}
// ExportStart starts the sweeper's background loop for tests.
func (s *ArchiveSweeper) ExportStart() {
s.start()
}
// ExportRegisterHooks registers the sweeper's real fx lifecycle
// hooks on a lifecycle supplied by a test, so a test can drive
// the exact OnStart/OnStop functions the application runs and
// hand OnStart the kind of context fx actually supplies.
func (s *ArchiveSweeper) ExportRegisterHooks(lc fx.Lifecycle) {
s.registerHooks(lc)
}
// ExportStop stops the sweeper's background loop for tests.
func (s *ArchiveSweeper) ExportStop(ctx context.Context) error {
return s.stop(ctx)
}
// ExportWedgeLoop adds a goroutine to the sweeper's WaitGroup
// that never observes cancellation and returns only when release
// is closed. It stands in for a prune stuck on a locked archive.
func (s *ArchiveSweeper) ExportWedgeLoop(
release <-chan struct{},
) {
s.wg.Go(func() {
<-release
})
}
// ExportSetInterval overrides the sweep interval for tests.
func (s *ArchiveSweeper) ExportSetInterval(d time.Duration) {
s.interval = d
}
// ExportParseArchiveExpiry exposes parseArchiveExpiry.
func ExportParseArchiveExpiry(
configJSON string,
) (time.Duration, error) {
return parseArchiveExpiry(configJSON)
}
internal/delivery/inflight.go
+9 -9
View File
@@ -6,10 +6,10 @@ import "sync"
//
// A delivery is owned from the moment a task for it is handed to a
// channel or to a retry timer until the engine has no further plan for
// it in memory. Restart recovery and both arms of the periodic sweep
// re-dispatch only deliveries the set does not hold, which is what
// makes them exact rather than a guess about how long a row has sat at
// pending.
// it in memory. The recovery routine, at start and on every tick of the
// sweep, re-dispatches only deliveries the set does not hold, which is
// what makes it exact rather than a guess about how long a row has sat
// at pending.
//
// This replaces reasoning from timestamps. A delivery's row says
// pending from creation until its outcome is written, which covers
@@ -25,8 +25,8 @@ import "sync"
// In-memory state is sufficient because a data directory admits one
// process: internal/datadir takes an flock on it at startup and a
// second instance refuses to run. Deliveries owned by a process that
// died are not in any successor's set, and restart recovery is what
// picks those up.
// died are not in any successor's set, and the recovery routine's run
// at start is what picks those up.
//
// References are counted rather than held as a plain set because
// ownership outlives the worker that took it. A target that schedules
@@ -61,9 +61,9 @@ func (s *inflightSet) retain(deliveryID string) {
// retainIdle takes the first reference on a delivery, and reports
// whether it got it. It fails when the engine already owns the
// delivery, which is what makes two claimants — restart recovery and
// the sweep run concurrently, or two sweep arms — mutually exclusive
// rather than merely atomic.
// delivery, which is what makes two claimants — notify and the
// recovery routine, say — mutually exclusive rather than merely
// atomic.
func (s *inflightSet) retainIdle(deliveryID string) bool {
s.mu.Lock()
defer s.mu.Unlock()
internal/delivery/inflight_test.go
+47 -61
View File
@@ -2,8 +2,6 @@ package delivery_test
import (
"context"
"path/filepath"
"strings"
"sync"
"testing"
"time"
@@ -32,7 +30,7 @@ func fSweepSetup(
iCreateTarget(t, s.MainDB, targetID,
s.WebhookID, name,
database.TargetTypeLog, "", 0,
database.TargetTypeLog, "", 1,
)
require.NoError(t, s.MainDB.Create(&database.Webhook{
@@ -48,8 +46,9 @@ func fSweepSetup(
//
// Every caller drives the dispatch paths synchronously and has already
// waited for them to return, so anything they queued is in the channel
// by now. The short grace covers nothing but scheduler jitter, and is
// kept small because one of these tests runs the drain forty times.
// by now, and nothing is waited for. A timer here would race the queued
// tasks: on a busy host it can be due by the time select looks, and
// select picks at random among the cases that are ready.
func fDrain(e *delivery.Engine) []delivery.Task {
var out []delivery.Task
@@ -59,39 +58,12 @@ func fDrain(e *delivery.Engine) []delivery.Task {
out = append(out, task)
case task := <-e.ExportRetryCh():
out = append(out, task)
case <-time.After(25 * time.Millisecond):
default:
return out
}
}
}
// TestArchiveHandleIsWAL closes the last gap in the durability
// evidence: the main and per-webhook tiers each assert their journal
// mode on a live handle, and the archive tier gets its settings from
// the same code path but nothing checked the running file.
func TestArchiveHandleIsWAL(t *testing.T) {
t.Parallel()
w := delivery.NewExportArchiveWriter(
filepath.Join(t.TempDir(), "archive-wal.db"),
archiveTestLogger(), 0,
)
require.NoError(t, w.Open(0))
var mode string
row := w.DB().Raw("pragma journal_mode").Row()
require.NoError(t, row.Scan(&mode))
assert.Equal(t, "wal", strings.ToLower(mode))
var busy string
row = w.DB().Raw("pragma busy_timeout").Row()
require.NoError(t, row.Scan(&busy))
assert.Equal(t, "10000", busy)
}
// TestSweepLeavesAQueuedDeliveryAlone is the case the age bound cannot
// see. The delivery is queued and untouched, so its row is arbitrarily
// old and still perfectly healthy; only ownership distinguishes it
@@ -112,9 +84,9 @@ func TestSweepLeavesAQueuedDeliveryAlone(t *testing.T) {
)
rAgePending(t, s.WebhookDB, d.ID)
// Queued exactly as the receiver queues it, and never dequeued:
// Queued exactly as StoreEvent queues it, and never dequeued:
// no workers are running in this engine.
s.Engine.Notify([]delivery.Task{{
s.Engine.ExportNotify([]delivery.Task{{
DeliveryID: d.ID,
EventID: event.ID,
WebhookID: s.WebhookID,
@@ -123,8 +95,8 @@ func TestSweepLeavesAQueuedDeliveryAlone(t *testing.T) {
require.Equal(t, 1, s.Engine.ExportInflightHeld())
s.Engine.ExportSweepWebhookRetries(
context.Background(), s.WebhookID,
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID, onTick,
)
tasks := fDrain(s.Engine)
@@ -135,9 +107,11 @@ func TestSweepLeavesAQueuedDeliveryAlone(t *testing.T) {
)
}
// TestRecoveryAndSweepDoNotDoubleDispatch drives the two entry points
// the engine starts concurrently against one aged pending row. Before
// ownership they both dispatched it.
// TestRecoveryAndSweepDoNotDoubleDispatch drives the recovery routine's
// run at start and a tick of the sweep at once against one aged pending
// row. The engine runs them one after the other, but notify can race
// either, and ownership is what keeps any two claimants from both
// sending. Before ownership the two runs both dispatched the row.
func TestRecoveryAndSweepDoNotDoubleDispatch(t *testing.T) {
t.Parallel()
@@ -161,12 +135,12 @@ func TestRecoveryAndSweepDoNotDoubleDispatch(t *testing.T) {
wg.Go(func() {
s.Engine.ExportRecoverPendingDeliveries(
ctx, s.WebhookDB, s.WebhookID,
ctx, s.WebhookDB, s.WebhookID, atStart,
)
})
wg.Go(func() {
s.Engine.ExportSweepWebhookRetries(
ctx, s.WebhookID,
s.Engine.ExportRecoverWebhookDeliveries(
ctx, s.WebhookID, onTick,
)
})
wg.Wait()
@@ -228,7 +202,7 @@ func TestOwnershipIsReleasedAfterDelivery(t *testing.T) {
iCreateTarget(t, s.MainDB, targetID,
s.WebhookID, "released",
database.TargetTypeLog, "", 0,
database.TargetTypeLog, "", 1,
)
event := iSeedEvent(
@@ -250,7 +224,7 @@ func TestOwnershipIsReleasedAfterDelivery(t *testing.T) {
body := `{"released":true}`
s.Engine.Notify([]delivery.Task{{
s.Engine.ExportNotify([]delivery.Task{{
DeliveryID: d.ID,
EventID: event.ID,
WebhookID: s.WebhookID,
@@ -274,10 +248,11 @@ func TestOwnershipIsReleasedAfterDelivery(t *testing.T) {
}
// TestNotifyAfterRecoveryDoesNotSendAgain is the startup race of
// https://git.eeqj.de/sneak/webhooker/issues/299. The receiver has
// written a delivery, restart recovery finds it pending, sends it and
// releases it, and only then does the receiver's Notify for it arrive.
// Nothing owns the delivery by then, so Notify takes it.
// https://git.eeqj.de/sneak/webhooker/issues/299. StoreEvent has
// written a delivery, the recovery routine's run at start finds it
// pending, sends it and releases it, and only then does StoreEvent's
// notify for it arrive. Nothing owns the delivery by then, so notify
// takes it.
func TestNotifyAfterRecoveryDoesNotSendAgain(t *testing.T) {
t.Parallel()
@@ -301,7 +276,8 @@ func TestNotifyAfterRecoveryDoesNotSendAgain(t *testing.T) {
)
}()
// Restart recovery sends the delivery and lets it go.
// The recovery routine's run at start sends the delivery and lets
// it go.
iWaitForDelivered(t, s.WebhookDB, d.ID)
require.Eventually(
t,
@@ -313,7 +289,7 @@ func TestNotifyAfterRecoveryDoesNotSendAgain(t *testing.T) {
body := event.Body
s.Engine.Notify([]delivery.Task{{
s.Engine.ExportNotify([]delivery.Task{{
DeliveryID: d.ID,
EventID: event.ID,
WebhookID: s.WebhookID,
@@ -324,7 +300,7 @@ func TestNotifyAfterRecoveryDoesNotSendAgain(t *testing.T) {
EntrypointID: event.EntrypointID,
}})
// Notify took the delivery, and a worker releases it once it has
// notify took the delivery, and a worker releases it once it has
// run the task.
require.Eventually(
t,
@@ -362,7 +338,7 @@ func TestRetryingRecoverySkipsASuccessfulResult(t *testing.T) {
rSeedResult(t, s.WebhookDB, d.ID, 2, true)
s.Engine.ExportRecoverRetryingDeliveries(
s.WebhookDB, s.WebhookID,
context.Background(), s.WebhookDB, s.WebhookID,
)
assert.Empty(
@@ -377,8 +353,8 @@ func TestRetryingRecoverySkipsASuccessfulResult(t *testing.T) {
)
}
// TestRetryingSweepSkipsASuccessfulResult is the same rule on the
// periodic sweep's retrying arm.
// TestRetryingSweepSkipsASuccessfulResult is the same rule on a tick of
// the sweep.
func TestRetryingSweepSkipsASuccessfulResult(t *testing.T) {
t.Parallel()
@@ -396,8 +372,8 @@ func TestRetryingSweepSkipsASuccessfulResult(t *testing.T) {
rSeedResult(t, s.WebhookDB, d.ID, 1, false)
rSeedResult(t, s.WebhookDB, d.ID, 2, true)
s.Engine.ExportSweepWebhookRetries(
context.Background(), s.WebhookID,
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID, onTick,
)
assert.Empty(t, fDrain(s.Engine))
@@ -448,8 +424,8 @@ func TestScheduledRetryIsNotSweptDuringBackoff(t *testing.T) {
require.Equal(t, 1, s.Engine.ExportInflightHeld())
s.Engine.ExportSweepWebhookRetries(
context.Background(), s.WebhookID,
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID, onTick,
)
assert.Empty(
@@ -461,7 +437,8 @@ func TestScheduledRetryIsNotSweptDuringBackoff(t *testing.T) {
// TestRedispatchStampsTheRow pins the cadence control: a stranded
// delivery that has just been handed out is not selected again by the
// next tick a minute later.
// next tick a minute later. The stamp is stored in the same form as
// created_at.
func TestRedispatchStampsTheRow(t *testing.T) {
t.Parallel()
@@ -480,7 +457,7 @@ func TestRedispatchStampsTheRow(t *testing.T) {
ctx := context.Background()
s.Engine.ExportSweepWebhookRetries(ctx, s.WebhookID)
s.Engine.ExportRecoverWebhookDeliveries(ctx, s.WebhookID, onTick)
require.Len(t, fDrain(s.Engine), 1)
var row database.Delivery
@@ -492,4 +469,13 @@ func TestRedispatchStampsTheRow(t *testing.T) {
"a re-dispatched delivery must be stamped so the "+
"next tick does not select it again",
)
var createdAt, updatedAt string
require.NoError(t, s.WebhookDB.Raw(
`SELECT CAST(created_at AS TEXT), CAST(updated_at AS TEXT)
FROM deliveries WHERE id = ?`, d.ID,
).Row().Scan(&createdAt, &updatedAt))
assertStoredTime(t, createdAt)
assertStoredTime(t, updatedAt)
}
internal/delivery/metrics_test.go
+19 -23
View File
@@ -35,9 +35,8 @@ const (
)
// mIsolate gives the setup's engine a metric set registered on a
// private registry. The process-wide collectors are moved by every
// other delivery test running in parallel, so exact assertions are
// only possible against a registry this test owns.
// registry this test holds, so its exact assertions can gather from
// it.
func mIsolate(
t *testing.T, s iSetup,
) *prometheus.Registry {
@@ -193,7 +192,7 @@ func mDeliverOK(t *testing.T, s iSetup) {
body := event.Body
task := iTask(
d, event, s.WebhookID, targetID,
"metrics-ok", iHTTPConfig(ts.URL), 3, 1, &body,
"metrics-ok", ts.URL, 3, 1, &body,
)
s.Engine.ExportProcessNewTask(context.TODO(), &task)
@@ -227,18 +226,17 @@ func mExhaustRetries(t *testing.T, s iSetup) {
)
body := event.Body
cfg := iHTTPConfig(ts.URL)
// The retry below is only run if its target still exists; see
// https://git.eeqj.de/sneak/webhooker/issues/107.
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "metrics-fail",
database.TargetTypeHTTP, cfg, 2,
database.TargetTypeHTTP, ts.URL, 2,
)
first := iTask(
d, event, s.WebhookID, targetID,
"metrics-fail", cfg, 2, 1, &body,
"metrics-fail", ts.URL, 2, 1, &body,
)
s.Engine.ExportProcessNewTask(context.TODO(), &first)
@@ -252,7 +250,7 @@ func mExhaustRetries(t *testing.T, s iSetup) {
// deterministic and off the wall clock.
second := iTask(
d, event, s.WebhookID, targetID,
"metrics-fail", cfg, 2, 2, &body,
"metrics-fail", ts.URL, 2, 2, &body,
)
s.Engine.ExportProcessRetryTask(
@@ -290,22 +288,21 @@ func TestDeliveryMetrics_CircuitBreakerGauge(t *testing.T) {
)
body := event.Body
cfg := iHTTPConfig(ts.URL)
// A retry budget above the failure threshold, so the breaker
// An attempt budget above the failure threshold, so the breaker
// rather than the budget is what stops the delivery.
maxRetries := delivery.ExportDefaultFailureThreshold + 5
maxAttempts := delivery.ExportDefaultFailureThreshold + 5
// The retries below are only run if their target still exists;
// see https://git.eeqj.de/sneak/webhooker/issues/107.
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "metrics-trip",
database.TargetTypeHTTP, cfg, maxRetries,
database.TargetTypeHTTP, ts.URL, maxAttempts,
)
first := iTask(
d, event, s.WebhookID, targetID,
"metrics-trip", cfg, maxRetries, 1, &body,
"metrics-trip", ts.URL, maxAttempts, 1, &body,
)
s.Engine.ExportProcessNewTask(context.TODO(), &first)
@@ -317,7 +314,7 @@ func TestDeliveryMetrics_CircuitBreakerGauge(t *testing.T) {
ExportDefaultFailureThreshold; attempt++ {
task := iTask(
d, event, s.WebhookID, targetID,
"metrics-trip", cfg, maxRetries, attempt, &body,
"metrics-trip", ts.URL, maxAttempts, attempt, &body,
)
s.Engine.ExportProcessRetryTask(
@@ -364,17 +361,16 @@ func TestDeliveryMetrics_BreakerBlockedIsNotAnAttempt(
)
body := event.Body
cfg := iHTTPConfig(ts.URL)
maxRetries := delivery.ExportDefaultFailureThreshold + 5
maxAttempts := delivery.ExportDefaultFailureThreshold + 5
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "metrics-blocked",
database.TargetTypeHTTP, cfg, maxRetries,
database.TargetTypeHTTP, ts.URL, maxAttempts,
)
first := iTask(
d, event, s.WebhookID, targetID,
"metrics-blocked", cfg, maxRetries, 1, &body,
"metrics-blocked", ts.URL, maxAttempts, 1, &body,
)
s.Engine.ExportProcessNewTask(context.TODO(), &first)
@@ -383,7 +379,7 @@ func TestDeliveryMetrics_BreakerBlockedIsNotAnAttempt(
ExportDefaultFailureThreshold; attempt++ {
task := iTask(
d, event, s.WebhookID, targetID,
"metrics-blocked", cfg, maxRetries, attempt, &body,
"metrics-blocked", ts.URL, maxAttempts, attempt, &body,
)
s.Engine.ExportProcessRetryTask(context.TODO(), &task)
@@ -406,7 +402,7 @@ func TestDeliveryMetrics_BreakerBlockedIsNotAnAttempt(
blocked := iTask(
d, event, s.WebhookID, targetID,
"metrics-blocked", cfg, maxRetries,
"metrics-blocked", ts.URL, maxAttempts,
delivery.ExportDefaultFailureThreshold+1, &body,
)
@@ -445,8 +441,8 @@ func TestDeliveryMetrics_OrphanedRetryFailureLabelled(
t, s, database.TargetTypeLog,
)
s.Engine.ExportSweepWebhookRetries(
context.Background(), s.WebhookID,
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID, onTick,
)
iAssertStatus(t, s.WebhookDB, deliveryID,
@@ -475,7 +471,7 @@ func TestDeliveryMetrics_QueueDepthGauges(t *testing.T) {
iCreateTarget(t, s.MainDB, targetID, s.WebhookID,
"queue-depth-target", database.TargetTypeHTTP,
iHTTPConfig("https://example.com/hook"), 3,
"https://example.com/hook", 3,
)
event := iSeedEvent(
internal/delivery/queue_depth.go
+2 -1
View File
@@ -20,7 +20,8 @@ const queueDepthSampleInterval = 30 * time.Second
// to be seeded correctly at startup from rows written by a previous
// process, and would drift permanently on any transition that failed
// to persist. Counting is the measurement that cannot go wrong, and
// it is the same whole-database walk the retry sweep already makes.
// it is the same whole-database walk each tick of the sweep already
// makes.
func (e *Engine) queueDepthSampler(ctx context.Context) {
defer e.wg.Done()
internal/delivery/queue_depth_gormlog_test.go
+10 -27
View File
@@ -5,7 +5,6 @@ import (
"context"
"log/slog"
"net/http"
"path/filepath"
"strings"
"sync"
"testing"
@@ -14,11 +13,9 @@ import (
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/database/databasetest"
"sneak.berlin/go/webhooker/internal/delivery"
"sneak.berlin/go/webhooker/internal/gormlog"
)
// qdAggregateMarker identifies the queue-depth aggregate in the
@@ -49,27 +46,13 @@ func (q *qdSyncBuf) String() string {
// qdMainDB opens a main database whose GORM logger is the service's
// adapter, writing through log.
func qdMainDB(t *testing.T, log *slog.Logger) *gorm.DB {
func qdMainDB(t *testing.T, log *slog.Logger) *database.Database {
t.Helper()
sqlDB, err := database.OpenSQLite(
filepath.Join(t.TempDir(), "main-gormlog.db"),
database.SQLiteModeCreate,
)
db, err := database.Open(t.TempDir(), log)
require.NoError(t, err)
t.Cleanup(func() { _ = sqlDB.Close() })
db, err := gorm.Open(
sqlite.Dialector{Conn: sqlDB},
&gorm.Config{Logger: gormlog.New(log)},
)
require.NoError(t, err)
require.NoError(t, db.AutoMigrate(
&database.Webhook{},
&database.Target{},
))
t.Cleanup(func() { _ = db.Close() })
return db
}
@@ -106,20 +89,20 @@ func TestQueueDepthSample_LogsNoBoundValue(t *testing.T) {
))
mainDB := qdMainDB(t, log)
dbMgr := database.NewTestWebhookDBManagerWithLogger(
t.TempDir(), log,
dbMgr := databasetest.NewWebhookDBManagerWithLogger(
t, t.TempDir(), log,
)
webhookID := uuid.New().String()
webhookDB := iSeedWebhookDB(t, dbMgr, webhookID)
iCreateWebhook(t, mainDB, webhookID, "queue-depth-gormlog")
iCreateWebhook(t, mainDB.DB(), webhookID, "queue-depth-gormlog")
targetID := uuid.New().String()
iCreateTarget(t, mainDB, targetID, webhookID,
iCreateTarget(t, mainDB.DB(), targetID, webhookID,
"queue-depth-gormlog-target", database.TargetTypeHTTP,
iHTTPConfig("https://example.com/hook"), 3,
"https://example.com/hook", 3,
)
event := iSeedEvent(
@@ -136,7 +119,7 @@ func TestQueueDepthSample_LogsNoBoundValue(t *testing.T) {
)
engine := delivery.NewTestEngineWithDB(
database.NewTestDatabase(mainDB),
mainDB,
dbMgr,
log,
&http.Client{Timeout: 5 * time.Second},
internal/delivery/recovery_durability_test.go
+114 -19
View File
@@ -66,7 +66,7 @@ func TestRecoverySkipsPendingWithSuccessfulResult(
iCreateTarget(t, s.MainDB, targetID,
s.WebhookID, "already-delivered",
database.TargetTypeLog, "", 0,
database.TargetTypeLog, "", 1,
)
event := iSeedEvent(
@@ -88,7 +88,7 @@ func TestRecoverySkipsPendingWithSuccessfulResult(
)
s.Engine.ExportRecoverPendingDeliveries(
context.Background(), s.WebhookDB, s.WebhookID,
context.Background(), s.WebhookDB, s.WebhookID, atStart,
)
select {
@@ -129,7 +129,7 @@ func TestRecoveryContinuesTheAttemptNumbering(t *testing.T) {
iCreateTarget(t, s.MainDB, targetID,
s.WebhookID, "numbering",
database.TargetTypeLog, "", 0,
database.TargetTypeLog, "", 1,
)
event := iSeedEvent(
@@ -145,7 +145,7 @@ func TestRecoveryContinuesTheAttemptNumbering(t *testing.T) {
rSeedResult(t, s.WebhookDB, d.ID, 2, false)
s.Engine.ExportRecoverPendingDeliveries(
context.Background(), s.WebhookDB, s.WebhookID,
context.Background(), s.WebhookDB, s.WebhookID, atStart,
)
select {
@@ -183,8 +183,8 @@ func TestSweepRecoversStrandedPending(t *testing.T) {
database.DeliveryStatusPending,
)
s.Engine.ExportSweepWebhookRetries(
context.Background(), s.WebhookID,
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID, onTick,
)
select {
@@ -230,7 +230,7 @@ func TestSweepClaimsAStrandedDeliveryOnlyOnce(t *testing.T) {
ctx := context.Background()
s.Engine.ExportSweepWebhookRetries(ctx, s.WebhookID)
s.Engine.ExportRecoverWebhookDeliveries(ctx, s.WebhookID, onTick)
select {
case task := <-s.Engine.ExportDeliveryCh():
@@ -246,7 +246,7 @@ func TestSweepClaimsAStrandedDeliveryOnlyOnce(t *testing.T) {
database.DeliveryStatusPending,
)
s.Engine.ExportSweepWebhookRetries(ctx, s.WebhookID)
s.Engine.ExportRecoverWebhookDeliveries(ctx, s.WebhookID, onTick)
select {
case task := <-s.Engine.ExportDeliveryCh():
@@ -258,9 +258,9 @@ func TestSweepClaimsAStrandedDeliveryOnlyOnce(t *testing.T) {
}
}
// TestSweepSettlesStrandedPendingWithoutResending is the sweep's own
// version of the reconcile: a stranded delivery holding a successful
// result is settled where it stands, and the receiver hears nothing.
// TestSweepSettlesStrandedPendingWithoutResending is the reconcile on a
// tick of the sweep: a stranded delivery holding a successful result is
// settled where it stands, and the receiver hears nothing.
func TestSweepSettlesStrandedPendingWithoutResending(
t *testing.T,
) {
@@ -280,8 +280,8 @@ func TestSweepSettlesStrandedPendingWithoutResending(
rSeedResult(t, s.WebhookDB, d.ID, 1, true)
rAgePending(t, s.WebhookDB, d.ID)
s.Engine.ExportSweepWebhookRetries(
context.Background(), s.WebhookID,
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID, onTick,
)
select {
@@ -312,8 +312,9 @@ func TestSweepSettlesStrandedPendingWithoutResending(
// TestFailedResultWriteLeavesDeliveryRecoverable is the rule the
// targets now follow: a bookkeeping write that fails must not advance
// the status, because pending and retrying are the states the sweeps
// recover and delivered is a claim the database refused to record.
// the status, because pending and retrying are the states the recovery
// routine recovers and delivered is a claim the database refused to
// record.
func TestFailedResultWriteLeavesDeliveryRecoverable(
t *testing.T,
) {
@@ -354,16 +355,17 @@ func TestFailedResultWriteLeavesDeliveryRecoverable(
Status: database.DeliveryStatusPending,
Event: event,
Target: database.Target{
Name: "unwritable",
Type: database.TargetTypeHTTP,
Config: iHTTPConfig(ts.URL),
Name: "unwritable",
Type: database.TargetTypeHTTP,
URL: ts.URL,
MaxAttempts: 1,
},
}
full.ID = d.ID
s.Engine.ExportDeliverHTTP(
context.Background(), s.WebhookDB, full,
&delivery.Task{DeliveryID: d.ID, AttemptNum: 1},
&delivery.Task{DeliveryID: d.ID, MaxAttempts: 1, AttemptNum: 1},
)
assert.Equal(
@@ -376,3 +378,96 @@ func TestFailedResultWriteLeavesDeliveryRecoverable(
database.DeliveryStatusPending,
)
}
// TestFailedResultWriteWithRetriesLeavesDeliveryRecoverable is the same
// rule for a target with retries: whatever the receiver answered, the
// delivery stays pending and no retry is scheduled. The circuit breaker
// still learns the answer, because it describes the target's health,
// not the database's.
func TestFailedResultWriteWithRetriesLeavesDeliveryRecoverable(
t *testing.T,
) {
t.Parallel()
// The "send succeeded" case starts with the breaker tripped open,
// so the delivery goes out as its probe and only a recorded
// success closes it again.
tests := []struct {
name string
answer int
tripped bool
wantBreaker delivery.CircuitState
}{
{"send succeeded", http.StatusOK, true, delivery.CircuitClosed},
{"send failed", http.StatusBadGateway, false, delivery.CircuitOpen},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
s := newISetup(t)
targetID := uuid.New().String()
ts := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(tc.answer)
},
))
defer ts.Close()
event := iSeedEvent(
t, s.WebhookDB, s.WebhookID, `{"unwritable":true}`,
)
d := iSeedDelivery(
t, s.WebhookDB, event.ID, targetID,
database.DeliveryStatusPending,
)
require.NoError(
t,
s.WebhookDB.Exec("drop table delivery_results").Error,
)
// A single failure opens this breaker, and with no
// cooldown an open breaker lets the next delivery
// through as a probe.
cb := delivery.NewTestCircuitBreaker(1, 0)
if tc.tripped {
cb.RecordFailure()
}
s.Engine.ExportSetCircuitBreaker(targetID, cb)
full := &database.Delivery{
EventID: event.ID,
TargetID: targetID,
Status: database.DeliveryStatusPending,
Event: event,
Target: database.Target{
Name: "unwritable",
Type: database.TargetTypeHTTP,
URL: ts.URL,
MaxAttempts: 3,
},
}
full.ID = d.ID
sched := &recordingScheduler{}
s.Engine.ExportDeliverHTTPWithScheduler(
context.Background(), s.WebhookDB, full,
&delivery.Task{
DeliveryID: d.ID, TargetID: targetID,
MaxAttempts: 3, AttemptNum: 1,
},
sched,
)
iAssertStatus(t, s.WebhookDB, d.ID, database.DeliveryStatusPending)
assert.Empty(t, sched.delays, "no retry may be scheduled")
assert.Equal(t, tc.wantBreaker, cb.State())
})
}
}
internal/delivery/recovery_test.go
+389
View File
@@ -0,0 +1,389 @@
package delivery_test
import (
"context"
"log/slog"
"net/http"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/database/databasetest"
"sneak.berlin/go/webhooker/internal/delivery"
)
// These tests cover the recovery routine, which the engine runs once
// when it starts and then on every tick of the sweep
// (https://git.eeqj.de/sneak/webhooker/issues/542). Every case runs it
// both ways, each on a webhook of its own.
// atStart and onTick are the two values of the recovery routine's
// atStart parameter: its run when the engine starts, and a tick of the
// sweep.
const (
atStart = true
onTick = false
)
// recoveryRun names one of the two runs of the recovery routine.
type recoveryRun struct {
name string
atStart bool
}
// recoveryRuns returns the routine's run at start and a tick.
func recoveryRuns() []recoveryRun {
return []recoveryRun{
{"at start", atStart},
{"on a tick", onTick},
}
}
// recoverySetup gives a fresh webhook its main-database row and an http
// target with retries, and returns the setup and the target's ID. No
// worker runs, so whatever the routine queues stays in the channels for
// the test to read, and nothing is sent to the target.
func recoverySetup(t *testing.T) (iSetup, string) {
t.Helper()
s := newISetup(t)
targetID := uuid.New().String()
iCreateWebhook(t, s.MainDB, s.WebhookID, "recovery")
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "recovery",
database.TargetTypeHTTP,
"http://example.com/hook", 20,
)
return s, targetID
}
// seedRecoveryDelivery seeds a delivery at status with the given
// recorded attempts, failed but for the last when lastSucceeded is set,
// all recorded now.
func seedRecoveryDelivery(
t *testing.T,
s iSetup,
targetID string,
status database.DeliveryStatus,
attempts int,
lastSucceeded bool,
) string {
t.Helper()
event := iSeedEvent(t, s.WebhookDB, s.WebhookID, `{"recovery":true}`)
d := iSeedDelivery(t, s.WebhookDB, event.ID, targetID, status)
for n := 1; n <= attempts; n++ {
rSeedResult(
t, s.WebhookDB, d.ID, n, lastSucceeded && n == attempts,
)
}
return d.ID
}
// backdateAttempts moves every recorded attempt of a delivery an hour
// into the past, long past the backoff after its first attempt.
func backdateAttempts(t *testing.T, s iSetup, deliveryID string) {
t.Helper()
require.NoError(t, s.WebhookDB.Model(&database.DeliveryResult{}).
Where("delivery_id = ?", deliveryID).
UpdateColumn("created_at", s.WebhookDB.NowFunc().Add(-time.Hour)).
Error)
}
// requireQueued waits for a task on ch. A retrying delivery reaches the
// retry channel through a timer, even one due now, so it is waited for.
func requireQueued(
t *testing.T, ch <-chan delivery.Task,
) delivery.Task {
t.Helper()
select {
case task := <-ch:
return task
case <-time.After(5 * time.Second):
t.Fatal("the recovery routine queued nothing")
return delivery.Task{}
}
}
// assertNothingQueued asserts the engine queues nothing, giving a retry
// timer that is due now the moment it needs to fire.
func assertNothingQueued(t *testing.T, e *delivery.Engine) {
t.Helper()
select {
case task := <-e.ExportDeliveryCh():
t.Errorf("queued pending delivery %s", task.DeliveryID)
case task := <-e.ExportRetryCh():
t.Errorf("queued retrying delivery %s", task.DeliveryID)
case <-time.After(200 * time.Millisecond):
}
}
// TestRecovery_RetryingBackoffPassed: a retrying delivery whose backoff
// is over is queued for its next attempt at once.
func TestRecovery_RetryingBackoffPassed(t *testing.T) {
t.Parallel()
for _, run := range recoveryRuns() {
t.Run(run.name, func(t *testing.T) {
t.Parallel()
s, targetID := recoverySetup(t)
id := seedRecoveryDelivery(
t, s, targetID, database.DeliveryStatusRetrying, 1, false,
)
backdateAttempts(t, s, id)
s.Engine.ExportRecoverDeliveries(context.Background(), run.atStart)
task := requireQueued(t, s.Engine.ExportRetryCh())
assert.Equal(t, id, task.DeliveryID)
assert.Equal(t, 2, task.AttemptNum)
iAssertStatus(t, s.WebhookDB, id, database.DeliveryStatusRetrying)
})
}
}
// TestRecovery_RetryingBackoffNotPassed: a retrying delivery whose last
// attempt was just now waits out the rest of its backoff, ten attempts
// in, eight and a half minutes. It is not sent now, and the engine holds
// it, so a later tick leaves it to the timer.
func TestRecovery_RetryingBackoffNotPassed(t *testing.T) {
t.Parallel()
for _, run := range recoveryRuns() {
t.Run(run.name, func(t *testing.T) {
t.Parallel()
s, targetID := recoverySetup(t)
id := seedRecoveryDelivery(
t, s, targetID, database.DeliveryStatusRetrying, 10, false,
)
s.Engine.ExportRecoverDeliveries(context.Background(), run.atStart)
assertNothingQueued(t, s.Engine)
iAssertStatus(t, s.WebhookDB, id, database.DeliveryStatusRetrying)
assert.Equal(t, 1, s.Engine.ExportInflightHeld(),
"no retry timer holds the delivery",
)
})
}
}
// TestRecovery_PendingOlderThanTheAgeBound: a pending delivery untouched
// for longer than the sweep's age bound is sent by either run.
func TestRecovery_PendingOlderThanTheAgeBound(t *testing.T) {
t.Parallel()
for _, run := range recoveryRuns() {
t.Run(run.name, func(t *testing.T) {
t.Parallel()
s, targetID := recoverySetup(t)
id := seedRecoveryDelivery(
t, s, targetID, database.DeliveryStatusPending, 0, false,
)
rAgePending(t, s.WebhookDB, id)
s.Engine.ExportRecoverDeliveries(context.Background(), run.atStart)
task := requireQueued(t, s.Engine.ExportDeliveryCh())
assert.Equal(t, id, task.DeliveryID)
assert.Equal(t, 1, task.AttemptNum)
})
}
}
// TestRecovery_TaskCarriesEveryTargetSetting: the task the recovery
// routine queues carries every one of the target's settings, all six
// set here.
// They are copied into the task one field at a time, and a field left
// out is sent empty without any error.
func TestRecovery_TaskCarriesEveryTargetSetting(t *testing.T) {
t.Parallel()
for _, run := range recoveryRuns() {
t.Run(run.name, func(t *testing.T) {
t.Parallel()
s := newISetup(t)
iCreateWebhook(t, s.MainDB, s.WebhookID, "recovery")
target := database.Target{
WebhookID: s.WebhookID,
Name: "recovery",
Type: database.TargetTypeHTTP,
Active: true,
URL: "http://example.com/hook",
Headers: map[string]string{headerAuthorization: "Bearer t"},
TimeoutSeconds: 7,
ForwardQuery: true,
ArchiveExpiry: time.Hour,
ArchiveRotation: "daily",
MaxAttempts: 1,
}
require.NoError(t, s.MainDB.Create(&target).Error)
id := seedRecoveryDelivery(
t, s, target.ID, database.DeliveryStatusPending, 0, false,
)
rAgePending(t, s.WebhookDB, id)
s.Engine.ExportRecoverDeliveries(context.Background(), run.atStart)
task := requireQueued(t, s.Engine.ExportDeliveryCh())
assert.Equal(t, target.ID, task.TargetID)
assert.Equal(t, target.URL, task.TargetURL)
assert.Equal(t, target.Headers, task.TargetHeaders)
assert.Equal(t, target.TimeoutSeconds, task.TimeoutSeconds)
assert.Equal(t, target.ForwardQuery, task.ForwardQuery)
assert.Equal(t, target.ArchiveExpiry, task.ArchiveExpiry)
assert.Equal(t, target.ArchiveRotation, task.ArchiveRotation)
})
}
}
// TestRecovery_PendingYoungerThanTheAgeBound: a pending delivery touched
// a moment ago, as one a stop cut off is, is sent at start, which is how
// it reaches its target on the next start
// (https://git.eeqj.de/sneak/webhooker/issues/527). A tick leaves it.
func TestRecovery_PendingYoungerThanTheAgeBound(t *testing.T) {
t.Parallel()
for _, run := range recoveryRuns() {
t.Run(run.name, func(t *testing.T) {
t.Parallel()
s, targetID := recoverySetup(t)
id := seedRecoveryDelivery(
t, s, targetID, database.DeliveryStatusPending, 0, false,
)
s.Engine.ExportRecoverDeliveries(context.Background(), run.atStart)
if run.atStart {
task := requireQueued(t, s.Engine.ExportDeliveryCh())
assert.Equal(t, id, task.DeliveryID)
return
}
assertNothingQueued(t, s.Engine)
iAssertStatus(t, s.WebhookDB, id, database.DeliveryStatusPending)
assert.Zero(t, s.Engine.ExportInflightHeld())
})
}
}
// TestRecovery_AlreadySucceeded: a pending or retrying delivery whose
// success is recorded but whose status write was lost is marked
// delivered where it stands, and not sent again.
func TestRecovery_AlreadySucceeded(t *testing.T) {
t.Parallel()
for _, run := range recoveryRuns() {
t.Run(run.name, func(t *testing.T) {
t.Parallel()
s, targetID := recoverySetup(t)
pending := seedRecoveryDelivery(
t, s, targetID, database.DeliveryStatusPending, 1, true,
)
rAgePending(t, s.WebhookDB, pending)
retrying := seedRecoveryDelivery(
t, s, targetID, database.DeliveryStatusRetrying, 2, true,
)
backdateAttempts(t, s, retrying)
s.Engine.ExportRecoverDeliveries(context.Background(), run.atStart)
assertNothingQueued(t, s.Engine)
for id, attempts := range map[string]int{pending: 1, retrying: 2} {
iAssertStatus(
t, s.WebhookDB, id, database.DeliveryStatusDelivered,
)
assert.Len(t, iResults(t, s.WebhookDB, id), attempts,
"marking the delivery delivered recorded an attempt",
)
}
})
}
}
// TestRecovery_MissingDatabase: a tick leaves a webhook whose database
// file is missing alone, and the run at start opens it, which creates an
// empty one and reports the loss (see
// TestRecoverDeliveries_ReportsAMissingWebhookDatabase).
func TestRecovery_MissingDatabase(t *testing.T) {
t.Parallel()
mainDB := iMainDB(t)
webhookID := uuid.New().String()
iCreateWebhook(t, mainDB.DB(), webhookID, "missing-database")
dbMgr := databasetest.NewWebhookDBManager(t, t.TempDir())
t.Cleanup(func() { _ = dbMgr.CloseAll() })
engine := delivery.NewTestEngineWithDB(
mainDB, dbMgr,
slog.New(slog.DiscardHandler),
&http.Client{Timeout: 5 * time.Second}, 1,
)
engine.ExportRecoverDeliveries(context.Background(), onTick)
assert.False(t, dbMgr.DBExists(webhookID),
"a tick created the missing database",
)
engine.ExportRecoverDeliveries(context.Background(), atStart)
assert.True(t, dbMgr.DBExists(webhookID),
"the run at start did not open the missing database",
)
}
// TestRecovery_LeavesADeliveryTheEngineHolds: a delivery the engine is
// working on, has queued, or holds a retry timer for is left alone by
// either run, pending and retrying alike, however old.
func TestRecovery_LeavesADeliveryTheEngineHolds(t *testing.T) {
t.Parallel()
for _, run := range recoveryRuns() {
t.Run(run.name, func(t *testing.T) {
t.Parallel()
s, targetID := recoverySetup(t)
pending := seedRecoveryDelivery(
t, s, targetID, database.DeliveryStatusPending, 0, false,
)
rAgePending(t, s.WebhookDB, pending)
retrying := seedRecoveryDelivery(
t, s, targetID, database.DeliveryStatusRetrying, 1, false,
)
backdateAttempts(t, s, retrying)
require.True(t, s.Engine.ExportRetainDelivery(pending))
require.True(t, s.Engine.ExportRetainDelivery(retrying))
s.Engine.ExportRecoverDeliveries(context.Background(), run.atStart)
assertNothingQueued(t, s.Engine)
iAssertStatus(t, s.WebhookDB, pending, database.DeliveryStatusPending)
iAssertStatus(t, s.WebhookDB, retrying, database.DeliveryStatusRetrying)
assert.Equal(t, 2, s.Engine.ExportInflightHeld())
})
}
}
internal/delivery/redirect_test.go
+10 -17
View File
@@ -85,19 +85,14 @@ func deliverWithProbeHeaders(
database.DeliveryStatusPending,
)
cfg, err := json.Marshal(delivery.HTTPTargetConfig{
URL: targetURL,
Headers: map[string]string{
probeHeaderName: probeHeaderValue,
},
})
require.NoError(t, err)
body := event.Body
task := iTask(
d, event, s.WebhookID, targetID,
"redirect-target", string(cfg), 0, 1, &body,
"redirect-target", targetURL, 1, 1, &body,
)
task.TargetHeaders = map[string]string{
probeHeaderName: probeHeaderValue,
}
s.Engine.ExportProcessNewTask(context.TODO(), &task)
@@ -309,8 +304,7 @@ func TestRedirectPolicy_StopsAtHopCap(t *testing.T) {
)
client := engine.ExportClientForRequest(
&delivery.HTTPTargetConfig{URL: srv.URL},
[]string{probeHeaderName},
0, []string{probeHeaderName},
)
require.NotNil(t, client.CheckRedirect)
@@ -350,7 +344,7 @@ func TestApplyRequestHeaders_ReportsOriginScopedNames(t *testing.T) {
inbound, err := json.Marshal(map[string][]string{
inboundHeaderName: {inboundHeaderValue},
"Content-Type": {testContentType},
"User-Agent": {"curl/8.7.1"},
headerUserAgent: {"curl/8.7.1"},
"Host": {"inbound.example.com"},
})
require.NoError(t, err)
@@ -369,12 +363,11 @@ func TestApplyRequestHeaders_ReportsOriginScopedNames(t *testing.T) {
Headers: string(inbound),
ContentType: testContentType,
},
&delivery.HTTPTargetConfig{
Headers: map[string]string{
probeHeaderName: probeHeaderValue,
"Content-Type": testContentType,
},
map[string]string{
probeHeaderName: probeHeaderValue,
"Content-Type": testContentType,
},
"webhooker/dev",
)
assert.Equal(t,
internal/delivery/ssrf.go
+105 -33
View File
@@ -25,36 +25,64 @@ var (
errNoIPs = errors.New(
"hostname resolved to no IP addresses",
)
errBlockedIP = errors.New(
"blocked private, reserved or cloud metadata address",
// ErrBlockedPrivateOrReservedIP reports an address in the
// default blocklist's private and reserved ranges,
// blockedNetworks.
ErrBlockedPrivateOrReservedIP = errors.New(
"blocked private or reserved address",
)
// errBlockedPublicMetadata reports a public address on the
// default blocklist, one in blockedPublicNetworks.
errBlockedPublicMetadata = errors.New(
"blocked cloud metadata address",
)
errBlockedMetadata = errors.New(
"blocked link-local or cloud instance metadata " +
"address: ALLOWED_EGRESS_CIDRS cannot open it",
"blocked link-local, cloud instance metadata or " +
"unspecified address: ALLOWED_EGRESS_CIDRS cannot open it",
)
errInvalidScheme = errors.New(
"only http and https are allowed",
)
)
// blockedNetworks is the default blocklist: the private and
// reserved IP ranges, plus the public cloud metadata addresses,
// that are blocked to prevent SSRF attacks. An operator can
// permit specific blocks out of this set with
// ALLOWED_EGRESS_CIDRS; see Guard.
// blockedNetworks and blockedPublicNetworks together are the
// default blocklist: the private and reserved IP ranges, plus
// the public cloud metadata addresses, that are blocked to
// prevent SSRF attacks. An operator can permit specific blocks
// out of this set with ALLOWED_EGRESS_CIDRS; see Guard.
//
// blockedNetworks holds the private and reserved IP ranges.
//
//nolint:gochecknoglobals // package-level network list is appropriate here
var blockedNetworks []*net.IPNet
// alwaysBlockedNetworks are the ranges no configuration can
// open: the link-local blocks and the cloud instance metadata
// endpoints that live outside them. Reaching one is credential
// or user-data theft rather than delivery to an internal
// service, so a supplied CIDR that covers such an address still
// leaves it blocked.
// blockedPublicNetworks holds the default blocklist's public
// addresses, kept apart from blockedNetworks so that they are
// refused as cloud metadata addresses, never as private or
// reserved ones.
//
// Inclusion criterion — an address belongs here only if BOTH
// hold, and every entry below satisfies both:
// A public address belongs on the default blocklist only if it
// hands credentials, user data or bootstrap material to whatever
// can reach it, without the caller presenting anything; it goes
// in this list. A provider's other public addresses are not
// refused, since reaching them can be legitimate and no list of
// them could be complete.
//
//nolint:gochecknoglobals // package-level network list is appropriate here
var blockedPublicNetworks []*net.IPNet
// alwaysBlockedNetworks are the ranges no configuration can
// open, so a supplied CIDR that covers one still leaves it
// blocked. An entry is here for one of two reasons: it is a
// metadata endpoint (the link-local blocks and the cloud
// instance metadata endpoints that live outside them), or it is
// an unspecified address. Reaching a metadata endpoint is
// credential or user-data theft rather than delivery to an
// internal service.
//
// Inclusion criterion for metadata endpoints — one belongs here
// only if BOTH hold, and every metadata entry below satisfies
// both:
//
// 1. It is a fixed address assigned by the provider, or a
// range reserved by IANA — never one the operator chose.
@@ -65,8 +93,8 @@ var blockedNetworks []*net.IPNet
// not cheaply rotated.
//
// Both halves are load-bearing, so use them to refuse a
// candidate and say why. An endpoint disclosing only the
// operator's own inventory (instance id, region, disks, NICs)
// metadata candidate and say why. An endpoint disclosing only
// the operator's own inventory (instance id, region, disks, NICs)
// fails (2): letting a delivery target reach the operator's own
// infrastructure is the feature ALLOWED_EGRESS_CIDRS exists to
// provide. But (2) is not "IAM credentials only" either —
@@ -81,12 +109,21 @@ var blockedNetworks []*net.IPNet
// when it clears both halves. Nothing in this list can be
// reopened, so putting a public address here leaves the operator
// no escape hatch at all — the condition ALLOWED_EGRESS_CIDRS
// exists to remove. Default-block it in blockedNetworks instead,
// which an allowlist can override.
// exists to remove. Default-block it in blockedPublicNetworks
// instead, which an allowlist can override.
//
// This is a criterion, not an enumeration of every metadata
// address in existence.
//
// The unspecified addresses 0.0.0.0 and :: are here for a
// separate reason: they disclose nothing, but no host can have
// either, and on Linux a connection to one reaches this host's
// own loopback. Listing them means an allowlist reaches loopback
// only through an entry that covers a loopback address
// (127.0.0.0/8, ::1/128, 0.0.0.0/0), never through one that
// covers only 0.0.0.0 or :: (0.0.0.0/8, for example). Nothing
// else lives at either address, so refusing them costs nothing.
//
// Every entry is either already in blockedNetworks — this list is
// what makes it unconditional — or an alternate encoding of
// 169.254.169.254 that Contains does not match against
@@ -106,23 +143,49 @@ var alwaysBlockedNetworks []*net.IPNet
//nolint:gochecknoinits // init is the idiomatic way to parse CIDRs once at startup
func init() {
blockedNetworks = mustParseCIDRs([]string{
// IPv4 loopback.
"127.0.0.0/8",
// RFC 1918 private network.
"10.0.0.0/8",
// RFC 1918 private network.
"172.16.0.0/12",
// RFC 1918 private network.
"192.168.0.0/16",
// IPv4 link-local.
"169.254.0.0/16",
// "This network", holding the IPv4 unspecified address 0.0.0.0.
"0.0.0.0/8",
// Carrier-grade NAT shared address space.
"100.64.0.0/10",
// IETF protocol assignments.
"192.0.0.0/24",
// IPv4 documentation (TEST-NET-1).
"192.0.2.0/24",
// Benchmarking.
"198.18.0.0/15",
// IPv4 documentation (TEST-NET-2).
"198.51.100.0/24",
// IPv4 documentation (TEST-NET-3).
"203.0.113.0/24",
// IPv4 multicast.
"224.0.0.0/4",
// Reserved, including the broadcast address.
"240.0.0.0/4",
// IPv6 loopback.
"::1/128",
// IPv6 unspecified address.
"::/128",
// IPv6 unique local addresses.
"fc00::/7",
// IPv6 link-local.
"fe80::/10",
// IPv6 multicast.
"ff00::/8",
// IPv6 documentation.
"2001:db8::/32",
})
blockedPublicNetworks = mustParseCIDRs([]string{
// Azure WireServer, a public address that serves VM credentials.
"168.63.129.16/32",
})
@@ -179,6 +242,14 @@ func init() {
// allowlist from opening it.
"192.0.0.192/32",
// The unspecified addresses, each of which reaches this
// host's loopback on Linux.
//
// IPv4 unspecified address, inside the blocked 0.0.0.0/8.
"0.0.0.0/32",
// IPv6 unspecified address.
"::/128",
// 169.254.169.254 as an IPv4-compatible IPv6 address.
"::a9fe:a9fe/128",
// 169.254.169.254 behind the NAT64 well-known prefix.
@@ -218,13 +289,6 @@ func matchesAny(networks []*net.IPNet, ip net.IP) bool {
return false
}
// isBlockedIP checks whether an IP address falls within
// the default blocklist, before any operator allowlist is
// considered.
func isBlockedIP(ip net.IP) bool {
return matchesAny(blockedNetworks, ip)
}
// Guard makes every SSRF decision in the process.
//
// It holds the operator's ALLOWED_EGRESS_CIDRS allowlist and
@@ -322,10 +386,12 @@ func (g *Guard) allows(ip net.IP) bool {
// The order is the policy:
//
// 1. alwaysBlockedNetworks is refused before the allowlist is
// consulted, so no configured CIDR reaches link-local or a
// cloud metadata endpoint at a non-public address.
// consulted, so no configured CIDR reaches link-local, a
// cloud metadata endpoint at a non-public address, or an
// unspecified address.
// 2. The allowlist is consulted next, so a listed private
// network becomes reachable.
// network, or a listed public address on the default
// blocklist, becomes reachable.
// 3. Everything else keeps the default blocklist's answer.
func (g *Guard) checkIP(ip net.IP) error {
if matchesAny(alwaysBlockedNetworks, ip) {
@@ -338,9 +404,15 @@ func (g *Guard) checkIP(ip net.IP) error {
return nil
}
if isBlockedIP(ip) {
if matchesAny(blockedNetworks, ip) {
return fmt.Errorf(
"target IP %s: %w", ip, errBlockedIP,
"target IP %s: %w", ip, ErrBlockedPrivateOrReservedIP,
)
}
if matchesAny(blockedPublicNetworks, ip) {
return fmt.Errorf(
"target IP %s: %w", ip, errBlockedPublicMetadata,
)
}
internal/delivery/ssrf_allowlist_test.go
+131 -12
View File
@@ -23,6 +23,10 @@ const (
metadataIP = "169.254.169.254"
metadataURL = "http://" + metadataIP + "/latest/meta-data/"
// linkLocalIPv4 is the IPv4 link-local block, which holds
// metadataIP.
linkLocalIPv4 = "169.254.0.0/16"
// loopbackHookURL is a target on this host: blocked by
// default, reachable only once an operator allowlists
// loopback.
@@ -164,12 +168,13 @@ func TestGuardAllowlist_UnlistedPrivateStillRefused(t *testing.T) {
// TestGuardAllowlist_MetadataAlwaysRefused is the load-bearing
// case: cloud instance metadata endpoints are credential theft
// rather than delivery to an internal service, so no allowlist
// reaches one. Every guard below names a CIDR that covers its
// target — including 0.0.0.0/0, ::/0, and the ordinary ULA and
// CGNAT blocks an operator would really list — and the address
// must stay refused anyway, on both the validation and the
// delivery path.
// rather than delivery to an internal service, and the
// unspecified addresses 0.0.0.0 and :: reach this host's loopback
// on Linux, so no allowlist reaches any of them. Every guard
// below names a CIDR that covers its target — including
// 0.0.0.0/0, ::/0, and the ordinary ULA and CGNAT blocks an
// operator would really list — and the address must stay
// refused anyway, on both the validation and the delivery path.
func TestGuardAllowlist_MetadataAlwaysRefused(t *testing.T) {
t.Parallel()
@@ -215,15 +220,17 @@ type metadataAlwaysRefusedCase struct {
}
// metadataAlwaysRefusedCases enumerates every unconditionally
// blocked address together with an allowlist entry that would
// otherwise reach it. Split by family of address only to stay
// under the function-length limit.
// blocked address (link-local, the cloud metadata endpoints and
// the unspecified addresses) together with an allowlist entry
// that would otherwise reach it. Split by family of address only
// to stay under the function-length limit.
func metadataAlwaysRefusedCases() []metadataAlwaysRefusedCase {
cases := linkLocalRefusedCases()
cases = append(cases, ulaMetadataRefusedCases()...)
cases = append(cases, ipv4MetadataRefusedCases()...)
cases = append(cases, encodedMetadataRefusedCases()...)
return append(cases, encodedMetadataRefusedCases()...)
return append(cases, unspecifiedRefusedCases()...)
}
// linkLocalRefusedCases covers the link-local blocks, including
@@ -237,7 +244,7 @@ func linkLocalRefusedCases() []metadataAlwaysRefusedCase {
},
{
name: "whole link-local block",
allow: "169.254.0.0/16",
allow: linkLocalIPv4,
target: metadataURL,
},
{
@@ -363,6 +370,23 @@ func encodedMetadataRefusedCases() []metadataAlwaysRefusedCase {
}
}
// unspecifiedRefusedCases covers the unspecified addresses, each
// of which reaches this host's loopback on Linux.
func unspecifiedRefusedCases() []metadataAlwaysRefusedCase {
return []metadataAlwaysRefusedCase{
{
name: "IPv4 unspecified address under 0.0.0.0/0",
allow: allowAllIPv4,
target: "http://0.0.0.0:8080/hook",
},
{
name: "IPv6 unspecified address under ::/0",
allow: allowAllIPv6,
target: "http://[::]:8080/hook",
},
}
}
// TestGuardAllowlist_PublicUnaffected asserts the allowlist does
// not narrow anything: public addresses were reachable before it
// existed and stay reachable, whether or not a list is set.
@@ -412,6 +436,9 @@ func TestGuardAllowlist_AzureWireServerReopenable(t *testing.T) {
"WireServer must be refused by the default blocklist, "+
"which an allowlist can override",
)
require.NotErrorIs(t, err, delivery.ErrBlockedPrivateOrReservedIP,
"WireServer is public, not private or reserved",
)
assertDialRefused(t, defaultGuard, target)
@@ -496,7 +523,7 @@ func TestAlwaysBlockedNetworks_PinnedSet(t *testing.T) {
want := []string{
// IPv4 link-local: the 169.254.169.254 metadata
// service on AWS, Azure and others.
"169.254.0.0/16",
linkLocalIPv4,
// IPv6 link-local.
"fe80::/10",
// AWS IPv6 IMDS, inside the ULA space an operator may
@@ -517,6 +544,10 @@ func TestAlwaysBlockedNetworks_PinnedSet(t *testing.T) {
// Oracle Cloud Classic metadata, inside the blocked
// 192.0.0.0/24.
"192.0.0.192/32",
// The IPv4 and IPv6 unspecified addresses, each of
// which reaches this host's loopback on Linux.
"0.0.0.0/32",
"::/128",
// 169.254.169.254 as an IPv4-compatible IPv6 address.
"::a9fe:a9fe/128",
// 169.254.169.254 behind the NAT64 well-known prefix.
@@ -526,6 +557,94 @@ func TestAlwaysBlockedNetworks_PinnedSet(t *testing.T) {
assert.Equal(t, want, got)
}
// TestDefaultBlocklist_PinnedSet pins each list of the default
// blocklist on its own, the private and reserved ranges in
// blockedNetworks and the public addresses in
// blockedPublicNetworks, so moving an entry from one list to the
// other fails it. For the first address of each entry it then
// checks that the default guard refuses it, and that listing the
// entry in ALLOWED_EGRESS_CIDRS opens it unless the unconditional
// set holds that address.
func TestDefaultBlocklist_PinnedSet(t *testing.T) {
t.Parallel()
// public marks an entry of blockedPublicNetworks; every other
// entry belongs in blockedNetworks.
tests := []struct {
cidr string
public bool
reopenable bool
}{
{cidr: "127.0.0.0/8", reopenable: true},
{cidr: "10.0.0.0/8", reopenable: true},
{cidr: "172.16.0.0/12", reopenable: true},
{cidr: "192.168.0.0/16", reopenable: true},
{cidr: linkLocalIPv4, reopenable: false},
// Its first address, 0.0.0.0, is in the unconditional set.
{cidr: "0.0.0.0/8", reopenable: false},
{cidr: "100.64.0.0/10", reopenable: true},
{cidr: "192.0.0.0/24", reopenable: true},
{cidr: "192.0.2.0/24", reopenable: true},
{cidr: "198.18.0.0/15", reopenable: true},
{cidr: "198.51.100.0/24", reopenable: true},
{cidr: "203.0.113.0/24", reopenable: true},
{cidr: "224.0.0.0/4", reopenable: true},
{cidr: "240.0.0.0/4", reopenable: true},
{cidr: "::1/128", reopenable: true},
{cidr: "::/128", reopenable: false},
{cidr: "fc00::/7", reopenable: true},
{cidr: "fe80::/10", reopenable: false},
{cidr: "ff00::/8", reopenable: true},
{cidr: "2001:db8::/32", reopenable: true},
{cidr: "168.63.129.16/32", public: true, reopenable: true},
}
wantPrivate := make([]string, 0, len(tests))
wantPublic := make([]string, 0, len(tests))
for _, tt := range tests {
if tt.public {
wantPublic = append(wantPublic, tt.cidr)
} else {
wantPrivate = append(wantPrivate, tt.cidr)
}
}
gotPrivate := make([]string, 0, len(tests))
for _, n := range delivery.ExportBlockedNetworks() {
gotPrivate = append(gotPrivate, n.String())
}
gotPublic := make([]string, 0, len(tests))
for _, n := range delivery.ExportBlockedPublicNetworks() {
gotPublic = append(gotPublic, n.String())
}
assert.ElementsMatch(t, wantPrivate, gotPrivate, "blockedNetworks")
assert.ElementsMatch(t, wantPublic, gotPublic, "blockedPublicNetworks")
for _, tt := range tests {
t.Run(tt.cidr, func(t *testing.T) {
t.Parallel()
prefix := netip.MustParsePrefix(tt.cidr)
ip := net.IP(prefix.Addr().AsSlice())
require.Error(t,
delivery.NewTestGuard().ExportCheckIP(ip),
"the default guard must refuse %s", ip,
)
err := delivery.NewTestGuard(prefix).ExportCheckIP(ip)
if tt.reopenable {
assert.NoError(t, err, "listing %s must open it", tt.cidr)
} else {
assert.Error(t, err, "listing %s must not open it", tt.cidr)
}
})
}
}
// requireLoopback fails the test unless rawURL's host is a
// loopback address, so the allowlist test cannot silently stop
// exercising a blocked range.
internal/delivery/ssrf_test.go
+42 -4
View File
@@ -10,7 +10,7 @@ import (
"sneak.berlin/go/webhooker/internal/delivery"
)
func TestIsBlockedIP_PrivateRanges(t *testing.T) {
func TestGuardCheckIP_PrivateRanges(t *testing.T) {
t.Parallel()
tests := []struct {
@@ -56,12 +56,14 @@ func TestIsBlockedIP_PrivateRanges(t *testing.T) {
"failed to parse IP %s", tt.ip,
)
refused := delivery.NewTestGuard().ExportCheckIP(ip) != nil
assert.Equal(t,
tt.blocked,
delivery.ExportIsBlockedIP(ip),
"isBlockedIP(%s) = %v, want %v",
refused,
"default guard refuses %s = %v, want %v",
tt.ip,
delivery.ExportIsBlockedIP(ip),
refused,
tt.blocked,
)
})
@@ -99,6 +101,42 @@ func TestValidateTargetURL_Blocked(t *testing.T) {
}
}
// TestDefaultGuard_RefusesUnspecifiedMulticastAndDocumentation
// covers the unspecified addresses and the IPv6 multicast and
// documentation ranges: with no allowlist set, each is refused
// both when a target is created and when a delivery dials it.
func TestDefaultGuard_RefusesUnspecifiedMulticastAndDocumentation(
t *testing.T,
) {
t.Parallel()
guard := delivery.NewTestGuard()
targets := []string{
// The unspecified addresses. On Linux a connection to
// either reaches this host's loopback.
"http://0.0.0.0:8080/hook",
"http://[::]:8080/hook",
// IPv6 multicast, all nodes.
"http://[ff02::1]/hook",
// IPv6 documentation.
"http://[2001:db8::1]/hook",
}
for _, target := range targets {
t.Run(target, func(t *testing.T) {
t.Parallel()
require.Error(t,
guard.ValidateTargetURL(context.Background(), target),
"%s must be refused at target creation", target,
)
assertDialRefused(t, guard, target)
})
}
}
func TestValidateTargetURL_Allowed(t *testing.T) {
t.Parallel()
internal/delivery/store.go
+164
View File
@@ -0,0 +1,164 @@
package delivery
import (
"fmt"
"gorm.io/gorm"
"sneak.berlin/go/webhooker/internal/database"
)
// StoreEvent stores event with one pending delivery for each of its
// webhook's active targets, and adds them to the totals, in one
// transaction, then queues the deliveries. It returns how many
// deliveries it created.
//
// The receiver stores every event it accepts through here, and a
// resubmit stores its copy of a stored event, so every event is
// stored, retried, SSRF-guarded and circuit-broken the same way however
// it entered the system. CreateReplay is the only other place a
// delivery is created.
//
// The targets are the webhook's active ones as they stand now: a
// target created after a resubmitted event first arrived is included,
// and an inactive one is skipped. An event with no active target is
// stored with no deliveries.
func (e *Engine) StoreEvent(event *database.Event) (int, error) {
var targets []database.Target
err := e.database.DB().Where(
"webhook_id = ? AND active = ?", event.WebhookID, true,
).Find(&targets).Error
if err != nil {
return 0, fmt.Errorf("loading active targets: %w", err)
}
webhookDB, err := e.dbManager.GetDB(event.WebhookID)
if err != nil {
return 0, fmt.Errorf("getting webhook database: %w", err)
}
event.BodyBytes = int64(len(event.Body))
tasks := make([]Task, 0, len(targets))
err = webhookDB.Transaction(func(tx *gorm.DB) error {
err := tx.Create(event).Error
if err != nil {
return fmt.Errorf("creating event: %w", err)
}
for i := range targets {
dlv, err := createDelivery(tx, event.ID, targets[i].ID, false)
if err != nil {
return err
}
tasks = append(tasks, buildTask(
dlv, event.WebhookID, event, &targets[i], 1,
))
}
err = database.AddEventTotals(tx, database.EventTotals{
Events: 1, LastEventAt: &event.CreatedAt,
})
if err != nil {
return err
}
// A resubmitted copy did not arrive on its entrypoint's URL, so
// it leaves the entrypoint's last event as it is.
if event.ResubmittedFromID != nil {
return nil
}
return database.AddEntrypointTotals(tx, database.EntrypointTotals{
EntrypointID: event.EntrypointID, LastEventAt: event.CreatedAt,
})
})
if err != nil {
return 0, err
}
// Counted here, after the commit: an event exists once it is
// durably stored, which is what the delivery counters are compared
// against on a dashboard. A resubmitted event counts too, because
// it produces deliveries that the delivery side counts; the
// resubmit counter is what separates the two.
e.mtr.EventReceived()
e.notify(tasks)
return len(tasks), nil
}
// CreateReplay creates one more pending delivery of the stored event
// eventID to target, marked as a replay, adds it to the target's totals
// in the same transaction, and queues it. It returns the new
// delivery's id.
//
// The caller passes the target as it stands now and has already
// decided the replay may go ahead. The delivery being replayed is not
// touched.
func (e *Engine) CreateReplay(
webhookID, eventID string, target *database.Target,
) (string, error) {
webhookDB, err := e.dbManager.GetDB(webhookID)
if err != nil {
return "", fmt.Errorf("getting webhook database: %w", err)
}
event, err := e.loadEvent(webhookDB, eventID)
if err != nil {
return "", err
}
var dlv *database.Delivery
err = webhookDB.Transaction(func(tx *gorm.DB) error {
var err error
dlv, err = createDelivery(tx, event.ID, target.ID, true)
return err
})
if err != nil {
return "", err
}
e.notify([]Task{buildTask(dlv, webhookID, &event, target, 1)})
return dlv.ID, nil
}
// createDelivery writes a pending delivery of the event to the target
// on the caller's transaction, and adds it to the target's totals.
//
// Neither Event nor Target is set on the row, and the per-webhook
// database omits associations on every write in any case (see
// internal/database/event_db_isolation.go), so the target's settings
// are never copied into the event database.
func createDelivery(
tx *gorm.DB, eventID, targetID string, replay bool,
) (*database.Delivery, error) {
dlv := &database.Delivery{
EventID: eventID,
TargetID: targetID,
Status: database.DeliveryStatusPending,
Replay: replay,
}
err := tx.Create(dlv).Error
if err != nil {
return nil, fmt.Errorf(
"creating delivery for target %s: %w", targetID, err,
)
}
err = database.AddTargetTotals(tx, database.TargetTotals{
TargetID: targetID, Deliveries: 1,
})
if err != nil {
return nil, err
}
return dlv, nil
}
internal/delivery/store_test.go
+386
View File
@@ -0,0 +1,386 @@
package delivery_test
import (
"net/http"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/delivery"
)
// storeTestTarget creates an active http target of the setup's webhook
// with every one of a target's settings set, so a task that drops one
// of them differs from the target.
func storeTestTarget(
t *testing.T, s iSetup, name string,
) database.Target {
t.Helper()
target := database.Target{
WebhookID: s.WebhookID,
Name: name,
Type: database.TargetTypeHTTP,
Active: true,
URL: "http://93.184.216.34/" + name,
Headers: map[string]string{"X-Api-Key": "k"},
TimeoutSeconds: 7,
ForwardQuery: true,
ArchiveExpiry: time.Hour,
ArchiveRotation: rotationDaily,
MaxAttempts: 3,
}
require.NoError(t, s.MainDB.Create(&target).Error)
return target
}
// newStoreTestEvent returns an event of the setup's webhook as the
// receiver builds one, not yet stored.
func newStoreTestEvent(s iSetup, body string) *database.Event {
return &database.Event{
WebhookID: s.WebhookID,
EntrypointID: uuid.New().String(),
Method: http.MethodPost,
RawQuery: "a=1&b=2",
Headers: `{"X-Test":["yes"]}`,
ContentType: testContentType,
Body: body,
}
}
// queuedTasks takes every task the engine has queued so far. The
// engine in these tests is never started, so nothing else takes them.
func queuedTasks(e *delivery.Engine) []delivery.Task {
var tasks []delivery.Task
for {
select {
case task := <-e.ExportDeliveryCh():
tasks = append(tasks, task)
default:
return tasks
}
}
}
// assertTaskFor proves task carries delivery deliveryID of event to
// target: every one of the target's settings and every event field.
// They are copied into the task one field at a time, and a field left
// out is sent empty without any error.
func assertTaskFor(
t *testing.T,
task delivery.Task,
deliveryID string,
event *database.Event,
target *database.Target,
) {
t.Helper()
assert.Equal(t, deliveryID, task.DeliveryID)
assert.Equal(t, event.ID, task.EventID)
assert.Equal(t, event.WebhookID, task.WebhookID)
assert.Equal(t, event.EntrypointID, task.EntrypointID)
assert.Equal(t, target.ID, task.TargetID)
assert.Equal(t, target.Name, task.TargetName)
assert.Equal(t, target.Type, task.TargetType)
assert.Equal(t, target.URL, task.TargetURL)
assert.Equal(t, target.Headers, task.TargetHeaders)
assert.Equal(t, target.TimeoutSeconds, task.TimeoutSeconds)
assert.Equal(t, target.ForwardQuery, task.ForwardQuery)
assert.Equal(t, target.ArchiveExpiry, task.ArchiveExpiry)
assert.Equal(t, target.ArchiveRotation, task.ArchiveRotation)
assert.Equal(t, target.MaxAttempts, task.MaxAttempts)
assert.Equal(t, event.Method, task.Method)
assert.Equal(t, event.RawQuery, task.RawQuery)
assert.Equal(t, event.Headers, task.Headers)
assert.Equal(t, event.ContentType, task.ContentType)
require.NotNil(t, task.Body)
assert.Equal(t, event.Body, *task.Body)
assert.Equal(t, 1, task.AttemptNum)
}
// storedDeliveries reads every delivery of an event.
func storedDeliveries(
t *testing.T, db *gorm.DB, eventID string,
) []database.Delivery {
t.Helper()
var deliveries []database.Delivery
require.NoError(t, db.Where("event_id = ?", eventID).
Find(&deliveries).Error)
return deliveries
}
// storedEventTotals reads the webhook's event totals.
func storedEventTotals(t *testing.T, db *gorm.DB) database.EventTotals {
t.Helper()
var totals database.EventTotals
require.NoError(t, db.First(&totals).Error)
return totals
}
// countRows counts the rows of model in db.
func countRows(t *testing.T, db *gorm.DB, model any) int64 {
t.Helper()
var n int64
require.NoError(t, db.Model(model).Count(&n).Error)
return n
}
// TestStoreEvent_StoresDeliveriesToActiveTargets proves StoreEvent
// stores the event with one pending delivery to each active target and
// none to an inactive one, adds them to the totals, and queues one task
// per delivery.
func TestStoreEvent_StoresDeliveriesToActiveTargets(t *testing.T) {
t.Parallel()
s := newISetup(t)
targets := []database.Target{
storeTestTarget(t, s, "first"),
storeTestTarget(t, s, "second"),
}
off := storeTestTarget(t, s, "off")
require.NoError(t, s.MainDB.Model(&database.Target{}).
Where("id = ?", off.ID).Update("active", false).Error)
event := newStoreTestEvent(s, `{"hello":"world"}`)
created, err := s.Engine.StoreEvent(event)
require.NoError(t, err)
assert.Equal(t, 2, created)
var stored database.Event
require.NoError(t, s.WebhookDB.First(&stored, "id = ?", event.ID).Error)
assert.Equal(t, event.Body, stored.Body)
assert.Equal(t, int64(len(event.Body)), stored.BodyBytes)
rows := storedDeliveries(t, s.WebhookDB, event.ID)
queued := queuedTasks(s.Engine)
require.Len(t, rows, 2)
require.Len(t, queued, 2)
deliveries := map[string]database.Delivery{}
for _, dlv := range rows {
deliveries[dlv.TargetID] = dlv
}
tasks := map[string]delivery.Task{}
for _, task := range queued {
tasks[task.TargetID] = task
}
for i := range targets {
dlv, ok := deliveries[targets[i].ID]
require.True(t, ok, "no delivery to %s", targets[i].Name)
assert.Equal(t, database.DeliveryStatusPending, dlv.Status)
assert.False(t, dlv.Replay)
assert.Equal(t, int64(1),
targetTotals(t, s.WebhookDB, targets[i].ID).Deliveries)
assertTaskFor(t, tasks[targets[i].ID], dlv.ID, event, &targets[i])
}
totals := storedEventTotals(t, s.WebhookDB)
assert.Equal(t, int64(1), totals.Events)
require.NotNil(t, totals.LastEventAt)
assert.True(t, totals.LastEventAt.Equal(stored.CreatedAt))
var entrypoint database.EntrypointTotals
require.NoError(t, s.WebhookDB.First(
&entrypoint, "entrypoint_id = ?", event.EntrypointID,
).Error)
assert.True(t, entrypoint.LastEventAt.Equal(stored.CreatedAt))
}
// TestStoreEvent_ResubmittedCopyLeavesTheEntrypoint proves a
// resubmitted copy is counted among the webhook's events but leaves its
// entrypoint's last event as it is: it did not arrive on that URL.
func TestStoreEvent_ResubmittedCopyLeavesTheEntrypoint(t *testing.T) {
t.Parallel()
s := newISetup(t)
storeTestTarget(t, s, "only")
sourceID := uuid.New().String()
event := newStoreTestEvent(s, `{"copy":true}`)
event.ResubmittedFromID = &sourceID
created, err := s.Engine.StoreEvent(event)
require.NoError(t, err)
assert.Equal(t, 1, created)
assert.Len(t, queuedTasks(s.Engine), 1)
assert.Equal(t, int64(1), storedEventTotals(t, s.WebhookDB).Events)
assert.Zero(t, countRows(t, s.WebhookDB, &database.EntrypointTotals{}))
}
// TestStoreEvent_NoActiveTarget proves an event of a webhook with no
// active target is stored with no delivery and nothing queued.
func TestStoreEvent_NoActiveTarget(t *testing.T) {
t.Parallel()
s := newISetup(t)
event := newStoreTestEvent(s, `{"to":"nobody"}`)
created, err := s.Engine.StoreEvent(event)
require.NoError(t, err)
assert.Zero(t, created)
assert.Equal(t, int64(1), countRows(t, s.WebhookDB, &database.Event{}))
assert.Empty(t, storedDeliveries(t, s.WebhookDB, event.ID))
assert.Empty(t, queuedTasks(s.Engine))
}
// TestStoreEvent_LargeBodyIsNotCarried proves a body at the inline
// limit is stored whole but left off the task, for the attempt to read
// from the event row.
func TestStoreEvent_LargeBodyIsNotCarried(t *testing.T) {
t.Parallel()
s := newISetup(t)
storeTestTarget(t, s, "only")
event := newStoreTestEvent(
s, strings.Repeat("x", delivery.MaxInlineBodySize),
)
_, err := s.Engine.StoreEvent(event)
require.NoError(t, err)
tasks := queuedTasks(s.Engine)
require.Len(t, tasks, 1)
assert.Nil(t, tasks[0].Body)
var stored database.Event
require.NoError(t, s.WebhookDB.First(&stored, "id = ?", event.ID).Error)
assert.Equal(t, event.Body, stored.Body)
}
// TestStoreEvent_RollsBackWhenAWriteFails proves the event, its
// deliveries and its totals are one transaction: when the entrypoint
// totals write fails, the last one, after the event, its delivery, the
// target totals and the event totals were written, none of them is left
// and nothing is queued.
func TestStoreEvent_RollsBackWhenAWriteFails(t *testing.T) {
t.Parallel()
s := newISetup(t)
storeTestTarget(t, s, "only")
require.NoError(t,
s.WebhookDB.Migrator().DropTable(&database.EntrypointTotals{}))
_, err := s.Engine.StoreEvent(newStoreTestEvent(s, `{"lost":true}`))
require.Error(t, err)
assert.Zero(t, countRows(t, s.WebhookDB, &database.Event{}))
assert.Zero(t, countRows(t, s.WebhookDB, &database.Delivery{}))
assert.Zero(t, countRows(t, s.WebhookDB, &database.TargetTotals{}))
totals := storedEventTotals(t, s.WebhookDB)
assert.Zero(t, totals.Events)
assert.Nil(t, totals.LastEventAt)
assert.Empty(t, queuedTasks(s.Engine))
assert.Zero(t, s.Engine.ExportInflightHeld())
}
// TestCreateReplay_CreatesAndQueuesAReplay proves CreateReplay adds a
// new pending delivery marked as a replay, counts it in the target's
// totals, queues its task built from the target it was given and the
// stored event, and leaves the delivery it repeats as it was. The
// stored event has every event field set, so a task that drops one of
// them differs from the event.
func TestCreateReplay_CreatesAndQueuesAReplay(t *testing.T) {
t.Parallel()
s := newISetup(t)
target := storeTestTarget(t, s, "replayed")
event := newStoreTestEvent(s, `{"replay":"me"}`)
require.NoError(t, s.WebhookDB.Create(event).Error)
original := iSeedDelivery(
t, s.WebhookDB, event.ID, target.ID,
database.DeliveryStatusFailed,
)
deliveryID, err := s.Engine.CreateReplay(s.WebhookID, event.ID, &target)
require.NoError(t, err)
replay := storedDelivery(t, s.WebhookDB, deliveryID)
assert.Equal(t, event.ID, replay.EventID)
assert.Equal(t, target.ID, replay.TargetID)
assert.Equal(t, database.DeliveryStatusPending, replay.Status)
assert.True(t, replay.Replay)
assert.Equal(t, database.DeliveryStatusFailed,
storedDelivery(t, s.WebhookDB, original.ID).Status)
assert.Equal(t, int64(1),
targetTotals(t, s.WebhookDB, target.ID).Deliveries)
tasks := queuedTasks(s.Engine)
require.Len(t, tasks, 1)
assertTaskFor(t, tasks[0], deliveryID, event, &target)
}
// TestCreateReplay_RollsBackWhenAWriteFails proves the replay delivery
// and its target totals are one transaction: when the totals write
// fails, after the delivery was written, no new delivery is left and
// nothing is queued.
func TestCreateReplay_RollsBackWhenAWriteFails(t *testing.T) {
t.Parallel()
s := newISetup(t)
target := storeTestTarget(t, s, "replayed")
event := iSeedEvent(t, s.WebhookDB, s.WebhookID, `{"replay":"me"}`)
iSeedDelivery(
t, s.WebhookDB, event.ID, target.ID,
database.DeliveryStatusFailed,
)
require.NoError(t,
s.WebhookDB.Migrator().DropTable(&database.TargetTotals{}))
_, err := s.Engine.CreateReplay(s.WebhookID, event.ID, &target)
require.Error(t, err)
assert.Len(t, storedDeliveries(t, s.WebhookDB, event.ID), 1)
assert.Empty(t, queuedTasks(s.Engine))
assert.Zero(t, s.Engine.ExportInflightHeld())
}
// storedDelivery reads one delivery back.
func storedDelivery(
t *testing.T, db *gorm.DB, deliveryID string,
) database.Delivery {
t.Helper()
var dlv database.Delivery
require.NoError(t, db.First(&dlv, "id = ?", deliveryID).Error)
return dlv
}
internal/delivery/target.go
+13 -23
View File
@@ -13,8 +13,8 @@ import (
// The engine provides one to each target so a target can own
// its retries durably: it records the attempt, marks the
// delivery retrying, and asks the Scheduler to deliver the
// next attempt after delay — exactly what the engine does for
// its own restart recovery.
// next attempt after delay — exactly what the engine's
// recovery routine does for a retrying delivery.
type Scheduler interface {
ScheduleRetry(task Task, delay time.Duration)
}
@@ -32,7 +32,8 @@ type Scheduler interface {
// Engine.observeAttempt, alongside the DeliveryResult it records for
// it. Deliver is also entered for attempts that never happen — an
// open circuit breaker refuses one — so the count cannot be taken
// from around this call.
// from around this call. An attempt cut off by a stop is neither
// counted nor recorded.
type Target interface {
Deliver(
ctx context.Context,
@@ -44,29 +45,20 @@ type Target interface {
}
// rescheduler is implemented by targets that own durable
// retries. The engine's restart recovery and periodic sweep
// use it to let the target recompute the schedule for an
// orphaned retrying delivery, keeping the retry schedule
// target-owned. Fire-and-forget targets do not implement it
// and their (never-occurring) retrying deliveries are
// skipped.
// retries. The engine's recovery routine uses it to let the
// target recompute the schedule for an orphaned retrying
// delivery, keeping the retry schedule target-owned.
// Fire-and-forget targets do not implement it, and a retrying
// delivery found for one is terminally failed.
type rescheduler interface {
// remainingBackoff returns how long to wait before the
// next attempt of a recovered retrying delivery.
// next attempt of a recovered retrying delivery: zero once
// its backoff has passed.
remainingBackoff(
webhookDB *gorm.DB,
deliveryID string,
attemptNum int,
) time.Duration
// backoffElapsed reports whether the backoff window for
// the last attempt has already passed, so the periodic
// sweep can re-enqueue the delivery now.
backoffElapsed(
webhookDB *gorm.DB,
deliveryID string,
attemptNum int,
) bool
}
// attemptResult is the outcome of a single delivery attempt,
@@ -102,15 +94,13 @@ func (e *Engine) initTargets(client *http.Client) {
client: client,
}
dbT := &databaseTarget{eng: e}
e.httpTarget = httpT
e.dbTarget = dbT
e.slackTarget = slackT
e.targets = map[database.TargetType]Target{
database.TargetTypeHTTP: httpT,
database.TargetTypeSlack: slackT,
database.TargetTypeDatabase: dbT,
database.TargetTypeDatabase: &databaseTarget{eng: e},
database.TargetTypeLog: &logTarget{eng: e},
}
}
internal/delivery/target_config_edit.go
-119
View File
@@ -1,119 +0,0 @@
package delivery
import (
"encoding/json"
"errors"
"fmt"
"sneak.berlin/go/webhooker/internal/database"
)
// errUnknownTargetTypeForEdit is returned when a stored target has a
// type the edit form has no field set for.
var errUnknownTargetTypeForEdit = errors.New(
"unknown target type",
)
// TargetConfigForm is the UNMASKED projection of a target's stored
// configuration, for pre-filling the target edit form.
//
// It is the deliberate exception to the rule TargetView enforces
// everywhere else: TargetView exists so that no template can render
// a target's stored blob, because a destination URL's path segments
// and a header value are both routinely the credential. An operator
// cannot correct a value they cannot see, so the edit form — and
// only the edit form — is shown the full value.
//
// Everything that keeps that exception narrow lives at the call
// site: the route is behind RequireAuth and the webhook's ownership
// check, and its group sets NoCache so the rendered secret is not
// written to a shared cache. Do not reach for this type from any
// other page.
type TargetConfigForm struct {
// URL is the destination for an HTTP target and the webhook
// URL for a Slack target.
URL string
// Headers is the HTTP target's configured headers in the
// textarea representation, one "Name: value" per line.
Headers string
// Timeout is the HTTP target's per-request timeout in seconds,
// empty when unset.
Timeout string
// Expiry is the database (archive) target's row expiry.
Expiry string
}
// NewTargetConfigForm parses a target's stored configuration into
// the edit form's fields.
//
// A configuration that does not parse is an error rather than a
// zero-valued form that silently looks like a target with no
// settings. The caller shows the operator that the stored value
// could not be read, so that saving the form is understood as
// replacing it rather than preserving it.
func NewTargetConfigForm(
t *database.Target,
) (TargetConfigForm, error) {
switch t.Type {
case database.TargetTypeHTTP:
cfg, err := parseHTTPConfig(t.Config)
if err != nil {
return TargetConfigForm{}, err
}
return TargetConfigForm{
URL: cfg.URL,
Headers: FormatTargetHeaders(cfg.Headers),
Timeout: FormatTargetTimeout(cfg.Timeout),
}, nil
case database.TargetTypeSlack:
cfg, err := parseSlackConfig(t.Config)
if err != nil {
return TargetConfigForm{}, err
}
return TargetConfigForm{URL: cfg.WebhookURL}, nil
case database.TargetTypeDatabase:
return databaseConfigForm(t.Config)
case database.TargetTypeLog:
// The log target takes no configuration.
return TargetConfigForm{}, nil
default:
return TargetConfigForm{}, fmt.Errorf(
"%w: %q", errUnknownTargetTypeForEdit, t.Type,
)
}
}
// databaseConfigForm parses an archive target's optional expiry.
// An absent or empty configuration is the keep-forever default and
// yields an empty field, so re-saving the form unchanged stores the
// same empty configuration it started with. An expiry that is set
// but not a valid duration is an error, not a blank field.
func databaseConfigForm(
configJSON string,
) (TargetConfigForm, error) {
if configJSON == "" {
return TargetConfigForm{}, nil
}
var cfg databaseTargetConfig
err := json.Unmarshal([]byte(configJSON), &cfg)
if err != nil {
return TargetConfigForm{}, fmt.Errorf(
"parsing config JSON: %w", err,
)
}
if cfg.Expiry == "" || cfg.Expiry == archiveExpiryNever {
return TargetConfigForm{}, nil
}
err = ValidateArchiveExpiry(cfg.Expiry)
if err != nil {
return TargetConfigForm{}, err
}
return TargetConfigForm{Expiry: cfg.Expiry}, nil
}
internal/delivery/target_config_view.go
+90 -100
View File
@@ -1,23 +1,25 @@
package delivery
import (
"encoding/json"
"fmt"
"strconv"
"time"
"sneak.berlin/go/webhooker/internal/archive"
"sneak.berlin/go/webhooker/internal/database"
)
// configUnavailable is what a target's configuration renders
// as when it is absent, of an unknown type, or does not
// parse. The stored blob is never shown as a fallback: it can
// hold a credential (a Slack incoming webhook URL is a bearer
// token) and a UI that prints it leaks that credential into
// browser history, screenshots and screen shares.
// configUnavailable is what a target's settings render as when
// the target is of an unknown type, and what its URL renders as
// when it does not parse into a scheme and host. The stored URL is
// never shown as a fallback: it can be a credential (a Slack
// incoming webhook URL is a bearer token) and a UI that prints it
// leaks that credential into browser history, screenshots and
// screen shares.
const configUnavailable = "(unavailable)"
// ConfigField is one labelled, display-safe value derived
// from a target's stored configuration.
// from a target's settings.
type ConfigField struct {
Label string
Value string
@@ -31,9 +33,9 @@ type ConfigField struct {
const deletedNameSuffix = " (deleted)"
// TargetView is the display-safe projection of a target for
// the UI. It deliberately has no raw configuration field, so
// no template — present or future — can render the stored
// blob.
// the UI. It deliberately has no URL or header field, so no
// template — present or future — can render the stored
// credential.
type TargetView struct {
ID string
Name string
@@ -60,8 +62,7 @@ func (v TargetView) DisplayName() string {
}
// NewTargetViews projects targets for rendering, replacing
// each stored configuration blob with named, display-safe
// fields.
// each target's settings with named, display-safe fields.
//
// A soft-deleted row projects exactly as a live one does, minus
// the deleted marker on its name: masking is a property of the
@@ -90,47 +91,37 @@ func NewTargetViews(
}
// targetConfigFields returns the display-safe fields for a
// target's configuration. Anything it cannot parse becomes
// the neutral placeholder.
// target's settings. A target of an unknown type gets the
// neutral placeholder.
func targetConfigFields(
t *database.Target,
) []ConfigField {
switch t.Type {
case database.TargetTypeSlack:
return slackConfigFields(t.Config)
return slackConfigFields(t)
case database.TargetTypeHTTP:
return httpConfigFields(t)
case database.TargetTypeDatabase:
return databaseConfigFields(t.Config)
return databaseConfigFields(t)
case database.TargetTypeLog:
// The log target takes no configuration.
// The log target takes no settings.
return nil
default:
return unavailableConfigFields()
return []ConfigField{{
Label: "Configuration",
Value: configUnavailable,
}}
}
}
// unavailableConfigFields is the neutral placeholder shown
// for a configuration that could not be presented.
func unavailableConfigFields() []ConfigField {
return []ConfigField{{
Label: "Configuration",
Value: configUnavailable,
}}
}
// slackConfigFields describes a Slack target. Only the masked
// webhook URL is shown; the full URL is the credential.
func slackConfigFields(configJSON string) []ConfigField {
cfg, err := parseSlackConfig(configJSON)
if err != nil {
return unavailableConfigFields()
}
// slackConfigFields describes a Slack target: its masked
// webhook URL and its number of delivery attempts. Only the
// masked URL is shown; the full URL is the credential.
func slackConfigFields(t *database.Target) []ConfigField {
return []ConfigField{{
Label: "Webhook URL",
Value: cfg.MaskedWebhookURL(),
}}
Value: MaskURL(t.URL),
}, maxAttemptsField(t)}
}
// httpConfigFields describes an HTTP target: its destination
@@ -144,96 +135,95 @@ func slackConfigFields(configJSON string) []ConfigField {
// whose path segments are the credential, and the field takes
// an arbitrary URL, so no segment can be assumed non-secret.
func httpConfigFields(t *database.Target) []ConfigField {
cfg, err := parseHTTPConfig(t.Config)
if err != nil {
return unavailableConfigFields()
}
fields := []ConfigField{{
Label: "Destination URL",
Value: MaskURL(cfg.URL),
Value: MaskURL(t.URL),
}}
if cfg.Timeout > 0 {
if t.TimeoutSeconds > 0 {
fields = append(fields, ConfigField{
Label: "Timeout",
Value: strconv.Itoa(cfg.Timeout) + "s",
Value: strconv.Itoa(t.TimeoutSeconds) + "s",
})
}
if len(cfg.Headers) > 0 {
if len(t.Headers) > 0 {
fields = append(fields, ConfigField{
Label: "Headers",
Value: fmt.Sprintf(
"%d configured", len(cfg.Headers),
"%d configured", len(t.Headers),
),
})
}
return append(fields, retryFields(t)...)
}
// retryFields describes a target's retry settings, which live
// on the target row rather than in its configuration blob.
func retryFields(t *database.Target) []ConfigField {
retries := strconv.Itoa(t.MaxRetries)
if t.MaxRetries == 0 {
retries += " (fire-and-forget)"
}
fields := []ConfigField{{
Label: "Max Retries",
Value: retries,
}}
if t.MaxQueueSize > 0 {
if t.ForwardQuery {
fields = append(fields, ConfigField{
Label: "Max Queue Size",
Value: strconv.Itoa(t.MaxQueueSize),
Label: "Query string",
Value: "passed on to this target",
})
}
fields = append(fields, maxAttemptsField(t))
return fields
}
// databaseConfigFields describes an archive target. Its
// configuration is optional, and an absent or empty expiry
// means the archive is kept forever. An expiry that is set
// but not a valid duration is reported as unavailable rather
// than echoed back.
func databaseConfigFields(configJSON string) []ConfigField {
expiry := archiveExpiryNever
// maxAttemptsField describes a target's number of delivery
// attempts.
func maxAttemptsField(t *database.Target) ConfigField {
return ConfigField{
Label: "Delivery attempts",
Value: strconv.Itoa(t.MaxAttempts),
}
}
if configJSON != "" {
var cfg databaseTargetConfig
err := json.Unmarshal([]byte(configJSON), &cfg)
if err != nil {
return unavailableConfigFields()
}
if cfg.Expiry != "" {
if ValidateArchiveExpiry(cfg.Expiry) != nil {
return unavailableConfigFields()
}
expiry = cfg.Expiry
}
// databaseConfigFields describes an archive target by its
// expiry in plain units, such as "30 days", or "never" when
// the archive is kept forever, and by its rotation.
func databaseConfigFields(t *database.Target) []ConfigField {
value := archive.ExpiryNever
if t.ArchiveExpiry > 0 {
value = plainDuration(t.ArchiveExpiry)
}
return []ConfigField{{
Label: "Archive Expiry",
Value: expiry,
Label: "Archive expiry",
Value: value,
}, {
Label: "Archive rotation",
Value: t.ArchiveRotation,
}}
}
// MaskedWebhookURL returns the Slack webhook URL reduced to
// its scheme and host, with the path, query and any userinfo
// elided. The path segments are the credential, so none of
// them is shown: the field accepts an arbitrary URL, so no
// segment can be assumed non-secret. A URL that does not
// parse into a scheme and host yields the neutral
// placeholder, never the raw string.
func (c *SlackTargetConfig) MaskedWebhookURL() string {
return MaskURL(c.WebhookURL)
// plainDuration writes a positive duration as a count of the
// largest whole unit it divides into: "30 days", "12 hours",
// "1 minute". A duration with a fraction of a second is
// written as Go writes it.
func plainDuration(d time.Duration) string {
const day = 24 * time.Hour
units := []struct {
size time.Duration
name string
}{
{day, "day"},
{time.Hour, "hour"},
{time.Minute, "minute"},
{time.Second, "second"},
}
for _, unit := range units {
if d%unit.size != 0 {
continue
}
count := int64(d / unit.size)
if count == 1 {
return "1 " + unit.name
}
return fmt.Sprintf("%d %ss", count, unit.name)
}
return d.String()
}
internal/delivery/target_config_view_test.go
+125 -100
View File
@@ -32,9 +32,20 @@ const (
viewMaskedOrigin = viewExampleOrigin + "/..."
viewUnavailable = "(unavailable)"
viewExpiryNever = "never"
viewMaxAttempts = "Delivery attempts"
)
func TestMaskedWebhookURL(t *testing.T) {
// The archive rotations a database target's settings can name.
const (
rotationNone = "none"
rotationMonthly = "monthly"
rotationDaily = "daily"
rotationHourly = "hourly"
)
// TestNewTargetViews_SlackMasksWebhookURL shows what a Slack target's
// URL renders as on the target list.
func TestNewTargetViews_SlackMasksWebhookURL(t *testing.T) {
t.Parallel()
tests := map[string]struct {
@@ -78,28 +89,24 @@ func TestMaskedWebhookURL(t *testing.T) {
t.Run(name, func(t *testing.T) {
t.Parallel()
cfg := &delivery.SlackTargetConfig{
WebhookURL: tc.url,
}
target := slackTarget()
target.URL = tc.url
assert.Equal(
t, tc.want, cfg.MaskedWebhookURL(),
t, tc.want,
fieldMap(viewFor(t, target).Config)["Webhook URL"],
)
})
}
}
// TestMaskedWebhookURL_NeverLeaksPath is the direct
// TestNewTargetViews_SlackNeverLeaksPath is the direct
// expression of the rule: whatever the input, the masked
// value never contains a path segment of it.
func TestMaskedWebhookURL_NeverLeaksPath(t *testing.T) {
func TestNewTargetViews_SlackNeverLeaksPath(t *testing.T) {
t.Parallel()
cfg := &delivery.SlackTargetConfig{
WebhookURL: slackWebhookURL,
}
masked := cfg.MaskedWebhookURL()
masked := fieldMap(viewFor(t, slackTarget()).Config)["Webhook URL"]
assert.NotContains(t, masked, "T00000000")
assert.NotContains(t, masked, "B00000000")
@@ -157,9 +164,7 @@ func TestNewTargetViews_DeletedTarget(t *testing.T) {
t, slackTargetName+" (deleted)", view.DisplayName(),
)
assert.Equal(
t,
map[string]string{"Webhook URL": slackMaskedURL},
fieldMap(view.Config),
t, viewFor(t, slackTarget()).Config, view.Config,
)
}
@@ -167,11 +172,11 @@ func TestNewTargetViews_DeletedTarget(t *testing.T) {
// share.
func slackTarget() database.Target {
return database.Target{
Name: slackTargetName,
Type: database.TargetTypeSlack,
Active: true,
Config: `{"webhookUrl":"` +
slackWebhookURL + `"}`,
Name: slackTargetName,
Type: database.TargetTypeSlack,
Active: true,
URL: slackWebhookURL,
MaxAttempts: 1,
}
}
@@ -189,7 +194,31 @@ func TestNewTargetViews_Slack(t *testing.T) {
assert.Equal(
t,
map[string]string{"Webhook URL": slackMaskedURL},
map[string]string{
"Webhook URL": slackMaskedURL,
viewMaxAttempts: "1",
},
fieldMap(view.Config),
)
}
// TestNewTargetViews_SlackAttempts proves a Slack target shows
// its number of delivery attempts the same way an HTTP target
// does.
func TestNewTargetViews_SlackAttempts(t *testing.T) {
t.Parallel()
target := slackTarget()
target.MaxAttempts = 2
view := viewFor(t, target)
assert.Equal(
t,
map[string]string{
"Webhook URL": slackMaskedURL,
viewMaxAttempts: "2",
},
fieldMap(view.Config),
)
}
@@ -198,12 +227,12 @@ func TestNewTargetViews_HTTP(t *testing.T) {
t.Parallel()
view := viewFor(t, database.Target{
Type: database.TargetTypeHTTP,
Config: `{"url":"` + viewExampleHook + `",` +
`"timeout":30,` +
`"headers":{"Authorization":"Bearer sekrit"}}`,
MaxRetries: 5,
MaxQueueSize: 100,
Type: database.TargetTypeHTTP,
URL: viewExampleHook,
TimeoutSeconds: 30,
Headers: map[string]string{headerAuthorization: "Bearer sekrit"},
ForwardQuery: true,
MaxAttempts: 5,
})
fields := fieldMap(view.Config)
@@ -214,8 +243,8 @@ func TestNewTargetViews_HTTP(t *testing.T) {
"Destination URL": viewMaskedOrigin,
"Timeout": "30s",
"Headers": "1 configured",
"Max Retries": "5",
"Max Queue Size": "100",
"Query string": "passed on to this target",
viewMaxAttempts: "5",
},
fields,
)
@@ -226,19 +255,22 @@ func TestNewTargetViews_HTTP(t *testing.T) {
}
}
func TestNewTargetViews_HTTPFireAndForget(t *testing.T) {
// TestNewTargetViews_HTTPSingleAttempt proves a single attempt is
// shown as the plain number, like any other.
func TestNewTargetViews_HTTPSingleAttempt(t *testing.T) {
t.Parallel()
view := viewFor(t, database.Target{
Type: database.TargetTypeHTTP,
Config: `{"url":"` + viewExampleHook + `"}`,
Type: database.TargetTypeHTTP,
URL: viewExampleHook,
MaxAttempts: 1,
})
assert.Equal(
t,
map[string]string{
"Destination URL": viewMaskedOrigin,
"Max Retries": "0 (fire-and-forget)",
viewMaxAttempts: "1",
},
fieldMap(view.Config),
)
@@ -252,8 +284,9 @@ func TestNewTargetViews_HTTPMasksDestinationURL(t *testing.T) {
t.Parallel()
view := viewFor(t, database.Target{
Type: database.TargetTypeHTTP,
Config: `{"url":"` + slackWebhookURL + `"}`,
Type: database.TargetTypeHTTP,
URL: slackWebhookURL,
MaxAttempts: 1,
})
fields := fieldMap(view.Config)
@@ -276,19 +309,20 @@ func TestNewTargetViews_Database(t *testing.T) {
t.Parallel()
tests := map[string]struct {
config string
expiry time.Duration
want string
}{
"empty config": {config: "", want: viewExpiryNever},
"empty expiry": {config: `{}`, want: viewExpiryNever},
"explicit": {
config: `{"expiry":"720h"}`,
want: "720h",
},
"never literal": {
config: `{"expiry":"` + viewExpiryNever + `"}`,
want: viewExpiryNever,
},
"never": {expiry: 0, want: viewExpiryNever},
"1h": {expiry: time.Hour, want: "1 hour"},
"12h": {expiry: 12 * time.Hour, want: "12 hours"},
"24h": {expiry: 24 * time.Hour, want: "1 day"},
"720h": {expiry: 720 * time.Hour, want: "30 days"},
"2160h": {expiry: 2160 * time.Hour, want: "90 days"},
"8760h": {expiry: 8760 * time.Hour, want: "365 days"},
"36h": {expiry: 36 * time.Hour, want: "36 hours"},
"1h30m": {expiry: 90 * time.Minute, want: "90 minutes"},
"45s": {expiry: 45 * time.Second, want: "45 seconds"},
"1.5s": {expiry: 1500 * time.Millisecond, want: "1.5s"},
}
for name, tc := range tests {
@@ -296,81 +330,72 @@ func TestNewTargetViews_Database(t *testing.T) {
t.Parallel()
view := viewFor(t, database.Target{
Type: database.TargetTypeDatabase,
Config: tc.config,
Type: database.TargetTypeDatabase,
ArchiveExpiry: tc.expiry,
ArchiveRotation: rotationNone,
})
assert.Equal(
t,
map[string]string{"Archive Expiry": tc.want},
map[string]string{
"Archive expiry": tc.want,
"Archive rotation": rotationNone,
},
fieldMap(view.Config),
)
})
}
}
// TestNewTargetViews_DatabaseRotation proves the target list shows a
// database target's rotation.
func TestNewTargetViews_DatabaseRotation(t *testing.T) {
t.Parallel()
for _, rotation := range []string{
rotationNone, rotationMonthly, rotationDaily, rotationHourly,
} {
t.Run(rotation, func(t *testing.T) {
t.Parallel()
view := viewFor(t, database.Target{
Type: database.TargetTypeDatabase,
ArchiveRotation: rotation,
})
assert.Equal(
t, rotation, fieldMap(view.Config)["Archive rotation"],
)
})
}
}
func TestNewTargetViews_Log(t *testing.T) {
t.Parallel()
view := viewFor(t, database.Target{
Type: database.TargetTypeLog,
Config: "",
Type: database.TargetTypeLog,
})
assert.Empty(t, view.Config)
}
// TestNewTargetViews_Unpresentable proves that no config the
// view cannot present falls back to the stored blob.
func TestNewTargetViews_Unpresentable(t *testing.T) {
// TestNewTargetViews_UnknownType proves that a target of a type the
// view does not know shows the neutral placeholder, and never its
// URL.
func TestNewTargetViews_UnknownType(t *testing.T) {
t.Parallel()
const blob = `{"webhookUrl":"https://hooks.slack.com` +
slackSecretPath + `"`
view := viewFor(t, database.Target{
Type: database.TargetType("carrier-pigeon"),
URL: slackWebhookURL,
})
tests := map[string]database.Target{
"unknown target type": {
Type: database.TargetType("carrier-pigeon"),
Config: blob,
assert.Equal(
t,
map[string]string{
"Configuration": viewUnavailable,
},
"unparseable json": {
Type: database.TargetTypeSlack,
Config: blob,
},
"empty slack config": {
Type: database.TargetTypeSlack,
},
"slack config without url": {
Type: database.TargetTypeSlack,
Config: `{}`,
},
"unparseable http json": {
Type: database.TargetTypeHTTP,
Config: `{"url":`,
},
"unparseable archive json": {
Type: database.TargetTypeDatabase,
Config: `{"expiry":`,
},
"invalid archive expiry": {
Type: database.TargetTypeDatabase,
Config: `{"expiry":"a fortnight"}`,
},
}
for name, target := range tests {
t.Run(name, func(t *testing.T) {
t.Parallel()
view := viewFor(t, target)
assert.Equal(
t,
map[string]string{
"Configuration": viewUnavailable,
},
fieldMap(view.Config),
)
})
}
fieldMap(view.Config),
)
}
internal/delivery/target_database.go
+10 -256
View File
@@ -2,29 +2,23 @@ package delivery
import (
"context"
"fmt"
"path/filepath"
"sync"
"time"
"gorm.io/gorm"
"sneak.berlin/go/webhooker/internal/database"
)
// databaseTarget is a no-retry target that archives the
// full inbound event into a per-webhook archive SQLite file,
// separate from the per-webhook event database. The event is
// already persisted in the per-webhook event DB by the time
// delivery runs; the database target additionally writes a
// durable long-term copy into archive-{webhookID}.db and then
// records a single attempt whose outcome reflects whether the
// archive write succeeded. See archiveWriter for the
// close/reopen, auto-recreate, and expiry semantics.
// databaseTarget is a no-retry target that archives the full
// inbound event into the target's own archive SQLite file, separate
// from the per-webhook event database. The event is already
// persisted in the per-webhook event DB by the time delivery runs;
// the database target additionally writes a durable long-term copy
// through the engine's archive.Writers, and then records a single
// attempt whose outcome reflects whether the archive write
// succeeded. Package archive holds everything about the files
// themselves.
type databaseTarget struct {
eng *Engine
mu sync.Mutex
writers map[string]*archiveWriter
}
// Deliver implements Target. It archives the event, then
@@ -44,7 +38,7 @@ func (t *databaseTarget) Deliver(
) {
start := time.Now()
err := t.archive(d)
err := t.eng.archives.Write(d)
elapsed := time.Since(start)
@@ -91,243 +85,3 @@ func (t *databaseTarget) Deliver(
database.DeliveryStatusDelivered,
)
}
// archive writes the full event as a row into the webhook's
// archive database, honouring the optional per-target expiry
// parsed from the target config JSON.
func (t *databaseTarget) archive(d *database.Delivery) error {
webhookID := d.Event.WebhookID
if webhookID == "" {
return errArchiveMissingWebhookID
}
expiry, err := parseArchiveExpiry(d.Target.Config)
if err != nil {
return err
}
w, err := t.writerFor(webhookID)
if err != nil {
return err
}
row := archivedEvent{
EventID: d.Event.ID,
WebhookID: webhookID,
EntrypointID: d.Event.EntrypointID,
Method: d.Event.Method,
Headers: d.Event.Headers,
Body: d.Event.Body,
ContentType: d.Event.ContentType,
}
return w.write(row, expiry)
}
// writerFor returns the archiveWriter for a webhook, creating
// and caching it on first use. Each webhook has one writer so
// its close/reopen debounce state is shared across concurrent
// deliveries. The archive file lives beside the per-webhook
// event database in the data directory.
func (t *databaseTarget) writerFor(
webhookID string,
) (*archiveWriter, error) {
path, err := t.archivePath(webhookID)
if err != nil {
return nil, err
}
t.mu.Lock()
defer t.mu.Unlock()
if t.writers == nil {
t.writers = make(map[string]*archiveWriter)
}
w, ok := t.writers[webhookID]
if !ok {
w = newArchiveWriter(path, t.eng.log)
t.writers[webhookID] = w
}
// A delivery claims the entry: even if the idle sweep created
// it moments ago, it now belongs to the registry proper and
// the sweep must leave it in place when it finishes.
w.sweepOwned = false
return w, nil
}
// sweepWriterFor returns the archive writer the idle sweep should
// prune a webhook through, together with whether the sweep itself
// created the registry entry.
//
// The sweep must route its prune through the registered writer so
// the writer's mutex orders it against concurrent writes, but it
// must never leave a registry entry behind: a sweep that ran
// concurrently with the webhook's deletion would otherwise
// re-create an entry that nothing will ever evict again, which is
// exactly the leak eviction exists to prevent. An entry the sweep
// creates is therefore marked sweep-owned and handed back to
// releaseSweepWriter when the sweep is done.
func (t *databaseTarget) sweepWriterFor(
webhookID string,
) (*archiveWriter, bool, error) {
path, err := t.archivePath(webhookID)
if err != nil {
return nil, false, err
}
t.mu.Lock()
defer t.mu.Unlock()
if t.writers == nil {
t.writers = make(map[string]*archiveWriter)
}
w, ok := t.writers[webhookID]
if ok {
return w, false, nil
}
w = newArchiveWriter(path, t.eng.log)
w.sweepOwned = true
t.writers[webhookID] = w
return w, true, nil
}
// releaseSweepWriter drops a registry entry that the idle sweep
// created, so a sweep leaves the registry exactly as it found it.
//
// The entry is removed only if it is still the very writer the
// sweep installed and no delivery has claimed it in the meantime
// (writerFor clears sweepOwned when it hands a writer to the
// write path). Both conditions are evaluated under the registry
// lock, so an eviction that raced the sweep — which removes the
// entry outright — simply finds nothing left to do here, and a
// delivery that adopted the writer keeps a registered, evictable
// one.
func (t *databaseTarget) releaseSweepWriter(
webhookID string, w *archiveWriter,
) {
t.mu.Lock()
defer t.mu.Unlock()
cur, ok := t.writers[webhookID]
if !ok || cur != w || !cur.sweepOwned {
return
}
delete(t.writers, webhookID)
}
// archivePath returns the archive file path for a webhook: it
// lives beside the per-webhook event database in the data
// directory. It does not touch the filesystem.
func (t *databaseTarget) archivePath(
webhookID string,
) (string, error) {
if t.eng.dbManager == nil {
return "", errArchiveNoDataDir
}
dir := filepath.Dir(t.eng.dbManager.DBPath(webhookID))
return filepath.Join(
dir, fmt.Sprintf("archive-%s.db", webhookID),
), nil
}
// evict drops a webhook's archive writer from the registry and
// closes its handle, so a deleted webhook does not leave a
// writer (and an open archive handle within its debounce
// window) alive for the process lifetime.
//
// The map entry is removed under the registry lock, which is
// then released before the handle is closed under the writer's
// own lock: that ordering keeps the registry available to other
// webhooks while an in-flight write on this one drains, and
// closing under the writer's lock means eviction can never race
// a write.
//
// Eviction is idempotent and silent for a webhook with no
// writer, which is the common case: a webhook with no database
// target never creates one. It never deletes the archive file.
func (t *databaseTarget) evict(webhookID string) {
t.mu.Lock()
w, ok := t.writers[webhookID]
if ok {
delete(t.writers, webhookID)
}
t.mu.Unlock()
if !ok {
return
}
w.evict()
t.eng.log.Info(
"evicted archive writer",
"webhook_id", webhookID,
"path", w.path,
)
}
// evictAll evicts every cached archive writer, exactly as evict
// does for one webhook. The engine calls it at shutdown, once its
// workers have returned. Closing the last handle on an archive
// moves the contents of its -wal into the .db and removes the
// -wal, so a clean stop leaves each archive as a single file.
func (t *databaseTarget) evictAll() {
t.mu.Lock()
writers := t.writers
t.writers = nil
t.mu.Unlock()
for _, w := range writers {
w.evict()
}
}
// sweepWebhook prunes one webhook's archive of rows older than
// expiry, without requiring a write. It returns nil (nothing to
// do) when the archive file does not exist, so a sweep never
// creates an archive for a webhook that has a database target
// but has never received an event.
//
// It also never leaves a registry entry behind: an entry it had
// to create to reach the writer's mutex is released again once
// the prune is done, so a sweep racing a webhook deletion cannot
// resurrect the writer the eviction just dropped.
func (t *databaseTarget) sweepWebhook(
webhookID string, expiry time.Duration,
) error {
path, err := t.archivePath(webhookID)
if err != nil {
return err
}
// Check before taking a writer at all: a webhook whose
// archive has never been created gets no writer, no handle,
// and no file.
if !fileExists(path) {
return nil
}
w, created, err := t.sweepWriterFor(webhookID)
if err != nil {
return err
}
if created {
defer t.releaseSweepWriter(webhookID, w)
}
return w.sweepExpired(expiry)
}
internal/delivery/target_database_archive.go
-438
View File
@@ -1,438 +0,0 @@
package delivery
import (
"encoding/json"
"errors"
"fmt"
"log/slog"
"os"
"sync"
"time"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/gormlog"
)
// archiveExpiryNever is the expiry sentinel (and default) that
// disables pruning so archived rows are kept forever.
const archiveExpiryNever = "never"
// archiveReopenDebounce bounds how often an archive file is
// closed and reopened. After each write the handle is closed
// and reopened so an operator can move the file away for
// offline archiving, but never more than once per this window.
const archiveReopenDebounce = time.Second
const (
// archiveModeCreate is the SQLite URI mode used by the write
// path: open the archive file, creating it if missing, so a
// first write (or a write after the operator moved the file
// away) recreates it.
archiveModeCreate = database.SQLiteModeCreate
// archiveModeExisting is the SQLite URI mode used by the idle
// sweep: open read-write but never create. A sweep must never
// conjure an empty archive file for a webhook that has a
// database target but has never received an event.
archiveModeExisting = database.SQLiteModeExisting
)
var (
// errArchiveMissingWebhookID is returned when an event to
// archive has no webhook id to key its archive file on.
errArchiveMissingWebhookID = errors.New(
"cannot archive event without a webhook id",
)
// errArchiveNoDataDir is returned when the database target
// has no webhook database manager and so cannot locate the
// data directory for archive files.
errArchiveNoDataDir = errors.New(
"database target has no data directory",
)
// errArchiveExpiryNotPositive is returned when a
// user-supplied archive expiry parses as a duration but is
// zero or negative; "never" is the way to disable pruning.
errArchiveExpiryNotPositive = errors.New(
"expiry must be a positive duration or \"never\"",
)
// errArchiveWriterEvicted is returned when a writer that has
// been evicted (its webhook was deleted, or its last database
// target was removed) is used again. An evicted writer is no
// longer in the registry, so reopening its file would leak a
// handle nothing owns.
errArchiveWriterEvicted = errors.New(
"archive writer has been evicted",
)
)
// databaseTargetConfig is the optional per-target JSON config
// for a database (archive) target.
type databaseTargetConfig struct {
// Expiry is a Go duration (e.g. "720h") after which
// archived rows are pruned, or "never" (the default) to
// keep them forever.
Expiry string `json:"expiry"`
}
// archivedEvent is one fully captured webhook event stored in a
// per-webhook archive database for long-term retention. It is a
// self-contained copy — independent of the per-webhook event
// database, which may prune events under its own retention.
type archivedEvent struct {
ID uint `gorm:"primaryKey;autoIncrement"`
EventID string `gorm:"index"`
WebhookID string
EntrypointID string
Method string
Headers string
Body string
ContentType string
// ArchivedAt is when the row was archived and is the age
// basis for expiry pruning.
ArchivedAt time.Time `gorm:"index"`
}
// parseArchiveExpiry reads the optional expiry from a database
// target's config JSON. An empty config, an empty expiry, or
// the literal "never" all mean keep forever, returned as a zero
// duration. Any other value must parse as a positive Go
// duration; a set-but-invalid value (unparseable, zero, or
// negative) is an error rather than a silent default, matching
// ValidateArchiveExpiry at target creation.
func parseArchiveExpiry(
configJSON string,
) (time.Duration, error) {
if configJSON == "" {
return 0, nil
}
var cfg databaseTargetConfig
err := json.Unmarshal([]byte(configJSON), &cfg)
if err != nil {
return 0, fmt.Errorf(
"parsing database target config: %w", err,
)
}
if cfg.Expiry == "" || cfg.Expiry == archiveExpiryNever {
return 0, nil
}
dur, err := time.ParseDuration(cfg.Expiry)
if err != nil {
return 0, fmt.Errorf(
"parsing archive expiry %q: %w", cfg.Expiry, err,
)
}
if dur <= 0 {
return 0, fmt.Errorf(
"%w: %q", errArchiveExpiryNotPositive, cfg.Expiry,
)
}
return dur, nil
}
// ValidateArchiveExpiry checks a user-supplied archive expiry
// for a database target at configuration time. Valid values are
// empty, "never" (both meaning keep forever), or a positive Go
// duration such as "720h". Anything else is an error, so a bad
// expiry is rejected when the target is created rather than
// failing every subsequent delivery.
func ValidateArchiveExpiry(expiry string) error {
if expiry == "" || expiry == archiveExpiryNever {
return nil
}
dur, err := time.ParseDuration(expiry)
if err != nil {
return fmt.Errorf(
"expiry must be %q or a Go duration "+
"such as \"720h\": %w",
archiveExpiryNever, err,
)
}
if dur <= 0 {
return fmt.Errorf(
"%w: %q", errArchiveExpiryNotPositive, expiry,
)
}
return nil
}
// archiveWriter owns one per-webhook archive SQLite file. It
// serialises writes, and after each write closes and reopens
// the file (debounced to at most once per debounce window) so
// an operator can move the file away for offline archiving. The
// next write recreates a moved or removed file, because the
// file is opened create-if-missing and its schema is migrated
// on every open.
type archiveWriter struct {
mu sync.Mutex
path string
log *slog.Logger
debounce time.Duration
db *gorm.DB
lastReopen time.Time
reopens int
// evicted marks a writer that has been removed from the
// per-webhook registry. Its handle is closed and it must
// never open the file again: nothing holds it any more, so a
// reopen would leak the handle for the process lifetime.
evicted bool
// sweepOwned marks a registry entry that the idle sweep
// created because no writer was cached for the webhook. The
// sweep removes such an entry again when it is done, so a
// sweep can never leave — or resurrect — a registry entry
// for a webhook that has been deleted. A delivery that adopts
// the writer clears the flag, handing the entry to the
// registry proper.
//
// Unlike every other field here it is guarded by
// databaseTarget.mu, not by this writer's mu: it describes the
// registry entry rather than the file.
sweepOwned bool
}
// newArchiveWriter builds an archiveWriter for a file path with
// the default reopen debounce.
func newArchiveWriter(
path string, log *slog.Logger,
) *archiveWriter {
return &archiveWriter{
path: path,
log: log,
debounce: archiveReopenDebounce,
}
}
// write appends the event as a row, then applies the debounced
// close/reopen. It recreates the archive file if it was moved
// or removed since the last open. A positive expiry prunes rows
// older than it on each (re)open.
func (w *archiveWriter) write(
row archivedEvent, expiry time.Duration,
) error {
w.mu.Lock()
defer w.mu.Unlock()
if w.evicted {
return fmt.Errorf(
"%w: %s", errArchiveWriterEvicted, w.path,
)
}
if w.db == nil || !fileExists(w.path) {
err := w.reopen(expiry)
if err != nil {
return err
}
}
row.ArchivedAt = time.Now()
err := w.db.Create(&row).Error
if err != nil {
return fmt.Errorf(
"archiving event to %s: %w", w.path, err,
)
}
if time.Since(w.lastReopen) >= w.debounce {
return w.reopen(expiry)
}
return nil
}
// open opens (creating if missing) the archive file, migrates
// its schema, records the reopen time, and prunes expired rows
// when expiry is positive.
func (w *archiveWriter) open(expiry time.Duration) error {
return w.openMode(archiveModeCreate, expiry)
}
// openMode opens the archive file with the given SQLite URI
// mode, migrates its schema, records the reopen time, and
// prunes expired rows when expiry is positive. The write path
// passes archiveModeCreate so a missing file is recreated; the
// idle sweep passes archiveModeExisting so a missing file is an
// error rather than a newly conjured empty archive.
func (w *archiveWriter) openMode(
mode string, expiry time.Duration,
) error {
// Opened through database.OpenSQLite so an archive file carries
// the same WAL journaling, busy timeout, immediate-transaction
// locking, and pool bounds as every other database file. See
// internal/database/sqlite_open.go.
sqlDB, err := database.OpenSQLite(w.path, mode)
if err != nil {
return fmt.Errorf(
"opening archive database %s: %w", w.path, err,
)
}
gdb, err := gorm.Open(
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{
// Never leave this at GORM's default. See
// internal/gormlog.
Logger: gormlog.New(w.log),
},
)
if err != nil {
_ = sqlDB.Close()
return fmt.Errorf(
"connecting to archive database %s: %w",
w.path, err,
)
}
err = gdb.AutoMigrate(&archivedEvent{})
if err != nil {
_ = sqlDB.Close()
return fmt.Errorf(
"migrating archive database %s: %w", w.path, err,
)
}
w.db = gdb
w.lastReopen = time.Now()
w.reopens++
if expiry > 0 {
w.prune(expiry)
}
return nil
}
// reopen closes any open handle and opens the file afresh. The
// fresh open recreates the file if it was moved away.
func (w *archiveWriter) reopen(expiry time.Duration) error {
w.close()
return w.open(expiry)
}
// close closes the underlying handle, if any.
func (w *archiveWriter) close() {
if w.db == nil {
return
}
sqlDB, err := w.db.DB()
if err == nil {
_ = sqlDB.Close()
}
w.db = nil
}
// sweepExpired prunes an archive that may have gone idle, with
// no write to trigger the usual on-reopen prune. It takes the
// writer's own mutex for the whole operation, so a sweep is
// ordered against concurrent writes rather than reaching around
// them to the file.
//
// It never creates the archive file: a missing file is skipped,
// and the reopen uses archiveModeExisting so SQLite itself
// refuses to create one if the file disappears between the
// check and the open.
//
// The archive is left CLOSED afterwards. An idle archive holding
// no handle is what keeps the operator's move-the-file-away
// workflow working; the next write reopens (and recreates) the
// file as it always has.
func (w *archiveWriter) sweepExpired(expiry time.Duration) error {
w.mu.Lock()
defer w.mu.Unlock()
if w.evicted {
return fmt.Errorf(
"%w: %s", errArchiveWriterEvicted, w.path,
)
}
if !fileExists(w.path) {
return nil
}
// Drop any live handle first so the prune runs against a
// freshly opened file, matching the write path's semantics.
w.close()
err := w.openMode(archiveModeExisting, expiry)
if err != nil {
return err
}
w.close()
return nil
}
// evict closes the writer's handle and marks it unusable. It is
// called when the writer leaves the registry, either because the
// webhook was deleted or because its last database target was
// removed. The archive FILE is deliberately left on disk: it is
// long-term storage an operator may still want.
func (w *archiveWriter) evict() {
w.mu.Lock()
defer w.mu.Unlock()
w.evicted = true
w.close()
}
// prune deletes archived rows older than expiry, measured from
// each row's archived time. It runs on every (re)open, so a
// steadily written archive is swept by its own write traffic. An
// archive that goes idle receives no further reopens, which is
// why ArchiveSweeper exists to drive sweepExpired on a timer.
// Failures are logged, not fatal: a prune error must not stop
// archiving.
func (w *archiveWriter) prune(expiry time.Duration) {
cutoff := time.Now().Add(-expiry)
res := w.db.Where("archived_at < ?", cutoff).
Delete(&archivedEvent{})
if res.Error != nil {
w.log.Error(
"failed to prune expired archive rows",
"path", w.path,
"error", res.Error,
)
return
}
if res.RowsAffected > 0 {
w.log.Info(
"pruned expired archive rows",
"path", w.path,
"rows_deleted", res.RowsAffected,
)
}
}
// fileExists reports whether a path currently exists.
func fileExists(path string) bool {
_, err := os.Stat(path)
return err == nil
}
internal/delivery/target_database_archive_gormlog_test.go
-155
View File
@@ -1,155 +0,0 @@
package delivery_test
import (
"bytes"
"log"
"log/slog"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
gormlogger "gorm.io/gorm/logger"
"sneak.berlin/go/webhooker/internal/delivery"
"sneak.berlin/go/webhooker/internal/middleware"
)
// archiveGORMTailMarker sits at the far end of the value this file
// drives into an archive lookup. Its presence in a log line means the
// whole value reached the log, so nothing truncated it.
const archiveGORMTailMarker = "ENDOFCLIENTVALUE"
// archiveGORMFillBytes is how much text the lookup carries. It is far
// past every budget in play.
const archiveGORMFillBytes = 8 << 10
// gormDefaultBuf collects what GORM's package-level default logger
// writes, if anything reaches it.
type gormDefaultBuf struct {
mu sync.Mutex
b bytes.Buffer
}
func (g *gormDefaultBuf) Write(p []byte) (int, error) {
g.mu.Lock()
defer g.mu.Unlock()
return g.b.Write(p)
}
func (g *gormDefaultBuf) String() string {
g.mu.Lock()
defer g.mu.Unlock()
return g.b.String()
}
// captureArchiveGORMDefault replaces GORM's package-level default
// logger with one configured exactly as GORM configures its own,
// writing to a buffer.
//
// This duplicates the detector in internal/handlers rather than
// sharing it: a test helper cannot cross a package's test boundary
// without exporting production code to carry it, and a logging
// detector is not worth a production symbol. What it detects is the
// third gorm.Open in this service, at
// internal/delivery/target_database_archive.go — the archive writer,
// whose type is unexported, so nothing outside this package can drive
// it.
func captureArchiveGORMDefault(t *testing.T) *gormDefaultBuf {
t.Helper()
buf := &gormDefaultBuf{}
orig := gormlogger.Default
gormlogger.Default = gormlogger.New(
log.New(buf, "", log.LstdFlags),
gormlogger.Config{
SlowThreshold: 200 * time.Millisecond,
LogLevel: gormlogger.Warn,
IgnoreRecordNotFoundError: false,
Colorful: false,
},
)
t.Cleanup(func() { gormlogger.Default = orig })
return buf
}
// TestArchiveWriter_NeverUsesGORMsDefaultLogger pins the archive
// writer's gorm.Open to the adapter.
//
// Restore a bare &gorm.Config{} at
// internal/delivery/target_database_archive.go and this fails: the
// default logger prints the fully interpolated SELECT on every
// ErrRecordNotFound, so the client-chosen event id below arrives whole
// and unbounded on stdout, answering to no level the operator set.
//
// Not parallel: gormlogger.Default is process-global. Go runs every
// non-parallel top-level test to completion before it resumes the
// parallel ones.
//
//nolint:paralleltest // Deliberately sequential; see above.
func TestArchiveWriter_NeverUsesGORMsDefaultLogger(t *testing.T) {
var captured bytes.Buffer
gormDefault := captureArchiveGORMDefault(t)
w := delivery.NewExportArchiveWriter(
filepath.Join(t.TempDir(), "archive.db"),
slog.New(slog.NewTextHandler(
&captured, &slog.HandlerOptions{Level: slog.LevelDebug},
)),
0,
)
require.NoError(t, w.Open(0))
t.Cleanup(w.Evict)
// A lookup that misses, carrying a value the size of an inbound
// event id. Under the default logger this is the line that gets
// interpolated and printed.
value := strings.Repeat("\x01", archiveGORMFillBytes) +
archiveGORMTailMarker
var row delivery.ExportArchivedEvent
err := w.DB().Where("event_id = ?", value).First(&row).Error
require.ErrorIs(t, err, gorm.ErrRecordNotFound)
got := gormDefault.String()
assert.Empty(
t, got,
"GORM's default logger wrote %d bytes, so the archive "+
"writer's gorm.Open is back on a bare &gorm.Config{}; "+
"the first of them: %s",
len(got), got[:min(len(got), 300)],
)
// The adapter drops a miss, so this should be silent too — and
// whatever it does write stays inside the stated ceiling.
out := captured.String()
assert.NotContains(
t, out, archiveGORMTailMarker,
"the far end of the client-chosen value reached the log",
)
for line := range strings.SplitSeq(strings.TrimRight(out, "\n"), "\n") {
if line == "" {
continue
}
assert.LessOrEqual(
t, len(line), middleware.MaxAccessLogLineBytes,
"log line exceeded its bound: %s",
line[:min(len(line), 300)],
)
}
}
internal/delivery/target_database_evict_test.go
-407
View File
@@ -1,407 +0,0 @@
package delivery_test
import (
"context"
"errors"
"fmt"
"net/http"
"os"
"path/filepath"
"sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/delivery"
)
// evictTestEngine builds an engine backed by a temporary data
// directory and returns it along with that directory.
func evictTestEngine(t *testing.T) (*delivery.Engine, string) {
t.Helper()
dataDir := t.TempDir()
eng := delivery.NewTestEngineWithDB(
nil,
database.NewTestWebhookDBManager(dataDir),
archiveTestLogger(),
&http.Client{Timeout: 5 * time.Second},
1,
)
return eng, dataDir
}
// TestEvictWebhook_ClosesAndRemovesWriter proves that evicting
// a webhook drops its archive writer from the registry and
// closes the open archive handle, rather than leaving both
// alive for the process lifetime.
func TestEvictWebhook_ClosesAndRemovesWriter(t *testing.T) {
t.Parallel()
eng, dataDir := evictTestEngine(t)
webhookDB := testWebhookDB(t)
event := seedEvent(t, webhookDB, `{"archived":true}`)
d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
eng.ExportDeliverDatabase(webhookDB, d)
webhookID := event.WebhookID
require.True(
t, eng.ExportHasArchiveWriter(webhookID),
"a delivery should have cached an archive writer",
)
require.True(
t, eng.ExportArchiveHandleOpen(webhookID),
"the writer should hold an open handle after a write",
)
eng.EvictWebhook(webhookID)
assert.False(
t, eng.ExportHasArchiveWriter(webhookID),
"eviction should remove the registry entry",
)
assert.False(
t, eng.ExportArchiveHandleOpen(webhookID),
"eviction should close the archive handle",
)
archivePath := filepath.Join(
dataDir, fmt.Sprintf("archive-%s.db", webhookID),
)
assert.FileExists(
t, archivePath,
"eviction must not delete the archive file",
)
}
// TestEvictWebhook_UnknownWebhookIsNoOp proves eviction is safe
// for the common case of a webhook that never had a database
// target, and that repeating it does not panic.
func TestEvictWebhook_UnknownWebhookIsNoOp(t *testing.T) {
t.Parallel()
eng, _ := evictTestEngine(t)
assert.NotPanics(t, func() {
eng.EvictWebhook("no-such-webhook")
eng.EvictWebhook("no-such-webhook")
})
assert.False(
t, eng.ExportHasArchiveWriter("no-such-webhook"),
"eviction must not create a writer",
)
}
// evictTestRow builds an archive row for the eviction tests.
func evictTestRow(eventID string) delivery.ExportArchivedEvent {
return delivery.ExportArchivedEvent{
EventID: eventID,
WebhookID: "wh-evict",
Method: http.MethodPost,
Body: `{"seeded":true}`,
}
}
// TestEvictedWriter_WriteDoesNotReopenFile is the direct test of
// the evicted guard on the write path. A writer that has left
// the registry is held by nobody, so a handle it opened could
// never be closed again: it must refuse the write outright
// rather than recreate the archive behind the registry's back.
//
// The archive file is removed before the eviction, so an
// unguarded write is unmistakable — it recreates the file.
func TestEvictedWriter_WriteDoesNotReopenFile(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "archive-evicted.db")
w := delivery.NewExportArchiveWriter(
path, archiveTestLogger(), 0,
)
require.NoError(t, w.Write(evictTestRow("ev-1"), 0))
require.FileExists(t, path)
// The operator moves the archive away for offline retention,
// which the write path would ordinarily undo on the next
// write by recreating the file.
require.NoError(t, os.Remove(path))
w.Evict()
err := w.Write(evictTestRow("ev-2"), 0)
require.ErrorIs(
t, err, delivery.ErrExportArchiveWriterEvicted,
"an evicted writer must refuse writes",
)
assert.NoFileExists(
t, path,
"an evicted writer must not reopen (or recreate) the "+
"archive file",
)
assert.False(
t, w.HandleOpen(),
"an evicted writer must hold no handle",
)
}
// TestEvictedWriter_SweepDoesNotReopenFile is the same test for
// the sweep path: an idle sweep that reaches a writer already
// evicted underneath it must return the sentinel rather than
// reopen a file nothing owns.
func TestEvictedWriter_SweepDoesNotReopenFile(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "archive-evicted.db")
w := delivery.NewExportArchiveWriter(
path, archiveTestLogger(), 0,
)
require.NoError(t, w.Write(evictTestRow("ev-1"), 0))
require.FileExists(t, path)
w.Evict()
err := w.SweepExpired(time.Hour)
require.ErrorIs(
t, err, delivery.ErrExportArchiveWriterEvicted,
"an evicted writer must refuse an idle sweep",
)
assert.False(
t, w.HandleOpen(),
"a refused sweep must not leave a handle open",
)
}
// racingWrites drives a pack of goroutines writing to one
// archive writer until each is refused, so an eviction on the
// test goroutine has to take the writer's mutex away from writes
// that are already contending for it.
type racingWrites struct {
wg sync.WaitGroup
mu sync.Mutex
sawEvicted bool
otherErr error
started chan struct{}
}
// racingWriteGoroutines is how many goroutines contend for the
// writer's mutex while the eviction lands.
const racingWriteGoroutines = 4
// startRacingWrites launches the writing goroutines. Each writes
// in a loop and stops at its first error, recording whether that
// error was the eviction sentinel. The deadline is a backstop
// against a hang, not a timing assumption: the first write after
// the eviction is refused.
func startRacingWrites(
w *delivery.ExportArchiveWriter,
) *racingWrites {
r := &racingWrites{
started: make(chan struct{}, racingWriteGoroutines),
}
deadline := time.Now().Add(10 * time.Second)
r.wg.Add(racingWriteGoroutines)
for i := range racingWriteGoroutines {
go func() {
defer r.wg.Done()
first := true
for time.Now().Before(deadline) {
err := w.Write(
evictTestRow(fmt.Sprintf("ev-%d", i)), 0,
)
if first {
r.started <- struct{}{}
first = false
}
if err == nil {
continue
}
r.record(err)
return
}
}()
}
return r
}
// record classifies the error that stopped one goroutine.
func (r *racingWrites) record(err error) {
r.mu.Lock()
defer r.mu.Unlock()
if errors.Is(err, delivery.ErrExportArchiveWriterEvicted) {
r.sawEvicted = true
return
}
r.otherErr = err
}
// awaitFirstWrite blocks until at least one write has run, so
// the eviction that follows is a genuine race.
func (r *racingWrites) awaitFirstWrite() {
<-r.started
}
// wait joins the goroutines and reports whether any write was
// refused with the eviction sentinel, plus any unexpected error.
func (r *racingWrites) wait() (bool, error) {
r.wg.Wait()
r.mu.Lock()
defer r.mu.Unlock()
return r.sawEvicted, r.otherErr
}
// TestEvictWebhook_RacingWriteDoesNotReopenHandle exercises the
// interleaving the evicted flag exists for: writes already
// contending for the writer's mutex when the eviction takes it.
// The write that wins the mutex after the eviction must abandon
// its work rather than reopen the archive, leaving the writer
// permanently handle-free. Run under -race.
func TestEvictWebhook_RacingWriteDoesNotReopenHandle(
t *testing.T,
) {
t.Parallel()
eng, _ := evictTestEngine(t)
webhookDB := testWebhookDB(t)
event := seedEvent(t, webhookDB, `{"archived":true}`)
d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
// Prime the registry so the test can hold the very writer the
// eviction is about to detach.
eng.ExportDeliverDatabase(webhookDB, d)
w := eng.ExportArchiveWriterFor(event.WebhookID)
require.NotNil(t, w)
require.True(t, w.HandleOpen())
race := startRacingWrites(w)
// Evict only once writes are genuinely in flight, so the
// eviction has to contend for the writer's mutex.
race.awaitFirstWrite()
eng.EvictWebhook(event.WebhookID)
sawEvicted, otherErr := race.wait()
require.NoError(t, otherErr)
assert.True(
t, sawEvicted,
"a write after eviction must be refused",
)
assert.False(
t, w.HandleOpen(),
"no write may reopen the archive once the writer has "+
"been evicted",
)
assert.False(
t, eng.ExportHasArchiveWriter(event.WebhookID),
"the registry entry must stay gone",
)
}
// TestEvictWebhook_LaterDeliveryRecreatesWriter proves eviction
// does not break archiving for a webhook that is still alive: a
// subsequent delivery gets a brand new writer from the registry.
// It says nothing about the evicted writer itself — that is what
// TestEvictedWriter_WriteDoesNotReopenFile covers.
func TestEvictWebhook_LaterDeliveryRecreatesWriter(t *testing.T) {
t.Parallel()
eng, _ := evictTestEngine(t)
webhookDB := testWebhookDB(t)
event := seedEvent(t, webhookDB, `{"archived":true}`)
d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
eng.ExportDeliverDatabase(webhookDB, d)
require.True(
t, eng.ExportHasArchiveWriter(event.WebhookID),
)
eng.EvictWebhook(event.WebhookID)
// A fresh delivery for the same webhook gets a brand new
// writer from the registry, so archiving keeps working.
second := seedDatabaseTargetDelivery(
t, webhookDB, event, "",
)
eng.ExportDeliverDatabase(webhookDB, second)
assert.True(
t, eng.ExportHasArchiveWriter(event.WebhookID),
"a later delivery should recreate the writer",
)
}
// TestEngineStop_WriteAfterStopIsRefused proves the engine's stop
// closes each archive writer the way deleting its webhook does: a
// write that reaches a writer after the stop is refused, reopens
// nothing and adds no row.
func TestEngineStop_WriteAfterStopIsRefused(t *testing.T) {
t.Parallel()
eng, _ := evictTestEngine(t)
webhookDB := testWebhookDB(t)
event := seedEvent(t, webhookDB, `{"archived":true}`)
d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
eng.ExportDeliverDatabase(webhookDB, d)
w := eng.ExportArchiveWriterFor(event.WebhookID)
require.NotNil(t, w)
require.True(t, w.HandleOpen())
require.NoError(t, eng.ExportStop(context.Background()))
err := w.Write(evictTestRow("ev-after-stop"), 0)
require.ErrorIs(
t, err, delivery.ErrExportArchiveWriterEvicted,
"a write after the stop must be refused",
)
assert.False(
t, w.HandleOpen(),
"a refused write must not reopen the archive",
)
assert.False(
t, eng.ExportHasArchiveWriter(event.WebhookID),
"the stop should empty the registry",
)
count, err := countArchivedRows(w.Path())
require.NoError(t, err)
assert.Equal(
t, int64(1), count, "the refused row must not be written",
)
}
internal/delivery/target_database_test.go
+262 -269
View File
@@ -1,6 +1,7 @@
package delivery_test
import (
"context"
"database/sql"
"fmt"
"log/slog"
@@ -15,11 +16,54 @@ import (
"github.com/stretchr/testify/require"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/clause"
_ "modernc.org/sqlite" // Pure Go SQLite driver.
"sneak.berlin/go/webhooker/internal/archive"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/database/databasetest"
"sneak.berlin/go/webhooker/internal/db/migrations"
"sneak.berlin/go/webhooker/internal/delivery"
"sneak.berlin/go/webhooker/internal/gormlog"
)
// archiveTestWebhookName is the name of every webhook
// seedDatabaseTarget creates. It is not safe in a file name as it
// stands, so every archive file name made from it is made safe first.
const archiveTestWebhookName = "Sweep Test!"
// archiveEnv bundles the pieces a database target test drives: a main
// configuration database holding webhooks and targets, a delivery
// engine whose archive writers write into the data directory, and that
// directory.
type archiveEnv struct {
eng *delivery.Engine
mainDB *database.Database
dataDir string
}
func setupArchiveTest(t *testing.T) *archiveEnv {
t.Helper()
dataDir := t.TempDir()
mainDB, err := database.Open(dataDir, slog.New(slog.DiscardHandler))
require.NoError(t, err)
t.Cleanup(func() { _ = mainDB.Close() })
return &archiveEnv{
eng: delivery.NewTestEngineWithDB(
mainDB,
databasetest.NewWebhookDBManager(t, dataDir),
archiveTestLogger(),
&http.Client{Timeout: 5 * time.Second},
1,
),
mainDB: mainDB,
dataDir: dataDir,
}
}
func archiveTestLogger() *slog.Logger {
return slog.New(slog.NewTextHandler(
os.Stderr,
@@ -27,8 +71,87 @@ func archiveTestLogger() *slog.Logger {
))
}
// archivePath returns where the engine keeps a database target's
// archive file, for the names seedDatabaseTarget gave it.
func (env *archiveEnv) archivePath(tgt *database.Target) string {
return filepath.Join(
env.dataDir,
archive.FileName(archiveTestWebhookName, tgt.Name, tgt.ID),
)
}
// seedDatabaseTarget creates a webhook with one database target,
// which keeps its archive forever in one file, and returns the
// target.
func (env *archiveEnv) seedDatabaseTarget(
t *testing.T,
) *database.Target {
t.Helper()
wh := &database.Webhook{
UserID: uuid.New().String(),
Name: archiveTestWebhookName,
}
require.NoError(
t,
env.mainDB.DB().
Omit(clause.Associations).
Create(wh).Error,
)
tgt := &database.Target{
WebhookID: wh.ID,
Name: "Archive",
Type: database.TargetTypeDatabase,
Active: true,
ArchiveRotation: "none",
MaxAttempts: 1,
}
require.NoError(
t,
env.mainDB.DB().
Omit(clause.Associations).
Create(tgt).Error,
)
return tgt
}
// seedDatabaseTargetDelivery seeds a pending delivery of an event
// to a database target and returns the in-memory delivery the
// target handler is invoked with.
func seedDatabaseTargetDelivery(
t *testing.T,
webhookDB *gorm.DB,
event database.Event,
tgt *database.Target,
) *database.Delivery {
t.Helper()
dlv := seedDelivery(
t, webhookDB, event.ID, tgt.ID,
database.DeliveryStatusPending,
)
d := &database.Delivery{
EventID: event.ID,
TargetID: tgt.ID,
Status: database.DeliveryStatusPending,
Event: event,
Target: database.Target{
Name: tgt.Name,
Type: database.TargetTypeDatabase,
ArchiveExpiry: tgt.ArchiveExpiry,
ArchiveRotation: tgt.ArchiveRotation,
},
}
d.ID = dlv.ID
return d
}
// openArchiveDBForRead opens an archive file read-only so a
// test can inspect the rows the writer persisted.
// test can inspect the rows the database target archived.
func openArchiveDBForRead(
t *testing.T, path string,
) *gorm.DB {
@@ -43,56 +166,70 @@ func openArchiveDBForRead(
t.Cleanup(func() { _ = sqlDB.Close() })
gdb, err := gorm.Open(
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
sqlite.Dialector{Conn: sqlDB},
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
)
require.NoError(t, err)
return gdb
}
// archiveFileSuffixes returns the archive file itself and the
// SQLite sidecars that accompany an open database. A test that
// asserts no archive was created has to check all of them.
func archiveFileSuffixes() []string {
return []string{"", "-wal", "-shm"}
// countArchivedRows counts the rows in an archive file without
// asserting anything, so it is safe to poll from an
// assert.Eventually condition (which runs off the test
// goroutine, where testify assertions must not be used).
func countArchivedRows(path string) (int64, error) {
sqlDB, err := sql.Open(
"sqlite", fmt.Sprintf("file:%s?mode=ro", path),
)
if err != nil {
return 0, err
}
defer func() { _ = sqlDB.Close() }()
gdb, err := gorm.Open(
sqlite.Dialector{Conn: sqlDB},
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
)
if err != nil {
return 0, err
}
var count int64
err = gdb.Table("archived_events").Count(&count).Error
if err != nil {
return 0, err
}
return count, nil
}
// removeArchiveFiles simulates an operator moving the archive
// away by deleting the SQLite file and its sidecar files.
func removeArchiveFiles(t *testing.T, path string) {
t.Helper()
for _, suffix := range []string{
"", "-wal", "-shm", "-journal",
} {
err := os.Remove(path + suffix)
if err != nil && !os.IsNotExist(err) {
t.Fatalf("removing %s%s: %v", path, suffix, err)
}
}
// archivedRow is the columns of an archive row these tests read.
type archivedRow struct {
EventID string
WebhookID string
Method string
RawQuery string
Body string
}
// TestDeliverDatabase_ArchivesEvent verifies that delivering to
// a database target marks the delivery delivered and archives
// the full event into a separate per-webhook archive file.
// the full event into the target's own archive file.
func TestDeliverDatabase_ArchivesEvent(t *testing.T) {
t.Parallel()
dataDir := t.TempDir()
dbMgr := database.NewTestWebhookDBManager(dataDir)
e := delivery.NewTestEngineWithDB(
nil, dbMgr,
archiveTestLogger(),
&http.Client{Timeout: 5 * time.Second},
1,
)
env := setupArchiveTest(t)
tgt := env.seedDatabaseTarget(t)
webhookDB := testWebhookDB(t)
event := seedEvent(t, webhookDB, `{"archived":true}`)
d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
event.RawQuery = eventQuery
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
e.ExportDeliverDatabase(webhookDB, d)
env.eng.ExportDeliverDatabase(webhookDB, d)
var updated database.Delivery
@@ -105,247 +242,42 @@ func TestDeliverDatabase_ArchivesEvent(t *testing.T) {
)
archivePath := filepath.Join(
dataDir,
fmt.Sprintf("archive-%s.db", event.WebhookID),
env.dataDir, "archive-sweep-test-archive-"+tgt.ID+".db",
)
assert.FileExists(t, archivePath)
rdb := openArchiveDBForRead(t, archivePath)
var rows []delivery.ExportArchivedEvent
var rows []archivedRow
require.NoError(t, rdb.Find(&rows).Error)
require.NoError(t, rdb.Table("archived_events").Find(&rows).Error)
require.Len(t, rows, 1)
assert.Equal(t, event.ID, rows[0].EventID)
assert.Equal(t, event.WebhookID, rows[0].WebhookID)
assert.Equal(t, event.Method, rows[0].Method)
assert.Equal(t, eventQuery, rows[0].RawQuery)
assert.JSONEq(t, `{"archived":true}`, rows[0].Body)
}
func TestArchiveWriter_WritesRow(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "archive-wh.db")
w := delivery.NewExportArchiveWriter(
path, archiveTestLogger(), 0,
)
row := delivery.ExportArchivedEvent{
EventID: "ev-1",
WebhookID: "wh-1",
EntrypointID: "ep-1",
Method: "POST",
Headers: `{"X":"Y"}`,
Body: `{"hello":"world"}`,
ContentType: "application/json",
}
require.NoError(t, w.Write(row, 0))
assert.FileExists(t, path)
var got []delivery.ExportArchivedEvent
require.NoError(t, w.DB().Find(&got).Error)
require.Len(t, got, 1)
assert.Equal(t, "ev-1", got[0].EventID)
assert.Equal(t, "wh-1", got[0].WebhookID)
assert.Equal(t, "ep-1", got[0].EntrypointID)
assert.Equal(t, row.Method, got[0].Method)
assert.Equal(t, row.ContentType, got[0].ContentType)
assert.JSONEq(t, `{"hello":"world"}`, got[0].Body)
assert.False(t, got[0].ArchivedAt.IsZero())
}
func TestArchiveWriter_RecreatesAfterRemoval(
t *testing.T,
) {
t.Parallel()
path := filepath.Join(t.TempDir(), "archive-wh.db")
w := delivery.NewExportArchiveWriter(
path, archiveTestLogger(), 0,
)
require.NoError(t, w.Write(
delivery.ExportArchivedEvent{EventID: "a"}, 0,
))
assert.FileExists(t, path)
// The operator moves the archive away while the handle is
// still open.
removeArchiveFiles(t, path)
require.NoFileExists(t, path)
// The next write recreates the file with a fresh schema and
// only the new row.
require.NoError(t, w.Write(
delivery.ExportArchivedEvent{EventID: "b"}, 0,
))
assert.FileExists(t, path)
var got []delivery.ExportArchivedEvent
require.NoError(t, w.DB().Find(&got).Error)
require.Len(t, got, 1)
assert.Equal(t, "b", got[0].EventID)
}
func TestArchiveWriter_ReopenDebounce(t *testing.T) {
t.Parallel()
// A generous debounce keeps the two rapid writes inside
// the window even on a heavily loaded test machine.
path := filepath.Join(t.TempDir(), "archive-wh.db")
w := delivery.NewExportArchiveWriter(
path, archiveTestLogger(), 2*time.Second,
)
require.NoError(t, w.Write(
delivery.ExportArchivedEvent{EventID: "a"}, 0,
))
require.NoError(t, w.Write(
delivery.ExportArchivedEvent{EventID: "b"}, 0,
))
// Two writes inside the debounce window trigger only the
// initial open — no extra close/reopen.
assert.Equal(t, 1, w.Reopens())
time.Sleep(2100 * time.Millisecond)
require.NoError(t, w.Write(
delivery.ExportArchivedEvent{EventID: "c"}, 0,
))
// A write after the window elapses closes and reopens once.
assert.Equal(t, 2, w.Reopens())
}
func TestArchiveWriter_ExpiryPrune(t *testing.T) {
t.Parallel()
path := filepath.Join(t.TempDir(), "archive-wh.db")
w := delivery.NewExportArchiveWriter(
path, archiveTestLogger(), 0,
)
require.NoError(t, w.Open(0))
old := delivery.ExportArchivedEvent{
EventID: "old",
ArchivedAt: time.Now().Add(-2 * time.Hour),
}
fresh := delivery.ExportArchivedEvent{
EventID: "fresh",
ArchivedAt: time.Now(),
}
require.NoError(t, w.DB().Create(&old).Error)
require.NoError(t, w.DB().Create(&fresh).Error)
// Reopening with a one-hour expiry prunes the old row.
require.NoError(t, w.Reopen(time.Hour))
var got []delivery.ExportArchivedEvent
require.NoError(t, w.DB().Find(&got).Error)
require.Len(t, got, 1)
assert.Equal(t, "fresh", got[0].EventID)
}
func TestParseArchiveExpiry(t *testing.T) {
t.Parallel()
cases := []struct {
name string
in string
want time.Duration
wantErr bool
}{
{"empty config", "", 0, false},
{"explicit never", `{"expiry":"never"}`, 0, false},
{"empty expiry", `{"expiry":""}`, 0, false},
{"duration", `{"expiry":"1h"}`, time.Hour, false},
{"unparseable", `{"expiry":"nonsense"}`, 0, true},
{"zero duration", `{"expiry":"0s"}`, 0, true},
{"negative duration", `{"expiry":"-5h"}`, 0, true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
got, err := delivery.ExportParseArchiveExpiry(tc.in)
if tc.wantErr {
require.Error(t, err)
return
}
require.NoError(t, err)
assert.Equal(t, tc.want, got)
})
}
}
// seedDatabaseTargetDelivery seeds a pending delivery for a
// database target with the given config JSON and returns the
// in-memory delivery the target handler is invoked with.
func seedDatabaseTargetDelivery(
t *testing.T,
webhookDB *gorm.DB,
event database.Event,
config string,
) *database.Delivery {
t.Helper()
dlv := seedDelivery(
t, webhookDB, event.ID, uuid.New().String(),
database.DeliveryStatusPending,
)
d := &database.Delivery{
EventID: event.ID,
TargetID: dlv.TargetID,
Status: database.DeliveryStatusPending,
Event: event,
Target: database.Target{
Name: "test-db",
Type: database.TargetTypeDatabase,
Config: config,
},
}
d.ID = dlv.ID
return d
}
// TestDeliverDatabase_ArchiveFailureFailsDelivery verifies that
// an archive error (here: an unparseable expiry in the target
// config) fails the delivery loudly: the attempt is recorded as
// failed with the error and the delivery is marked failed, not
// an archive error (here: an event with no webhook id to record)
// fails the delivery loudly: the attempt is recorded as failed
// with the error and the delivery is marked failed, not
// delivered.
func TestDeliverDatabase_ArchiveFailureFailsDelivery(
t *testing.T,
) {
t.Parallel()
dataDir := t.TempDir()
e := delivery.NewTestEngineWithDB(
nil, database.NewTestWebhookDBManager(dataDir),
archiveTestLogger(),
&http.Client{Timeout: 5 * time.Second},
1,
)
env := setupArchiveTest(t)
tgt := env.seedDatabaseTarget(t)
webhookDB := testWebhookDB(t)
event := seedEvent(t, webhookDB, `{"archived":false}`)
d := seedDatabaseTargetDelivery(
t, webhookDB, event, `{"expiry":"nonsense"}`,
)
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
d.Event.WebhookID = ""
e.ExportDeliverDatabase(webhookDB, d)
env.eng.ExportDeliverDatabase(webhookDB, d)
var updated database.Delivery
@@ -368,35 +300,96 @@ func TestDeliverDatabase_ArchiveFailureFailsDelivery(
"the attempt must be recorded as failed",
)
assert.Contains(t,
results[0].Error, "nonsense",
results[0].Error, "without a webhook id",
"the archive error must be recorded on the attempt",
)
assert.NoFileExists(t,
filepath.Join(
dataDir,
fmt.Sprintf("archive-%s.db", event.WebhookID),
),
"no archive file should exist for a failed config",
env.archivePath(tgt),
"no archive file should exist for a failed write",
)
}
func TestValidateArchiveExpiry(t *testing.T) {
// seedArchivedRow creates the archive file at path holding one row, for
// eventID, archived at archivedAt.
func seedArchivedRow(
t *testing.T, path, eventID string, archivedAt time.Time,
) {
t.Helper()
sqlDB, err := sql.Open(
"sqlite", fmt.Sprintf("file:%s?mode=rwc", path),
)
require.NoError(t, err)
defer func() { _ = sqlDB.Close() }()
gdb, err := gorm.Open(
sqlite.Dialector{Conn: sqlDB},
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
)
require.NoError(t, err)
require.NoError(t, migrations.Apply(gdb, migrations.Archive))
require.NoError(t, gdb.Exec(
"INSERT INTO archived_events (event_id, archived_at) VALUES (?, ?)",
eventID, archivedAt,
).Error)
}
// TestDeliverDatabase_TaskArchiveSettings proves a task's ArchiveExpiry
// and ArchiveRotation reach the archive through the engine. The
// target's row has neither, so only the task can bring them: with daily
// rotation the event goes to the file for its day, and opening that
// file with a one-hour expiry prunes the row it already held, archived
// two hours ago.
func TestDeliverDatabase_TaskArchiveSettings(t *testing.T) {
t.Parallel()
valid := []string{"", "never", "1h", "720h", "30m"}
for _, in := range valid {
require.NoError(t,
delivery.ValidateArchiveExpiry(in),
"expiry %q should be accepted", in,
)
}
s := newISetup(t)
invalid := []string{"nonsense", "7d", "-5h", "0s", "0"}
for _, in := range invalid {
require.Error(t,
delivery.ValidateArchiveExpiry(in),
"expiry %q should be rejected", in,
)
webhook := database.Webhook{UserID: uuid.NewString(), Name: "settings"}
webhook.ID = s.WebhookID
require.NoError(t, s.MainDB.Create(&webhook).Error)
target := database.Target{
WebhookID: s.WebhookID,
Name: "archive",
Type: database.TargetTypeDatabase,
Active: true,
MaxAttempts: 1,
}
require.NoError(t, s.MainDB.Create(&target).Error)
event := iSeedEvent(t, s.WebhookDB, s.WebhookID, `{"archived":true}`)
d := iSeedDelivery(
t, s.WebhookDB, event.ID, target.ID,
database.DeliveryStatusPending,
)
task := iTask(
d, event, s.WebhookID, target.ID, target.Name, "", 1, 1,
&event.Body,
)
task.TargetType = database.TargetTypeDatabase
task.ArchiveExpiry = time.Hour
task.ArchiveRotation = "daily"
daily := target
daily.ArchiveRotation = task.ArchiveRotation
dailyFile := archive.PathAt(s.DBMgr, &webhook, &daily, event.CreatedAt)
seedArchivedRow(t, dailyFile, "expired", time.Now().Add(-2*time.Hour))
s.Engine.ExportProcessNewTask(context.TODO(), &task)
iAssertStatus(t, s.WebhookDB, d.ID, database.DeliveryStatusDelivered)
var rows []archivedRow
require.NoError(t, openArchiveDBForRead(t, dailyFile).
Table("archived_events").Find(&rows).Error)
require.Len(t, rows, 1)
assert.Equal(t, event.ID, rows[0].EventID,
"the event goes to its day's file and the expired row is pruned",
)
}
internal/delivery/target_headers.go
+7 -9
View File
@@ -17,7 +17,7 @@ import (
const MaxTargetTimeoutSeconds = 300
// Errors returned when a target's header or timeout form input
// cannot be turned into a configuration.
// cannot be turned into the target's setting.
//
// None of these ever quotes a header VALUE. A target header value
// is routinely an authorization token, and these messages are shown
@@ -48,8 +48,8 @@ var (
)
// isReservedTargetHeader reports whether name (canonicalised) is a
// header a target configuration may not set, because the delivery
// path or net/http itself writes it regardless.
// header a target may not set, because the delivery path or
// net/http itself writes it regardless.
//
// These are rejected rather than accepted-and-ignored. Storing a
// header that provably never reaches the wire tells the operator
@@ -77,13 +77,11 @@ func isReservedTargetHeader(name string) bool {
// ParseTargetHeaders turns the target form's headers field — one
// "Name: value" pair per line, blank lines ignored — into the map
// stored in HTTPTargetConfig.Headers. Names are canonicalised, so a
// stored in database.Target.Headers. Names are canonicalised, so a
// name repeated in a different case is still a duplicate rather than
// one pair silently overwriting the other.
//
// An input with no pairs yields an empty map, which omitempty drops
// from the stored config: a target configured with no headers keeps
// the same config JSON it had before this field existed.
// An input with no pairs yields an empty map.
func ParseTargetHeaders(raw string) (map[string]string, error) {
headers := make(map[string]string)
@@ -223,8 +221,8 @@ func FormatTargetHeaders(headers map[string]string) string {
// ParseTargetTimeout interprets the target form's timeout field as
// a whole number of seconds. An empty field means "unset" and yields
// 0, which omitempty drops from the stored config and which the
// delivery path reads as "use the shared client's timeout".
// 0, which the delivery path reads as "use the shared client's
// timeout".
//
// Anything else that is not a whole number in range is an error, not
// a silently substituted default: a target whose timeout was typed
internal/delivery/target_headers_test.go
+4 -91
View File
@@ -1,23 +1,21 @@
package delivery_test
import (
"encoding/json"
"strconv"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/delivery"
)
// Literals these tests repeat, named so that the header names and the
// keep-forever archive config each have one definition.
// Literals these tests repeat, named so that each header name has one
// definition.
const (
headerAuthorization = "Authorization"
headerContentType = "Content-Type"
headerUserAgent = "User-Agent"
bearerValue = "Bearer abc"
archiveConfigNever = "{\"expiry\":\"never\"}"
)
func TestParseTargetHeaders_AcceptsPairs(t *testing.T) {
@@ -41,23 +39,13 @@ func TestParseTargetHeaders_AcceptsPairs(t *testing.T) {
)
}
// A configuration with no headers must stay indistinguishable from
// one written before the field existed, so omitempty drops the key.
// An input of blank lines is no headers, not an error.
func TestParseTargetHeaders_EmptyInputYieldsNoHeaders(t *testing.T) {
t.Parallel()
got, err := delivery.ParseTargetHeaders("\n \n")
require.NoError(t, err)
assert.Empty(t, got)
encoded, err := json.Marshal(delivery.HTTPTargetConfig{
URL: "https://example.com/h",
Headers: got,
})
require.NoError(t, err)
assert.JSONEq(
t, `{"url":"https://example.com/h"}`, string(encoded),
)
}
func TestParseTargetHeaders_Rejects(t *testing.T) {
@@ -181,81 +169,6 @@ func TestFormatTargetTimeout(t *testing.T) {
assert.Empty(t, delivery.FormatTargetTimeout(-1))
}
func TestNewTargetConfigForm(t *testing.T) {
t.Parallel()
form, err := delivery.NewTargetConfigForm(&database.Target{
Type: database.TargetTypeHTTP,
Config: `{"url":"https://example.com/h",` +
`"headers":{"Authorization":"Bearer abc"},` +
`"timeout":9}`,
})
require.NoError(t, err)
assert.Equal(t, "https://example.com/h", form.URL)
assert.Equal(t, "Authorization: Bearer abc\n", form.Headers)
assert.Equal(t, "9", form.Timeout)
form, err = delivery.NewTargetConfigForm(&database.Target{
Type: database.TargetTypeSlack,
Config: `{"webhookUrl":"https://hooks.example/s"}`,
})
require.NoError(t, err)
assert.Equal(t, "https://hooks.example/s", form.URL)
form, err = delivery.NewTargetConfigForm(&database.Target{
Type: database.TargetTypeDatabase,
Config: `{"expiry":"720h"}`,
})
require.NoError(t, err)
assert.Equal(t, "720h", form.Expiry)
form, err = delivery.NewTargetConfigForm(&database.Target{
Type: database.TargetTypeLog,
})
require.NoError(t, err)
assert.Empty(t, form.URL)
}
// A keep-forever archive target must pre-fill as an empty field, so
// saving the form back unchanged stores the same empty config.
func TestNewTargetConfigForm_DatabaseNeverIsBlank(t *testing.T) {
t.Parallel()
for _, cfg := range []string{"", `{}`, archiveConfigNever} {
form, err := delivery.NewTargetConfigForm(
&database.Target{
Type: database.TargetTypeDatabase,
Config: cfg,
},
)
require.NoError(t, err, cfg)
assert.Empty(t, form.Expiry, cfg)
}
}
// An unreadable stored config is an error rather than a blank form
// that looks like a target with no settings, so the caller can tell
// the operator that saving replaces the stored value.
func TestNewTargetConfigForm_UnreadableConfigErrors(t *testing.T) {
t.Parallel()
cases := []*database.Target{
{Type: database.TargetTypeHTTP, Config: "not json"},
{Type: database.TargetTypeHTTP, Config: `{}`},
{Type: database.TargetTypeSlack, Config: ""},
{
Type: database.TargetTypeDatabase,
Config: `{"expiry":"soon"}`,
},
{Type: database.TargetType("nope")},
}
for _, target := range cases {
_, err := delivery.NewTargetConfigForm(target)
require.Error(t, err, target.Type)
}
}
// The ceiling exists so one misconfigured target cannot hold a
// delivery worker indefinitely, and it is inclusive.
func TestParseTargetTimeout_CeilingIsInclusive(t *testing.T) {
internal/delivery/target_http.go
+109 -132
View File
@@ -4,10 +4,10 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"sort"
"sync"
"time"
@@ -16,24 +16,6 @@ import (
"sneak.berlin/go/webhooker/internal/database"
)
// Sentinel errors returned by the config parsers.
var (
errEmptyTargetConfig = errors.New(
"empty target config",
)
errMissingTargetURL = errors.New(
"target URL is required",
)
)
// HTTPTargetConfig holds configuration for http target
// types.
type HTTPTargetConfig struct {
URL string `json:"url"`
Headers map[string]string `json:"headers,omitempty"`
Timeout int `json:"timeout,omitempty"`
}
// httpCore holds the retry, backoff, and circuit-breaker
// machinery shared by the HTTP and Slack targets. Each of
// those targets owns its own httpCore instance (and thus its
@@ -47,34 +29,72 @@ type httpCore struct {
}
// deliver runs one delivery attempt through the retry core.
// A maxRetries of 0 is fire-and-forget: a single attempt is
// recorded and no circuit breaker is consulted. A positive
// maxRetries gates the attempt on the circuit breaker and
// schedules a backed-off retry on failure.
// A maxAttempts of 1 is a single attempt, recorded without
// consulting a circuit breaker. A higher maxAttempts gates the
// attempt on the circuit breaker and schedules a backed-off retry
// on failure, until maxAttempts attempts have been made.
//
// ctx is the context attempt sends its request with. Stopping the
// engine cancels it; see interruptedByStop.
func (c *httpCore) deliver(
ctx context.Context,
webhookDB *gorm.DB,
d *database.Delivery,
task *Task,
sched Scheduler,
maxRetries int,
maxAttempts int,
attempt func() attemptResult,
) {
if maxRetries == 0 {
c.fireAndForget(webhookDB, d, attempt())
if maxAttempts == 1 {
c.fireAndForget(ctx, webhookDB, d, attempt())
return
}
c.withRetry(
webhookDB, d, task, sched, maxRetries, attempt,
ctx, webhookDB, d, task, sched, maxAttempts, attempt,
)
}
// interruptedByStop reports whether an attempt failed because the
// engine is stopping, which cancels the context its request is made
// with. Such an attempt is not recorded: no DeliveryResult, no circuit
// breaker outcome, no status change, no count in the attempt metrics.
// The delivery stays pending or retrying, and the recovery routine
// sends it again on the next start.
//
// A target's own timeout leaves that context alone, so its attempt is
// still recorded as the failure it is.
func (c *httpCore) interruptedByStop(
ctx context.Context,
d *database.Delivery,
res attemptResult,
) bool {
if res.success || ctx.Err() == nil {
return false
}
c.eng.log.Warn(
"delivery attempt interrupted by shutdown; not recorded, "+
"it is sent again on the next start",
"delivery_id", d.ID,
"event_id", d.EventID,
"target_id", d.TargetID,
)
return true
}
func (c *httpCore) fireAndForget(
ctx context.Context,
webhookDB *gorm.DB,
d *database.Delivery,
res attemptResult,
) {
if c.interruptedByStop(ctx, d, res) {
return
}
c.eng.observeAttempt(d.Target.Type, res.elapsed())
err := c.eng.recordResult(
@@ -104,11 +124,12 @@ func (c *httpCore) fireAndForget(
}
func (c *httpCore) withRetry(
ctx context.Context,
webhookDB *gorm.DB,
d *database.Delivery,
task *Task,
sched Scheduler,
maxRetries int,
maxAttempts int,
attempt func() attemptResult,
) {
cb := c.getCircuitBreaker(task.TargetID)
@@ -125,6 +146,10 @@ func (c *httpCore) withRetry(
res := attempt()
if c.interruptedByStop(ctx, d, res) {
return
}
c.eng.observeAttempt(d.Target.Type, res.elapsed())
err := c.eng.recordResult(
@@ -156,7 +181,7 @@ func (c *httpCore) withRetry(
cb.RecordFailure()
c.handleRetry(
webhookDB, d, task, sched, maxRetries, attemptNum,
webhookDB, d, task, sched, maxAttempts, attemptNum,
)
}
@@ -217,10 +242,10 @@ func (c *httpCore) handleRetry(
d *database.Delivery,
task *Task,
sched Scheduler,
maxRetries int,
maxAttempts int,
attemptNum int,
) {
if attemptNum >= maxRetries {
if attemptNum >= maxAttempts {
c.eng.settleStatus(
webhookDB, d, d.Target.Type,
database.DeliveryStatusFailed,
@@ -234,7 +259,7 @@ func (c *httpCore) handleRetry(
database.DeliveryStatusRetrying,
)
backoff := calcBackoff(attemptNum)
backoff := Backoff(attemptNum)
retryTask := *task
retryTask.AttemptNum = attemptNum + 1
@@ -285,7 +310,8 @@ func (c *httpCore) publishCircuitState(
// remainingBackoff returns how long remains of the backoff
// window for the last attempt of a recovered retrying
// delivery. It implements rescheduler.
// delivery: zero once it has passed, or when the last attempt
// cannot be read. It implements rescheduler.
func (c *httpCore) remainingBackoff(
webhookDB *gorm.DB,
deliveryID string,
@@ -301,37 +327,16 @@ func (c *httpCore) remainingBackoff(
return 0
}
backoff := calcBackoff(attemptNum)
backoff := Backoff(attemptNum)
elapsed := time.Since(lastResult.CreatedAt)
remaining := backoff - elapsed
return max(remaining, 0)
}
// backoffElapsed reports whether the backoff window for the
// last attempt of a retrying delivery has passed. It
// implements rescheduler.
func (c *httpCore) backoffElapsed(
webhookDB *gorm.DB,
deliveryID string,
attemptNum int,
) bool {
var lastResult database.DeliveryResult
err := webhookDB.
Where("delivery_id = ?", deliveryID).
Order("created_at DESC").
First(&lastResult).Error
if err != nil {
return true
}
backoff := calcBackoff(attemptNum)
return time.Since(lastResult.CreatedAt) >= backoff
}
func calcBackoff(attemptNum int) time.Duration {
// Backoff is how long an http or slack target with retries waits after
// a delivery's failed attempt attemptNum before trying it again.
func Backoff(attemptNum int) time.Duration {
shift := max(attemptNum-1, 0)
shift = min(shift, maxBackoffShift)
@@ -356,39 +361,13 @@ func (t *httpTarget) Deliver(
task *Task,
sched Scheduler,
) {
cfg, err := parseHTTPConfig(d.Target.Config)
if err != nil {
t.eng.log.Error(
"invalid HTTP target config",
"target_id", d.TargetID,
"error", err,
)
recErr := t.eng.recordResult(
webhookDB, d, task.AttemptNum,
false, 0, "", err.Error(), 0,
)
if recErr != nil {
t.eng.bookkeepingFailed(d, recErr)
return
}
t.eng.settleStatus(
webhookDB, d, d.Target.Type,
database.DeliveryStatusFailed,
)
return
}
attempt := func() attemptResult {
return t.attempt(ctx, cfg, &d.Event)
return t.attempt(ctx, &d.Target, &d.Event)
}
t.deliver(
webhookDB, d, task, sched,
d.Target.MaxRetries, attempt,
ctx, webhookDB, d, task, sched,
d.Target.MaxAttempts, attempt,
)
}
@@ -398,11 +377,11 @@ func (t *httpTarget) Deliver(
// string; only a transport-level error does.
func (t *httpTarget) attempt(
ctx context.Context,
cfg *HTTPTargetConfig,
target *database.Target,
event *database.Event,
) attemptResult {
statusCode, respBody, duration, reqErr :=
t.doHTTPRequest(ctx, cfg, event)
t.doHTTPRequest(ctx, target, event)
success := reqErr == nil &&
statusCode >= httpSuccessMin &&
@@ -424,7 +403,7 @@ func (t *httpTarget) attempt(
func (t *httpTarget) doHTTPRequest(
ctx context.Context,
cfg *HTTPTargetConfig,
target *database.Target,
event *database.Event,
) (int, string, int64, error) {
start := time.Now()
@@ -432,7 +411,7 @@ func (t *httpTarget) doHTTPRequest(
req, reqErr := http.NewRequestWithContext(
ctx,
http.MethodPost,
cfg.URL,
target.URL,
bytes.NewReader([]byte(event.Body)),
)
if reqErr != nil {
@@ -442,9 +421,15 @@ func (t *httpTarget) doHTTPRequest(
)
}
originScoped := applyRequestHeaders(req, event, cfg)
if target.ForwardQuery {
appendQuery(req.URL, event.RawQuery)
}
client := t.clientForRequest(cfg, originScoped)
originScoped := applyRequestHeaders(
req, event, target.Headers, t.eng.userAgent(),
)
client := t.clientForRequest(target.TimeoutSeconds, originScoped)
resp, doErr := executeHTTPRequest(client, req)
@@ -470,18 +455,31 @@ func (t *httpTarget) doHTTPRequest(
return resp.StatusCode, string(body), dur, nil
}
// appendQuery adds an event's query string to a delivery's URL, joined
// with "&" to any query string the target URL already has.
func appendQuery(u *url.URL, rawQuery string) {
switch {
case rawQuery == "":
return
case u.RawQuery == "":
u.RawQuery = rawQuery
default:
u.RawQuery += "&" + rawQuery
}
}
// clientForRequest returns the client for one delivery attempt.
// originScoped is the header set applyRequestHeaders built for that
// attempt; a request with neither a per-target timeout nor an
// origin-scoped header gets the shared client, because there is
// then nothing for the redirect policy to strip and net/http's
// default policy already withholds Authorization and Cookie across
// hosts.
// timeoutSeconds is the target's TimeoutSeconds, and originScoped is
// the header set applyRequestHeaders built for that attempt; a
// request with neither a per-target timeout nor an origin-scoped
// header gets the shared client, because there is then nothing for
// the redirect policy to strip and net/http's default policy already
// withholds Authorization and Cookie across hosts.
func (t *httpTarget) clientForRequest(
cfg *HTTPTargetConfig,
timeoutSeconds int,
originScoped []string,
) *http.Client {
if cfg.Timeout <= 0 && len(originScoped) == 0 {
if timeoutSeconds <= 0 && len(originScoped) == 0 {
return t.client
}
@@ -494,9 +492,9 @@ func (t *httpTarget) clientForRequest(
Transport: t.client.Transport,
}
if cfg.Timeout > 0 {
if timeoutSeconds > 0 {
client.Timeout = time.Duration(
cfg.Timeout,
timeoutSeconds,
) * time.Second
}
@@ -507,31 +505,6 @@ func (t *httpTarget) clientForRequest(
return client
}
func parseHTTPConfig(
configJSON string,
) (*HTTPTargetConfig, error) {
if configJSON == "" {
return nil, errEmptyTargetConfig
}
var cfg HTTPTargetConfig
err := json.Unmarshal(
[]byte(configJSON), &cfg,
)
if err != nil {
return nil, fmt.Errorf(
"parsing config JSON: %w", err,
)
}
if cfg.URL == "" {
return nil, errMissingTargetURL
}
return &cfg, nil
}
// isForwardableHeader returns true if the header should
// be forwarded to targets.
func isForwardableHeader(name string) bool {
@@ -562,10 +535,14 @@ func isForwardableHeader(name string) bool {
// Content-Type goes out once: a Content-Type configured on the target
// wins, otherwise the event's ContentType, otherwise none. The inbound
// Content-Type in the event's headers is never forwarded.
//
// configured is the target's Headers. userAgent is set last, over
// any configured or inbound User-Agent.
func applyRequestHeaders(
req *http.Request,
event *database.Event,
cfg *HTTPTargetConfig,
configured map[string]string,
userAgent string,
) []string {
if event.ContentType != "" {
req.Header.Set(
@@ -575,12 +552,12 @@ func applyRequestHeaders(
originScoped := forwardEventHeaders(req, event)
for k, v := range cfg.Headers {
for k, v := range configured {
req.Header.Set(k, v)
originScoped[http.CanonicalHeaderKey(k)] = struct{}{}
}
req.Header.Set("User-Agent", "webhooker/1.0")
req.Header.Set("User-Agent", userAgent)
// A Content-Type configured on the target describes the body
// being sent rather than the sender. A 307/308 preserves the
@@ -640,7 +617,7 @@ func forwardEventHeaders(
}
// executeHTTPRequest sends an HTTP request using the provided
// client. URLs are validated by the config parsers and the
// client. URLs are validated by the target forms and the
// SSRF-safe transport before reaching here.
//
// Transport failures are masked here, at the single point
internal/delivery/target_http_query_test.go
+163
View File
@@ -0,0 +1,163 @@
package delivery_test
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/delivery"
)
// eventQuery is the query string the events in these tests arrived
// with.
const eventQuery = "a=1&b=2"
// deliverWithQuery sends one event that arrived with eventQuery to an
// HTTP target at targetURL, with its ForwardQuery setting as given,
// through the path a received event's delivery takes, and returns the
// attempt it recorded.
func deliverWithQuery(
t *testing.T, targetURL string, forwardQuery bool,
) database.DeliveryResult {
t.Helper()
s := newISetup(t)
event := iSeedEvent(t, s.WebhookDB, s.WebhookID, "{}")
d := iSeedDelivery(
t, s.WebhookDB, event.ID, uuid.NewString(),
database.DeliveryStatusPending,
)
task := iTask(
d, event, s.WebhookID, d.TargetID, "query", targetURL, 1, 1,
&event.Body,
)
task.ForwardQuery = forwardQuery
task.RawQuery = eventQuery
s.Engine.ExportProcessNewTask(context.TODO(), &task)
var result database.DeliveryResult
require.NoError(t, s.WebhookDB.Where(
"delivery_id = ?", d.ID,
).First(&result).Error)
return result
}
// TestDeliverHTTP_ForwardQuery proves the URL a delivery is sent to:
// with the target's setting off, the target URL exactly as configured;
// with it on, the event's query string appended, joined with "&" to a
// query string the target URL already has.
func TestDeliverHTTP_ForwardQuery(t *testing.T) {
t.Parallel()
// The target URL's path, without and with a query string of its
// own.
const (
plain = "/in"
withQuery = "/in?key=k"
)
tests := map[string]struct {
path string
forwardQuery bool
want string
}{
"off": {
path: plain, want: plain,
},
"off, the target URL has a query string": {
path: withQuery, want: withQuery,
},
"on": {
path: plain, forwardQuery: true, want: plain + "?" + eventQuery,
},
"on, the target URL has a query string": {
path: withQuery, forwardQuery: true,
want: withQuery + "&" + eventQuery,
},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
t.Parallel()
received := make(chan string, 1)
ts := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
received <- r.RequestURI
w.WriteHeader(http.StatusOK)
},
))
t.Cleanup(ts.Close)
result := deliverWithQuery(
t, ts.URL+tc.path, tc.forwardQuery,
)
assert.True(t, result.Success)
require.Len(t, received, 1)
assert.Equal(t, tc.want, <-received)
})
}
}
// TestDeliverHTTP_ForwardedQueryKeepsTheTargetURLMasked proves the
// credential in a target URL's own query string stays masked once the
// event's query string is appended to it: in a response or error that
// echoes the URL the target was sent, as the event log's Redactor shows
// it, and in the error a failed connection stores.
func TestDeliverHTTP_ForwardedQueryKeepsTheTargetURLMasked(t *testing.T) {
t.Parallel()
const secret = "s3cr3t"
received := make(chan string, 1)
ts := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
received <- r.RequestURI
w.WriteHeader(http.StatusBadRequest)
},
))
t.Cleanup(ts.Close)
target := &database.Target{
Type: database.TargetTypeHTTP,
URL: ts.URL + "/in?token=" + secret,
ForwardQuery: true,
MaxAttempts: 1,
}
deliverWithQuery(t, target.URL, target.ForwardQuery)
require.Len(t, received, 1)
sent := <-received
require.Equal(t, "/in?token="+secret+"&"+eventQuery, sent)
redactor := delivery.NewRedactor(target)
for _, echoed := range []string{sent, ts.URL + sent} {
shown := redactor.Redact("rejected " + echoed)
assert.NotContains(t, shown, secret, echoed)
assert.Contains(t, shown, delivery.RedactionMarker, echoed)
}
// Nothing listens on port 1.
failed := deliverWithQuery(
t, "http://127.0.0.1:1/in?token="+secret, true,
)
require.NotEmpty(t, failed.Error)
assert.NotContains(t, failed.Error, secret)
assert.NotContains(t, failed.Error, eventQuery)
}
internal/delivery/target_log.go
+4 -3
View File
@@ -9,9 +9,9 @@ import (
)
// logTarget is a fire-and-forget target that logs the entire
// inbound webhook — the full request body and headers, plus
// the method, content type, and the webhook and entrypoint
// ids — then records a single successful attempt.
// inbound webhook — the full request body, query string and
// headers, plus the method, content type, and the webhook and
// entrypoint ids — then records a single successful attempt.
//
// This is the one log call in the service that deliberately writes
// unbounded client-chosen bytes, so it is the one exception to the
@@ -46,6 +46,7 @@ func (t *logTarget) Deliver(
"webhook_id", d.Event.WebhookID,
"entrypoint_id", d.Event.EntrypointID,
"method", d.Event.Method,
"raw_query", d.Event.RawQuery,
"content_type", d.Event.ContentType,
"headers", d.Event.Headers,
"body", d.Event.Body,
internal/delivery/target_redact.go
+28 -46
View File
@@ -13,12 +13,12 @@ import (
const RedactionMarker = "(redacted)"
// Redactor removes one target's own credential material from
// text that target's remote peer chose: a delivery response
// body, or a delivery error stored before the delivery path
// learned to mask the URLs it embeds.
// a delivery response body, whose text the target's remote
// peer chose, and from a stored delivery error as a second
// line behind the delivery path's own URL masking.
//
// It removes byte-identical echoes of strings taken from the
// target's stored configuration, and nothing else. Anything
// target's URL and headers, and nothing else. Anything
// the remote re-encodes survives: JSON "\/" escaping (what
// PHP's json_encode emits by default), percent-encoding, HTML
// entities, and an echo of only part of a path. It cannot
@@ -115,7 +115,7 @@ func (r Redactor) secretPrefixSuffix(s string) int {
}
// targetSecrets returns the credential-bearing strings a
// target's configuration carries.
// target's settings carry.
//
// The destination URL contributes. Its path, query and
// userinfo are the credential for both target types that have
@@ -129,51 +129,34 @@ func (r Redactor) secretPrefixSuffix(s string) int {
// picks out. That is the same class-based rule applied to
// URLs: an echoed Accept or User-Agent still renders, an
// echoed Authorization does not.
//
// A setting a type does not use is empty and contributes
// nothing, so a database or log target has nothing to redact.
func targetSecrets(t *database.Target) []string {
if t == nil {
return nil
}
switch t.Type {
case database.TargetTypeSlack:
cfg, err := parseSlackConfig(t.Config)
if err != nil {
return nil
}
return urlSecrets(cfg.WebhookURL)
case database.TargetTypeHTTP:
cfg, err := parseHTTPConfig(t.Config)
if err != nil {
return nil
}
return append(
urlSecrets(cfg.URL),
headerSecrets(cfg.Headers)...,
)
case database.TargetTypeDatabase, database.TargetTypeLog:
// Neither has a destination URL, so neither has
// anything to redact.
return nil
default:
return nil
}
return append(urlSecrets(t.URL), headerSecrets(t.Headers)...)
}
// urlSecrets returns the substrings of a destination URL that
// must not survive into a rendered page: the whole URL, the
// parts of it MaskURL elides, and any userinfo.
// must not survive into a rendered page: the whole URL; its
// path, unless that is empty or "/"; its query string, and the
// request URI that carries it, which a remote echoing the
// request line shows even when the URL has no path; and its
// userinfo and password.
//
// No length floor is applied to the path, and none to the
// userinfo. A short path or a four-byte username is treated as
// a credential exactly like a long one, because the field takes
// an arbitrary URL and no part of it can be assumed non-secret —
// the same rule MaskURL applies. headerSecrets does carry a
// floor, and the difference is deliberate: a header is picked
// out by a name-shaped guess and its value may be ordinary
// text, whereas a URL's path and userinfo are credential
// material by position.
// No length floor is applied to the path, the query string or
// the userinfo. A short path or a four-byte username is
// treated as a credential exactly like a long one, because the
// field takes an arbitrary URL and no part of it can be
// assumed non-secret — the same rule MaskURL applies.
// headerSecrets does carry a floor, and the difference is
// deliberate: a header is picked out by a name-shaped guess
// and its value may be ordinary text, whereas a URL's path,
// query string and userinfo are credential material by
// position.
func urlSecrets(raw string) []string {
raw = strings.TrimSpace(raw)
if raw == "" {
@@ -188,12 +171,11 @@ func urlSecrets(raw string) []string {
}
if parsed.Path != "" && parsed.Path != "/" {
requestURI := parsed.RequestURI()
secrets = append(secrets, requestURI)
secrets = append(secrets, parsed.EscapedPath())
}
if escaped := parsed.EscapedPath(); escaped != requestURI {
secrets = append(secrets, escaped)
}
if parsed.RawQuery != "" {
secrets = append(secrets, parsed.RequestURI(), parsed.RawQuery)
}
if parsed.User != nil {
internal/delivery/target_redact_test.go
+85 -34
View File
@@ -22,8 +22,9 @@ func TestRedactor_RemovesSlackWebhookURL(t *testing.T) {
t.Parallel()
r := delivery.NewRedactor(&database.Target{
Type: database.TargetTypeSlack,
Config: `{"webhookUrl":"` + redactWebhookURL + `"}`,
Type: database.TargetTypeSlack,
URL: redactWebhookURL,
MaxAttempts: 1,
})
got := r.Redact("no_service for " + redactWebhookURL)
@@ -52,8 +53,9 @@ func TestRedactor_RemovesSecretSeveredByACut(t *testing.T) {
t.Parallel()
r := delivery.NewRedactor(&database.Target{
Type: database.TargetTypeSlack,
Config: `{"webhookUrl":"` + redactWebhookURL + `"}`,
Type: database.TargetTypeSlack,
URL: redactWebhookURL,
MaxAttempts: 1,
})
// Every cut position inside the credential, not just a
@@ -84,18 +86,20 @@ func TestRedactor_RedactsCredentialShapedHeaderValues(
r := delivery.NewRedactor(&database.Target{
Type: database.TargetTypeHTTP,
Config: `{"url":"https://example.com/in",` +
`"headers":{` +
`"Authorization":"Bearer AAAAAAAAAAAA",` +
`"Cookie":"session=BBBBBBBBBBBB",` +
`"X-Api-Key":"CCCCCCCCCCCC",` +
`"X-Hub-Signature":"sha256=DDDDDDDDDDDD",` +
`"X-Sig":"EEEEEEEEEEEE",` +
`"X-Pass":"FFFFFFFFFFFF",` +
`"X-HMAC":"GGGGGGGGGGGG",` +
`"X-Credential":"HHHHHHHHHHHH",` +
`"Accept":"application/json",` +
`"User-Agent":"webhooker/1.0"}}`,
URL: "https://example.com/in",
Headers: map[string]string{
headerAuthorization: "Bearer AAAAAAAAAAAA",
"Cookie": "session=BBBBBBBBBBBB",
probeHeaderName: "CCCCCCCCCCCC",
"X-Hub-Signature": "sha256=DDDDDDDDDDDD",
"X-Sig": "EEEEEEEEEEEE",
"X-Pass": "FFFFFFFFFFFF",
"X-HMAC": "GGGGGGGGGGGG",
"X-Credential": "HHHHHHHHHHHH",
"Accept": "application/json",
headerUserAgent: "webhooker/1.0",
},
MaxAttempts: 1,
})
for _, secret := range []string{
@@ -133,9 +137,10 @@ func TestRedactor_IgnoresVeryShortHeaderValues(t *testing.T) {
t.Parallel()
r := delivery.NewRedactor(&database.Target{
Type: database.TargetTypeHTTP,
Config: `{"url":"https://example.com/in",` +
`"headers":{"X-Api-Key":"ab"}}`,
Type: database.TargetTypeHTTP,
URL: "https://example.com/in",
Headers: map[string]string{probeHeaderName: "ab"},
MaxAttempts: 1,
})
const response = "rabbit"
@@ -150,8 +155,9 @@ func TestRedactor_RemovesBarePath(t *testing.T) {
t.Parallel()
r := delivery.NewRedactor(&database.Target{
Type: database.TargetTypeSlack,
Config: `{"webhookUrl":"` + redactWebhookURL + `"}`,
Type: database.TargetTypeSlack,
URL: redactWebhookURL,
MaxAttempts: 1,
})
got := r.Redact("POST " + redactSecretPath + " 404")
@@ -183,8 +189,9 @@ func TestRedactor_RemovesHTTPURLQueryAndUserinfo(t *testing.T) {
raw := dest.String()
r := delivery.NewRedactor(&database.Target{
Type: database.TargetTypeHTTP,
Config: `{"url":"` + raw + `"}`,
Type: database.TargetTypeHTTP,
URL: raw,
MaxAttempts: 1,
})
for _, echoed := range []string{
@@ -202,6 +209,49 @@ func TestRedactor_RemovesHTTPURLQueryAndUserinfo(t *testing.T) {
}
}
// TestRedactor_RemovesEchoedQueryOfURLWithoutPath covers an
// HTTP target URL whose credential is all in its query string.
// Written with or without the "/", the request line sends it
// as "/?token=…", and a target passing the event's query string
// on sends that after an "&". The event's part stays visible:
// the event's page shows it anyway.
func TestRedactor_RemovesEchoedQueryOfURLWithoutPath(t *testing.T) {
t.Parallel()
const secret = "s3cr3t"
marker := delivery.RedactionMarker
// An echoed request line, and what the event log shows of it.
echoes := map[string]string{
"POST /?token=" + secret + " HTTP/1.1": "POST " + marker +
" HTTP/1.1",
"POST ?token=" + secret + " HTTP/1.1": "POST ?" + marker +
" HTTP/1.1",
"POST /?token=" + secret + "&a=1&b=2 HTTP/1.1": "POST " +
marker + "&a=1&b=2 HTTP/1.1",
"POST ?token=" + secret + "&a=1&b=2 HTTP/1.1": "POST ?" +
marker + "&a=1&b=2 HTTP/1.1",
}
for _, dest := range []string{
"https://example.com/?token=" + secret,
"https://example.com?token=" + secret,
} {
r := delivery.NewRedactor(&database.Target{
Type: database.TargetTypeHTTP,
URL: dest,
MaxAttempts: 1,
})
for echoed, want := range echoes {
assert.Equal(
t, want, r.Redact(echoed), "%s: %s", dest, echoed,
)
}
}
}
// TestRedactor_LeavesUnrelatedTextAlone pins that the
// redactor matches literally: it does not guess at what a
// secret looks like, so ordinary response content survives.
@@ -211,8 +261,9 @@ func TestRedactor_LeavesUnrelatedTextAlone(t *testing.T) {
const response = "ok=false error=channel_not_found"
r := delivery.NewRedactor(&database.Target{
Type: database.TargetTypeSlack,
Config: `{"webhookUrl":"` + redactWebhookURL + `"}`,
Type: database.TargetTypeSlack,
URL: redactWebhookURL,
MaxAttempts: 1,
})
assert.Equal(t, response, r.Redact(response))
@@ -237,8 +288,9 @@ func TestRedactor_EmptyUserinfoDoesNotShredTheBody(t *testing.T) {
require.Empty(t, parsed.User.String())
r := delivery.NewRedactor(&database.Target{
Type: database.TargetTypeHTTP,
Config: `{"url":"` + dest + `"}`,
Type: database.TargetTypeHTTP,
URL: dest,
MaxAttempts: 1,
})
const body = "ok=false error=channel_not_found"
@@ -255,11 +307,10 @@ func TestRedactor_EmptyUserinfoDoesNotShredTheBody(t *testing.T) {
)
}
// TestRedactor_ZeroValueAndConfiglessTargets pins that a
// caller with no target, an unparseable config, or a target
// type with no destination URL gets a redactor that changes
// nothing rather than one that panics.
func TestRedactor_ZeroValueAndConfiglessTargets(t *testing.T) {
// TestRedactor_ZeroValueAndURLlessTargets pins that a caller
// with no target, or a target with no destination URL, gets a
// redactor that changes nothing rather than one that panics.
func TestRedactor_ZeroValueAndURLlessTargets(t *testing.T) {
t.Parallel()
const text = "some response body"
@@ -272,8 +323,8 @@ func TestRedactor_ZeroValueAndConfiglessTargets(t *testing.T) {
for _, tgt := range []database.Target{
{Type: database.TargetTypeLog},
{Type: database.TargetTypeDatabase},
{Type: database.TargetTypeSlack, Config: "not json"},
{Type: database.TargetTypeHTTP, Config: ""},
{Type: database.TargetTypeSlack, MaxAttempts: 1},
{Type: database.TargetTypeHTTP, MaxAttempts: 1},
} {
assert.Equal(
t, text,
internal/delivery/target_slack.go
+24 -87
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -15,24 +14,11 @@ import (
"sneak.berlin/go/webhooker/internal/database"
)
// errMissingWebhookURL is returned when a Slack target config
// omits its webhook URL.
var errMissingWebhookURL = errors.New(
"webhook_url is required",
)
// SlackTargetConfig holds configuration for slack target
// types.
type SlackTargetConfig struct {
WebhookURL string `json:"webhookUrl"`
}
// slackTarget delivers events to Slack incoming webhooks. It
// formats the event into a Slack message and posts it as
// JSON. It shares the retry core with the HTTP target: a
// MaxRetries of 0 stays single-attempt fire-and-forget
// (preserving existing Slack targets), while a positive
// MaxRetries adds backoff and circuit breaking.
// MaxAttempts of 1 is a single attempt, while a higher one
// adds backoff and circuit breaking.
type slackTarget struct {
*httpCore
@@ -47,19 +33,6 @@ func (t *slackTarget) Deliver(
task *Task,
sched Scheduler,
) {
cfg, err := parseSlackConfig(d.Target.Config)
if err != nil {
t.eng.log.Error(
"invalid Slack target config",
"target_id", d.TargetID,
"error", err,
)
t.failConfig(webhookDB, d, err)
return
}
msg := FormatSlackMessage(&d.Event)
payload, err := json.Marshal(
@@ -72,52 +45,41 @@ func (t *slackTarget) Deliver(
"error", err,
)
t.failConfig(webhookDB, d, err)
recErr := t.eng.recordResult(
webhookDB, d, 1,
false, 0, "", err.Error(), 0,
)
if recErr != nil {
t.eng.bookkeepingFailed(d, recErr)
return
}
t.eng.settleStatus(
webhookDB, d, d.Target.Type,
database.DeliveryStatusFailed,
)
return
}
attempt := func() attemptResult {
return t.attempt(ctx, cfg, payload)
return t.attempt(ctx, d.Target.URL, payload)
}
t.deliver(
webhookDB, d, task, sched,
d.Target.MaxRetries, attempt,
ctx, webhookDB, d, task, sched,
d.Target.MaxAttempts, attempt,
)
}
// failConfig records a first-attempt failure for a delivery
// that could not be prepared (bad config or unmarshalable
// payload) and marks it failed.
func (t *slackTarget) failConfig(
webhookDB *gorm.DB,
d *database.Delivery,
err error,
) {
recErr := t.eng.recordResult(
webhookDB, d, 1,
false, 0, "", err.Error(), 0,
)
if recErr != nil {
t.eng.bookkeepingFailed(d, recErr)
return
}
t.eng.settleStatus(
webhookDB, d, d.Target.Type,
database.DeliveryStatusFailed,
)
}
// attempt performs a single Slack POST and derives its
// outcome, preserving the engine's original semantics: a
// attempt performs a single Slack POST to targetURL and derives
// its outcome, preserving the engine's original semantics: a
// non-2xx response records an "HTTP <code>" error string and
// a transport error records a "sending request" error.
func (t *slackTarget) attempt(
ctx context.Context,
cfg *SlackTargetConfig,
targetURL string,
payload []byte,
) attemptResult {
start := time.Now()
@@ -125,7 +87,7 @@ func (t *slackTarget) attempt(
req, err := http.NewRequestWithContext(
ctx,
http.MethodPost,
cfg.WebhookURL,
targetURL,
bytes.NewReader(payload),
)
if err != nil {
@@ -136,7 +98,7 @@ func (t *slackTarget) attempt(
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", "webhooker/1.0")
req.Header.Set("User-Agent", t.eng.userAgent())
resp, doErr := executeHTTPRequest(t.client, req)
durationMs := time.Since(start).Milliseconds()
@@ -187,31 +149,6 @@ func (t *slackTarget) readSlackResponse(
}
}
func parseSlackConfig(
configJSON string,
) (*SlackTargetConfig, error) {
if configJSON == "" {
return nil, errEmptyTargetConfig
}
var cfg SlackTargetConfig
err := json.Unmarshal(
[]byte(configJSON), &cfg,
)
if err != nil {
return nil, fmt.Errorf(
"parsing config JSON: %w", err,
)
}
if cfg.WebhookURL == "" {
return nil, errMissingWebhookURL
}
return &cfg, nil
}
// FormatSlackMessage builds a Slack-compatible message
// string from a webhook event.
func FormatSlackMessage(
internal/delivery/terminal_state_test.go
+87 -55
View File
@@ -45,7 +45,7 @@ func tSeedDeletedTarget(
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, name,
database.TargetTypeHTTP, iHTTPConfig(url), 5,
database.TargetTypeHTTP, url, 5,
)
event := iSeedEvent(
@@ -122,9 +122,9 @@ func TestProcessDelivery_UnknownTargetType_RecordsWhy(
)
target := database.Target{
Name: "mystery",
Type: tUnknownType,
Config: iHTTPConfig("http://example.com/hook"),
Name: "mystery",
Type: tUnknownType,
URL: "http://example.com/hook",
}
target.ID = targetID
@@ -140,7 +140,7 @@ func TestProcessDelivery_UnknownTargetType_RecordsWhy(
body := event.Body
task := iTask(
seeded, event, s.WebhookID, targetID, "mystery",
target.Config, 0, 1, &body,
target.URL, 0, 1, &body,
)
task.TargetType = tUnknownType
@@ -178,7 +178,7 @@ func TestRecoverSingleRetry_TargetDeleted(t *testing.T) {
)
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID,
context.Background(), s.WebhookID, atStart,
)
iAssertStatus(
@@ -201,7 +201,7 @@ func TestRecoverSingleRetry_TargetDeleted(t *testing.T) {
)
}
func TestSweepSingleRetry_TargetDeleted(t *testing.T) {
func TestRecoverSingleRetry_OnTick_TargetDeleted(t *testing.T) {
t.Parallel()
s := newISetup(t)
@@ -218,11 +218,11 @@ func TestSweepSingleRetry_TargetDeleted(t *testing.T) {
// Twice, because the bug was an error the sweep repeated every
// minute for the life of the database: the second sweep must
// find nothing left to do.
s.Engine.ExportSweepWebhookRetries(
context.Background(), s.WebhookID,
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID, onTick,
)
s.Engine.ExportSweepWebhookRetries(
context.Background(), s.WebhookID,
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID, onTick,
)
iAssertStatus(
@@ -239,10 +239,11 @@ func TestSweepSingleRetry_TargetDeleted(t *testing.T) {
assert.Zero(t, s.Engine.ExportInflightHeld())
}
// TestSweepSingleRetry_TargetNeverExisted covers the other half of the
// soft-delete distinction: an id with no row at all, deleted or
// otherwise, must not be reported as something the operator deleted.
func TestSweepSingleRetry_TargetNeverExisted(t *testing.T) {
// TestRecoverSingleRetry_OnTick_TargetNeverExisted covers the other
// half of the soft-delete distinction: an id with no row at all,
// deleted or otherwise, must not be reported as something the operator
// deleted.
func TestRecoverSingleRetry_OnTick_TargetNeverExisted(t *testing.T) {
t.Parallel()
s := newISetup(t)
@@ -264,8 +265,8 @@ func TestSweepSingleRetry_TargetNeverExisted(t *testing.T) {
iSeedFailedResult(t, s.WebhookDB, d.ID)
s.Engine.ExportSweepWebhookRetries(
context.Background(), s.WebhookID,
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID, onTick,
)
iAssertStatus(
@@ -303,8 +304,8 @@ func TestFailMissingTarget_WritesNoTargetRow(
database.DeliveryStatusRetrying,
)
s.Engine.ExportSweepWebhookRetries(
context.Background(), s.WebhookID,
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID, onTick,
)
iAssertStatus(
@@ -312,13 +313,9 @@ func TestFailMissingTarget_WritesNoTargetRow(
database.DeliveryStatusFailed,
)
var configs []string
require.NoError(t, s.WebhookDB.
Table("targets").
Pluck("config", &configs).Error)
assert.Empty(t, configs,
// The per-webhook database has no targets table, so writing a
// target row there would have failed the status write above.
assert.False(t, s.WebhookDB.Migrator().HasTable("targets"),
"the deleted-target terminal path wrote a target row "+
"into the per-webhook event database",
)
@@ -349,11 +346,10 @@ func tRetryChainSetup(
iCreateWebhook(t, s.MainDB, s.WebhookID, name)
targetID := uuid.New().String()
cfg := iHTTPConfig(ts.URL)
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, name,
database.TargetTypeHTTP, cfg, 5,
database.TargetTypeHTTP, ts.URL, 5,
)
event := iSeedEvent(
@@ -370,7 +366,7 @@ func tRetryChainSetup(
body := event.Body
return iTask(
d, event, s.WebhookID, targetID, name, cfg, 5, 2, &body,
d, event, s.WebhookID, targetID, name, ts.URL, 5, 2, &body,
), targetID
}
@@ -418,6 +414,38 @@ func TestProcessRetryTask_TargetDeleted_MakesNoAttempt(
assert.Zero(t, s.Engine.ExportInflightHeld())
}
// TestProcessRetryTask_TargetDeleted_CountsFailureOnTarget verifies
// that the failure of a retry abandoned because its target is gone is
// added to that target's own totals, not to a row with no target.
func TestProcessRetryTask_TargetDeleted_CountsFailureOnTarget(
t *testing.T,
) {
t.Parallel()
s := newISetup(t)
var hits atomic.Int64
task, targetID := tRetryChainSetup(
t, s, "gone-counted", &hits,
)
require.NoError(t, s.MainDB.Delete(
&database.Target{}, "id = ?", targetID,
).Error)
s.Engine.ExportProcessRetryTask(
context.Background(), &task,
)
var rows []database.TargetTotals
require.NoError(t, s.WebhookDB.Find(&rows).Error)
assert.Equal(t, []database.TargetTotals{
{TargetID: targetID, Failed: 1},
}, rows)
}
// TestProcessRetryTask_TargetPresent_StillDelivers is the guard's
// mutation check: a liveness check that refused every retry would pass
// the test above and break every retry there is.
@@ -499,7 +527,7 @@ func TestRecoverSingleRetry_TargetUnreadable_LeavesDeliveryAlone(
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "healthy",
database.TargetTypeHTTP,
iHTTPConfig("http://example.com/hook"), 5,
"http://example.com/hook", 5,
)
event := iSeedEvent(
@@ -518,7 +546,7 @@ func TestRecoverSingleRetry_TargetUnreadable_LeavesDeliveryAlone(
require.NoError(t, sqlDB.Close())
s.Engine.ExportRecoverRetryingDeliveries(
s.WebhookDB, s.WebhookID,
context.Background(), s.WebhookDB, s.WebhookID,
)
iAssertStatus(
@@ -536,18 +564,20 @@ func TestRecoverSingleRetry_TargetUnreadable_LeavesDeliveryAlone(
// --- 4. A pending delivery whose target is gone ---
func TestRecoverPending_TargetDeleted(t *testing.T) {
func TestRecoverPendingDeliveries_TargetDeleted(t *testing.T) {
t.Parallel()
s := newISetup(t)
iCreateWebhook(t, s.MainDB, s.WebhookID, "pending-recovery")
deliveryID := tSeedDeletedTarget(
t, s, "gone-while-pending", "http://example.com/hook",
database.DeliveryStatusPending,
)
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID,
context.Background(), s.WebhookID, atStart,
)
iAssertStatus(
@@ -570,16 +600,18 @@ func TestRecoverPending_TargetDeleted(t *testing.T) {
)
}
// TestRecoverPending_TargetDeleted_LeavesAnOwnedDeliveryAlone: the
// terminal write takes ownership like every other recovery write, so a
// delivery the engine still holds is not failed underneath its worker.
func TestRecoverPending_TargetDeleted_LeavesAnOwnedDeliveryAlone(
// TestRecoverPendingDeliveries_TargetDeleted_LeavesAnOwnedDeliveryAlone:
// the terminal write takes ownership like every other recovery write, so
// a delivery the engine still holds is not failed underneath its worker.
func TestRecoverPendingDeliveries_TargetDeleted_LeavesAnOwnedDeliveryAlone(
t *testing.T,
) {
t.Parallel()
s := newISetup(t)
iCreateWebhook(t, s.MainDB, s.WebhookID, "owned-recovery")
deliveryID := tSeedDeletedTarget(
t, s, "gone-but-owned", "http://example.com/hook",
database.DeliveryStatusPending,
@@ -588,7 +620,7 @@ func TestRecoverPending_TargetDeleted_LeavesAnOwnedDeliveryAlone(
require.True(t, s.Engine.ExportRetainDelivery(deliveryID))
s.Engine.ExportRecoverWebhookDeliveries(
context.Background(), s.WebhookID,
context.Background(), s.WebhookID, atStart,
)
iAssertStatus(
@@ -643,18 +675,18 @@ func TestFailMissingTarget_LeavesASettledDeliveryAlone(
assert.Zero(t, s.Engine.ExportInflightHeld())
}
// TestSweepPending_TargetDeleted sweeps twice over a batch that also
// holds a healthy stranded delivery. The one whose target is gone is
// failed once and then left alone; the healthy one is queued by the
// first sweep and not again by the second.
func TestSweepPending_TargetDeleted(t *testing.T) {
// TestRecoverPendingDeliveries_OnTick_TargetDeleted sweeps twice over a
// batch that also holds a healthy stranded delivery. The one whose
// target is gone is failed once and then left alone; the healthy one is
// queued by the first sweep and not again by the second.
func TestRecoverPendingDeliveries_OnTick_TargetDeleted(t *testing.T) {
t.Parallel()
liveTargetID := uuid.New().String()
s := fSweepSetup(t, liveTargetID, "still-there")
deliveryID := tSeedDeletedTarget(
t, s, "gone-on-pending-sweep", "http://example.com/hook",
t, s, "gone-while-pending-on-tick", "http://example.com/hook",
database.DeliveryStatusPending,
)
rAgePending(t, s.WebhookDB, deliveryID)
@@ -671,7 +703,7 @@ func TestSweepPending_TargetDeleted(t *testing.T) {
ctx := context.Background()
s.Engine.ExportSweepWebhookRetries(ctx, s.WebhookID)
s.Engine.ExportRecoverWebhookDeliveries(ctx, s.WebhookID, onTick)
tasks := fDrain(s.Engine)
require.Len(t, tasks, 1,
@@ -679,7 +711,7 @@ func TestSweepPending_TargetDeleted(t *testing.T) {
)
assert.Equal(t, healthy.ID, tasks[0].DeliveryID)
s.Engine.ExportSweepWebhookRetries(ctx, s.WebhookID)
s.Engine.ExportRecoverWebhookDeliveries(ctx, s.WebhookID, onTick)
assert.Empty(t, fDrain(s.Engine),
"the second sweep queued a delivery again",
@@ -692,7 +724,7 @@ func TestSweepPending_TargetDeleted(t *testing.T) {
last := tLastResult(t, s, deliveryID, 2)
assert.Contains(t, last.Error, "gone-on-pending-sweep")
assert.Contains(t, last.Error, "gone-while-pending-on-tick")
assert.Contains(t, last.Error, "was deleted")
}
@@ -711,7 +743,7 @@ func TestSendRecoveredDeliveries_TargetMissingFromMap(
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "found-on-lookup",
database.TargetTypeLog, "", 0,
database.TargetTypeLog, "", 1,
)
event := iSeedEvent(
@@ -743,11 +775,11 @@ func TestSendRecoveredDeliveries_TargetMissingFromMap(
)
}
// TestRecoverPending_TargetUnreadable_LeavesDeliveryAlone: a failed
// read of the main database is not a deleted target. Restart recovery
// holds every pending delivery of the webhook in one batch, so failing
// on this would fail all of them.
func TestRecoverPending_TargetUnreadable_LeavesDeliveryAlone(
// TestRecoverPendingDeliveries_TargetUnreadable_LeavesDeliveryAlone: a
// failed read of the main database is not a deleted target. At start the
// recovery routine holds every pending delivery of the webhook in one
// batch, so failing on this would fail all of them.
func TestRecoverPendingDeliveries_TargetUnreadable_LeavesDeliveryAlone(
t *testing.T,
) {
t.Parallel()
@@ -758,7 +790,7 @@ func TestRecoverPending_TargetUnreadable_LeavesDeliveryAlone(
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "healthy",
database.TargetTypeLog, "", 0,
database.TargetTypeLog, "", 1,
)
event := iSeedEvent(
@@ -775,7 +807,7 @@ func TestRecoverPending_TargetUnreadable_LeavesDeliveryAlone(
require.NoError(t, sqlDB.Close())
s.Engine.ExportRecoverPendingDeliveries(
context.Background(), s.WebhookDB, s.WebhookID,
context.Background(), s.WebhookDB, s.WebhookID, atStart,
)
iAssertStatus(
internal/delivery/url_mask_test.go
+24 -15
View File
@@ -2,7 +2,6 @@ package delivery_test
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
@@ -71,13 +70,6 @@ func deliverSlackTo(
e := testEngine(t, 1)
targetID := uuid.New().String()
slackCfg, err := json.Marshal(
delivery.SlackTargetConfig{
WebhookURL: webhookURL,
},
)
require.NoError(t, err)
event := seedEvent(t, db, `{"test":true}`)
dlv := seedDelivery(
@@ -87,7 +79,7 @@ func deliverSlackTo(
d := buildSlackDelivery(
dlv, event, targetID,
"test-slack-mask", string(slackCfg),
"test-slack-mask", webhookURL,
)
e.ExportDeliverSlack(context.TODO(), db, d)
@@ -160,13 +152,9 @@ func TestDoHTTPRequest_TransportErrorMasksURL(t *testing.T) {
e := testEngine(t, 1)
cfg, err := e.ExportParseHTTPConfig(
newHTTPTargetConfig(host + maskSecretPath),
)
require.NoError(t, err)
statusCode, _, _, reqErr := e.ExportDoHTTPRequest(
context.TODO(), cfg,
context.TODO(),
&database.Target{URL: host + maskSecretPath},
&database.Event{Body: `{"test":true}`},
)
require.Error(t, reqErr)
@@ -179,6 +167,27 @@ func TestDoHTTPRequest_TransportErrorMasksURL(t *testing.T) {
)
}
// TestDoHTTPRequest_UnparsableURLIsMasked is the same for an HTTP
// target URL that no request can be built from.
func TestDoHTTPRequest_UnparsableURLIsMasked(t *testing.T) {
t.Parallel()
e := testEngine(t, 1)
statusCode, _, _, reqErr := e.ExportDoHTTPRequest(
context.TODO(),
&database.Target{
URL: "https://hooks.example.com" + maskSecretPath + "\n",
},
&database.Event{},
)
require.Error(t, reqErr)
assert.Zero(t, statusCode)
assertNoCredential(t, reqErr.Error())
assert.Contains(t, reqErr.Error(), "invalid control character")
}
// TestValidateTargetURL_UnparsableURLIsMasked proves the SSRF
// validator's error does not carry the submitted URL, which
// the handler both logs and shows.
internal/delivery/user_agent_test.go
+85
View File
@@ -0,0 +1,85 @@
package delivery_test
import (
"context"
"net/http"
"net/http/httptest"
"net/netip"
"testing"
"github.com/google/uuid"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/fx/fxtest"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/delivery"
"sneak.berlin/go/webhooker/internal/globals"
"sneak.berlin/go/webhooker/internal/logger"
"sneak.berlin/go/webhooker/internal/metrics"
)
// Both the http and the slack target send webhooker/ and the version
// in Globals, the value the web UI footer shows. A User-Agent
// configured on the target or carried in by the sender does not
// replace it.
func TestUserAgent_IsTheBuildVersion(t *testing.T) {
t.Parallel()
const want = "webhooker/1.2.3-test"
userAgents := make(chan string, 1)
ts := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
userAgents <- r.Header.Get("User-Agent")
w.WriteHeader(http.StatusOK)
},
))
defer ts.Close()
g := &globals.Globals{Version: "1.2.3-test"}
lc := fxtest.NewLifecycle(t)
log, err := logger.New(lc, logger.LoggerParams{Globals: g})
require.NoError(t, err)
e := delivery.New(lc, delivery.EngineParams{
Globals: g,
Logger: log,
// httptest listens on loopback, which the default guard
// refuses.
SSRFGuard: delivery.NewTestGuard(
netip.MustParsePrefix("127.0.0.0/8"),
),
Metrics: metrics.New(prometheus.NewRegistry()),
})
statusCode, _, _, err := e.ExportDoHTTPRequest(
context.Background(),
&database.Target{
URL: ts.URL,
Headers: map[string]string{headerUserAgent: "configured/1"},
},
&database.Event{Headers: `{"User-Agent":["curl/8"]}`},
)
require.NoError(t, err)
require.Equal(t, http.StatusOK, statusCode)
require.Len(t, userAgents, 1, "the http target sent no request")
assert.Equal(t, want, <-userAgents, "http target")
db := testWebhookDB(t)
targetID := uuid.New().String()
event := seedEvent(t, db, `{"action":"test"}`)
dlv := seedDelivery(
t, db, event.ID, targetID, database.DeliveryStatusPending,
)
e.ExportDeliverSlack(context.Background(), db, buildSlackDelivery(
dlv, event, targetID, "test-slack", ts.URL,
))
require.Len(t, userAgents, 1, "the slack target sent no request")
assert.Equal(t, want, <-userAgents, "slack target")
}