sneak/webhooker
1
0
Fork 0
Code Issues 5 Pull Requests 1 Actions 11 Packages Projects Releases Wiki Activity

Store a target's settings as columns (closes #543)
check / check (push) Waiting to run
Details

Browse Source 
A target's settings move from the per-type JSON string in
targets.config to columns: url, headers, timeout_seconds,
forward_query, archive_expiry and archive_rotation. A column a type
does not use stays empty. headers holds the header map as JSON text
through GORM's JSON serializer, and archive_expiry holds a
time.Duration, 0 meaning never. The forms check each setting once,
with the same refusals and messages, and the edit form is filled from
the columns, so the per-type config structs, their parsers and the
"could not be read" branches are gone. A delivery task carries the
six settings in place of the JSON string; tests check that each copy
into a task, and back out of one, keeps all six.

Model: opus-5-5
This commit is contained in:
clawbot
2026-10-15 20:40:25 +00:00
parent ded39aa158
commit aa12ba19ca
71 changed files with 1665 additions and 2168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/delivery/client_ssrf_test.go
+3 -19
View File
@@ -46,12 +46,7 @@ func TestClientForRequest_TimeoutKeepsSSRFGuard(t *testing.T) {
t.Run(target, func(t *testing.T) {
t.Parallel()
cfg := &delivery.HTTPTargetConfig{
URL: target,
Timeout: 5,
}
client := engine.ExportClientForRequest(cfg, nil)
client := engine.ExportClientForRequest(5, nil)
require.NotSame(t, engine.ExportClient(), client,
"a per-target timeout must yield a "+
@@ -101,11 +96,7 @@ func TestClientForRequest_NoTimeoutUnchanged(t *testing.T) {
engine := newSSRFTestEngine()
cfg := &delivery.HTTPTargetConfig{
URL: "https://example.com/hook",
}
client := engine.ExportClientForRequest(cfg, nil)
client := engine.ExportClientForRequest(0, nil)
assert.Same(t, engine.ExportClient(), client,
"without a per-target timeout the shared client "+
@@ -123,15 +114,8 @@ func TestClientForRequest_HeadersKeepSSRFGuard(t *testing.T) {
engine := newSSRFTestEngine()
cfg := &delivery.HTTPTargetConfig{
URL: "https://example.com/with-headers",
Headers: map[string]string{
"X-Api-Key": "configured",
},
}
client := engine.ExportClientForRequest(
cfg, []string{"X-Api-Key"},
0, []string{probeHeaderName},
)
require.NotNil(t, client.CheckRedirect,
internal/delivery/engine.go
+41 -24
View File
@@ -105,12 +105,19 @@ type Task struct {
WebhookID string
EntrypointID string
TargetID string
TargetName string
TargetType database.TargetType
TargetConfig string
MaxAttempts int
// The target's fields, as they were when the task was made.
TargetID string
TargetName string
TargetType database.TargetType
TargetURL string
TargetHeaders map[string]string
TimeoutSeconds int
ForwardQuery bool
ArchiveExpiry time.Duration
ArchiveRotation string
MaxAttempts int
// The event's fields.
Method string
RawQuery string
Headers string
@@ -1543,10 +1550,15 @@ func buildEventFromTask(task *Task) database.Event {
func buildTargetFromTask(task *Task) database.Target {
target := database.Target{
Name: task.TargetName,
Type: task.TargetType,
Config: task.TargetConfig,
MaxAttempts: task.MaxAttempts,
Name: task.TargetName,
Type: task.TargetType,
URL: task.TargetURL,
Headers: task.TargetHeaders,
TimeoutSeconds: task.TimeoutSeconds,
ForwardQuery: task.ForwardQuery,
ArchiveExpiry: task.ArchiveExpiry,
ArchiveRotation: task.ArchiveRotation,
MaxAttempts: task.MaxAttempts,
}
target.ID = task.TargetID
@@ -1825,21 +1837,26 @@ func buildRecoveryTask(
}
return Task{
DeliveryID: d.ID,
EventID: d.EventID,
WebhookID: webhookID,
EntrypointID: event.EntrypointID,
TargetID: target.ID,
TargetName: target.Name,
TargetType: target.Type,
TargetConfig: target.Config,
MaxAttempts: target.MaxAttempts,
Method: event.Method,
RawQuery: event.RawQuery,
Headers: event.Headers,
ContentType: event.ContentType,
Body: bodyPtr,
AttemptNum: attemptNum,
DeliveryID: d.ID,
EventID: d.EventID,
WebhookID: webhookID,
EntrypointID: event.EntrypointID,
TargetID: target.ID,
TargetName: target.Name,
TargetType: target.Type,
TargetURL: target.URL,
TargetHeaders: target.Headers,
TimeoutSeconds: target.TimeoutSeconds,
ForwardQuery: target.ForwardQuery,
ArchiveExpiry: target.ArchiveExpiry,
ArchiveRotation: target.ArchiveRotation,
MaxAttempts: target.MaxAttempts,
Method: event.Method,
RawQuery: event.RawQuery,
Headers: event.Headers,
ContentType: event.ContentType,
Body: bodyPtr,
AttemptNum: attemptNum,
}
}
internal/delivery/engine_integration_test.go
+84 -136
View File
@@ -3,7 +3,6 @@ package delivery_test
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"log/slog"
@@ -97,17 +96,6 @@ func iSeedWebhookDB(
return db
}
func iHTTPConfig(url string) string {
cfg := delivery.HTTPTargetConfig{URL: url}
data, err := json.Marshal(cfg)
if err != nil {
panic("failed to marshal HTTPTargetConfig")
}
return string(data)
}
func iEngine(
t *testing.T, workers int,
) *delivery.Engine {
@@ -169,24 +157,24 @@ func iSeedDelivery(
func iTask(
d database.Delivery,
event database.Event,
webhookID, targetID, name, config string,
webhookID, targetID, name, targetURL string,
maxAttempts, attemptNum int,
body *string,
) delivery.Task {
return delivery.Task{
DeliveryID: d.ID,
EventID: event.ID,
WebhookID: webhookID,
TargetID: targetID,
TargetName: name,
TargetType: database.TargetTypeHTTP,
TargetConfig: config,
MaxAttempts: maxAttempts,
Method: event.Method,
Headers: event.Headers,
ContentType: event.ContentType,
Body: body,
AttemptNum: attemptNum,
DeliveryID: d.ID,
EventID: event.ID,
WebhookID: webhookID,
TargetID: targetID,
TargetName: name,
TargetType: database.TargetTypeHTTP,
TargetURL: targetURL,
MaxAttempts: maxAttempts,
Method: event.Method,
Headers: event.Headers,
ContentType: event.ContentType,
Body: body,
AttemptNum: attemptNum,
}
}
@@ -239,10 +227,9 @@ func TestProcessNewTask_InlineBody(t *testing.T) {
)
bodyStr := event.Body
cfg := iHTTPConfig(ts.URL)
task := iTask(
d, event, s.WebhookID, targetID,
"test-target", cfg, 1, 1, &bodyStr,
"test-target", ts.URL, 1, 1, &bodyStr,
)
s.Engine.ExportProcessNewTask(
@@ -290,10 +277,9 @@ func TestProcessNewTask_LargeBody_FetchFromDB(
database.DeliveryStatusPending,
)
cfg := iHTTPConfig(ts.URL)
task := iTask(
d, event, s.WebhookID, targetID,
"test-large", cfg, 1, 1, nil,
"test-large", ts.URL, 1, 1, nil,
)
s.Engine.ExportProcessNewTask(
@@ -313,16 +299,16 @@ func TestProcessNewTask_InvalidWebhookID(t *testing.T) {
s := newISetup(t)
task := delivery.Task{
DeliveryID: uuid.New().String(),
EventID: uuid.New().String(),
WebhookID: uuid.New().String(),
TargetID: uuid.New().String(),
TargetName: "test",
TargetType: database.TargetTypeHTTP,
TargetConfig: iHTTPConfig("http://localhost:9999"),
MaxAttempts: 1,
Body: nil,
AttemptNum: 1,
DeliveryID: uuid.New().String(),
EventID: uuid.New().String(),
WebhookID: uuid.New().String(),
TargetID: uuid.New().String(),
TargetName: "test",
TargetType: database.TargetTypeHTTP,
TargetURL: "http://localhost:9999",
MaxAttempts: 1,
Body: nil,
AttemptNum: 1,
}
s.Engine.ExportProcessNewTask(
@@ -363,7 +349,6 @@ func TestProcessRetryTask_SuccessfulRetry(t *testing.T) {
)
bodyStr := event.Body
cfg := iHTTPConfig(ts.URL)
// The target row exists because the engine confirms a scheduled
// retry's target has not been deleted before it runs it. A retry
@@ -372,12 +357,12 @@ func TestProcessRetryTask_SuccessfulRetry(t *testing.T) {
// task. See https://git.eeqj.de/sneak/webhooker/issues/107.
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "retry-target",
database.TargetTypeHTTP, cfg, 5,
database.TargetTypeHTTP, ts.URL, 5,
)
task := iTask(
d, event, s.WebhookID, targetID,
"retry-target", cfg, 5, 2, &bodyStr,
"retry-target", ts.URL, 5, 2, &bodyStr,
)
s.Engine.ExportProcessRetryTask(
@@ -410,10 +395,9 @@ func TestProcessRetryTask_SkipsNonRetryingDelivery(
)
bodyStr := event.Body
cfg := iHTTPConfig("http://localhost:1")
task := iTask(
d, event, s.WebhookID, targetID,
"skip-target", cfg, 5, 2, &bodyStr,
"skip-target", "http://localhost:1", 5, 2, &bodyStr,
)
s.Engine.ExportProcessRetryTask(
@@ -460,16 +444,14 @@ func TestProcessRetryTask_LargeBody_FetchFromDB(
database.DeliveryStatusRetrying,
)
cfg := iHTTPConfig(ts.URL)
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "retry-large",
database.TargetTypeHTTP, cfg, 5,
database.TargetTypeHTTP, ts.URL, 5,
)
task := iTask(
d, event, s.WebhookID, targetID,
"retry-large", cfg, 5, 2, nil,
"retry-large", ts.URL, 5, 2, nil,
)
s.Engine.ExportProcessRetryTask(
@@ -570,16 +552,15 @@ func TestWorkerLifecycle_ProcessesRetryChannel(
s.Engine.ExportStart()
bodyStr := event.Body
cfg := iHTTPConfig(ts.URL)
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "retry-chan-test",
database.TargetTypeHTTP, cfg, 5,
database.TargetTypeHTTP, ts.URL, 5,
)
task := iTask(
d, event, s.WebhookID, targetID,
"retry-chan-test", cfg, 5, 2, &bodyStr,
"retry-chan-test", ts.URL, 5, 2, &bodyStr,
)
s.Engine.ExportRetryCh() <- task
@@ -685,7 +666,7 @@ func iCreateTarget(
mainDB *gorm.DB,
targetID, webhookID, name string,
targetType database.TargetType,
config string,
targetURL string,
maxAttempts int,
) {
t.Helper()
@@ -695,7 +676,7 @@ func iCreateTarget(
Name: name,
Type: targetType,
Active: true,
Config: config,
URL: targetURL,
MaxAttempts: maxAttempts,
}
target.ID = targetID
@@ -736,7 +717,7 @@ func TestRecoverWebhookDeliveries_RetryingDeliveries(
iCreateTarget(t, s.MainDB, targetID,
s.WebhookID, "retry-recovery",
database.TargetTypeHTTP,
iHTTPConfig("http://example.com/hook"), 5,
"http://example.com/hook", 5,
)
event := iSeedEvent(
@@ -796,7 +777,7 @@ func iSeedRetryingWithType(
iCreateTarget(t, s.MainDB, targetID,
s.WebhookID, "mutated-target", targetType,
iHTTPConfig("http://example.com/hook"), 5,
"http://example.com/hook", 5,
)
event := iSeedEvent(
@@ -934,14 +915,14 @@ func TestFailUnretryableRetry_WritesNoTargetRow(
targetID := uuid.New().String()
// A Slack incoming-webhook URL: the target config IS the
// A Slack incoming-webhook URL: the target's URL IS the
// credential, which is what makes a leaked target row a
// disclosure rather than a curiosity.
hookURL := "https://hooks.slack.com/services/T00/B00/x"
iCreateTarget(t, s.MainDB, targetID,
s.WebhookID, "credential-bearing",
database.TargetTypeLog, iHTTPConfig(hookURL), 5,
database.TargetTypeLog, hookURL, 5,
)
event := iSeedEvent(
@@ -1186,7 +1167,7 @@ func TestRecoverDeliveries_SkipsAWebhookDeletedAfterTheListIsRead(
assert.False(t, dbMgr.DBExists(webhookID))
}
// --- HTTP Config with custom headers ---
// --- HTTP target with custom headers ---
func TestDeliverHTTP_CustomTargetHeaders(t *testing.T) {
t.Parallel()
@@ -1208,16 +1189,6 @@ func TestDeliverHTTP_CustomTargetHeaders(t *testing.T) {
)
defer ts.Close()
cfg := delivery.HTTPTargetConfig{
URL: ts.URL,
Headers: map[string]string{
"Authorization": "Bearer secret-token",
},
}
cfgJSON, err := json.Marshal(cfg)
require.NoError(t, err)
event := iSeedEvent(
t, s.WebhookDB, s.WebhookID,
`{"auth":"test"}`,
@@ -1232,9 +1203,12 @@ func TestDeliverHTTP_CustomTargetHeaders(t *testing.T) {
bodyStr := event.Body
task := iTask(
d, event, s.WebhookID, targetID,
"auth-target", string(cfgJSON),
"auth-target", ts.URL,
1, 1, &bodyStr,
)
task.TargetHeaders = map[string]string{
headerAuthorization: "Bearer secret-token",
}
s.Engine.ExportProcessNewTask(
context.TODO(), &task,
@@ -1247,11 +1221,14 @@ func TestDeliverHTTP_CustomTargetHeaders(t *testing.T) {
// --- HTTP delivery with custom timeout ---
// TestDeliverHTTP_TargetTimeout proves a task's TimeoutSeconds reaches
// the delivery through the engine. The destination answers after two
// seconds, inside the engine's own five-second client timeout, so only
// the task's one second can fail the delivery.
func TestDeliverHTTP_TargetTimeout(t *testing.T) {
t.Parallel()
db := testWebhookDB(t)
e := iEngine(t, 1)
s := newISetup(t)
ts := httptest.NewServer(
http.HandlerFunc(
@@ -1263,31 +1240,30 @@ func TestDeliverHTTP_TargetTimeout(t *testing.T) {
)
defer ts.Close()
cfg := delivery.HTTPTargetConfig{
URL: ts.URL,
Timeout: 1,
}
event := iSeedEvent(
t, s.WebhookDB, s.WebhookID, `{"timeout":"test"}`,
)
targetID := uuid.New().String()
cfgJSON, err := json.Marshal(cfg)
require.NoError(t, err)
event, del := iSeedEventAndDelivery(
t, db, `{"timeout":"test"}`,
string(cfgJSON),
d := iSeedDelivery(
t, s.WebhookDB, event.ID, targetID,
database.DeliveryStatusPending,
)
task, d := iHTTPTaskAndDelivery(
event, del, "timeout-target",
string(cfgJSON), 1, 1,
bodyStr := event.Body
task := iTask(
d, event, s.WebhookID, targetID,
"timeout-target", ts.URL, 1, 1, &bodyStr,
)
task.TimeoutSeconds = 1
e.ExportDeliverHTTP(context.TODO(), db, d, task)
s.Engine.ExportProcessNewTask(context.TODO(), &task)
iAssertStatus(t, db, del.ID,
iAssertStatus(t, s.WebhookDB, d.ID,
database.DeliveryStatusFailed,
)
iAssertResultFailed(t, db, del.ID)
iAssertResultFailed(t, s.WebhookDB, d.ID)
}
// TestDeliverHTTP_CutsStoredResponseAtMaxBodyLog pins the size
@@ -1325,8 +1301,6 @@ func TestDeliverHTTP_CutsStoredResponseAtMaxBodyLog(
))
defer ts.Close()
cfgJSON := iHTTPConfig(ts.URL)
event := iSeedEvent(
t, s.WebhookDB, s.WebhookID, `{"cut":"test"}`,
)
@@ -1340,7 +1314,7 @@ func TestDeliverHTTP_CutsStoredResponseAtMaxBodyLog(
bodyStr := event.Body
task := iTask(
del, event, s.WebhookID, targetID,
"cut-target", cfgJSON, 1, 1, &bodyStr,
"cut-target", ts.URL, 1, 1, &bodyStr,
)
s.Engine.ExportProcessNewTask(context.TODO(), &task)
@@ -1372,7 +1346,7 @@ func TestDeliverHTTP_CutsStoredResponseAtMaxBodyLog(
func iSeedEventAndDelivery(
t *testing.T,
db *gorm.DB,
body, _ string,
body string,
) (database.Event, database.Delivery) {
t.Helper()
@@ -1403,19 +1377,19 @@ func iSeedEventAndDelivery(
func iHTTPTaskAndDelivery(
event database.Event,
del database.Delivery,
name, config string,
name, targetURL string,
maxAttempts, attemptNum int,
) (*delivery.Task, *database.Delivery) {
task := &delivery.Task{
DeliveryID: del.ID,
EventID: event.ID,
WebhookID: event.WebhookID,
TargetID: del.TargetID,
TargetName: name,
TargetType: database.TargetTypeHTTP,
TargetConfig: config,
MaxAttempts: maxAttempts,
AttemptNum: attemptNum,
DeliveryID: del.ID,
EventID: event.ID,
WebhookID: event.WebhookID,
TargetID: del.TargetID,
TargetName: name,
TargetType: database.TargetTypeHTTP,
TargetURL: targetURL,
MaxAttempts: maxAttempts,
AttemptNum: attemptNum,
}
d := &database.Delivery{
@@ -1426,7 +1400,7 @@ func iHTTPTaskAndDelivery(
Target: database.Target{
Name: name,
Type: database.TargetTypeHTTP,
Config: config,
URL: targetURL,
MaxAttempts: maxAttempts,
},
}
@@ -1455,53 +1429,27 @@ func iAssertResultFailed(
assert.NotEmpty(t, result.Error)
}
// --- HTTP request with invalid config ---
// --- HTTP request with no URL ---
func TestDeliverHTTP_InvalidConfig(t *testing.T) {
// TestDeliverHTTP_NoURL proves an http target without a URL, which
// the forms never store, fails its delivery and records why.
func TestDeliverHTTP_NoURL(t *testing.T) {
t.Parallel()
db := testWebhookDB(t)
e := iEngine(t, 1)
event, del := iSeedEventAndDelivery(
t, db, `{"config":"invalid"}`, "",
)
event, del := iSeedEventAndDelivery(t, db, `{"url":"none"}`)
task, d := iHTTPTaskAndDelivery(
event, del, "bad-config", `not-json`, 1, 1,
)
task, d := iHTTPTaskAndDelivery(event, del, "no-url", "", 1, 1)
e.ExportDeliverHTTP(context.TODO(), db, d, task)
iAssertStatus(t, db, del.ID,
database.DeliveryStatusFailed,
)
}
// TestDeliverHTTP_InvalidConfigUnrecordedStaysPending: a delivery is
// failed for an invalid config only once the reason is recorded.
// Unrecorded, it stays pending, where the sweep finds it again.
func TestDeliverHTTP_InvalidConfigUnrecordedStaysPending(t *testing.T) {
t.Parallel()
db := testWebhookDB(t)
e := testEngine(t, 1)
event, del := iSeedEventAndDelivery(
t, db, `{"config":"invalid"}`, "",
)
task, d := iHTTPTaskAndDelivery(
event, del, "bad-config", `not-json`, 1, 1,
)
require.NoError(t, db.Exec("drop table delivery_results").Error)
e.ExportDeliverHTTP(context.TODO(), db, d, task)
iAssertStatus(t, db, del.ID,
database.DeliveryStatusPending,
)
iAssertResultFailed(t, db, del.ID)
}
// --- Notify batching ---
internal/delivery/engine_lifecycle_test.go
+2 -3
View File
@@ -309,13 +309,12 @@ func seedHeldDelivery(
))
t.Cleanup(ts.Close)
cfg := iHTTPConfig(ts.URL)
targetID := uuid.New().String()
iCreateWebhook(t, s.MainDB, s.WebhookID, "hook")
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "held",
database.TargetTypeHTTP, cfg, maxAttempts,
database.TargetTypeHTTP, ts.URL, maxAttempts,
)
event := iSeedEvent(t, s.WebhookDB, s.WebhookID, `{"held":true}`)
@@ -324,7 +323,7 @@ func seedHeldDelivery(
bodyStr := event.Body
return iTask(
d, event, s.WebhookID, targetID, "held", cfg,
d, event, s.WebhookID, targetID, "held", ts.URL,
maxAttempts, attemptNum, &bodyStr,
), arrived
}
internal/delivery/engine_test.go
+57 -209
View File
@@ -65,20 +65,6 @@ func testEngine(
)
}
func newHTTPTargetConfig(url string) string {
cfg := delivery.HTTPTargetConfig{URL: url}
data, err := json.Marshal(cfg)
if err != nil {
panic(
"failed to marshal HTTPTargetConfig: " +
err.Error(),
)
}
return string(data)
}
func seedEvent(
t *testing.T, db *gorm.DB, body string,
) database.Event {
@@ -141,19 +127,19 @@ type httpDeliveryFixture struct {
func buildHTTPFixture(
dlv database.Delivery,
event database.Event,
targetID, name, config string,
targetID, name, targetURL string,
maxAttempts, attemptNum int,
) httpDeliveryFixture {
task := &delivery.Task{
DeliveryID: dlv.ID,
EventID: event.ID,
WebhookID: event.WebhookID,
TargetID: targetID,
TargetName: name,
TargetType: database.TargetTypeHTTP,
TargetConfig: config,
MaxAttempts: maxAttempts,
AttemptNum: attemptNum,
DeliveryID: dlv.ID,
EventID: event.ID,
WebhookID: event.WebhookID,
TargetID: targetID,
TargetName: name,
TargetType: database.TargetTypeHTTP,
TargetURL: targetURL,
MaxAttempts: maxAttempts,
AttemptNum: attemptNum,
}
d := &database.Delivery{
@@ -164,7 +150,7 @@ func buildHTTPFixture(
Target: database.Target{
Name: name,
Type: database.TargetTypeHTTP,
Config: config,
URL: targetURL,
MaxAttempts: maxAttempts,
},
}
@@ -275,9 +261,8 @@ func TestDeliverHTTP_Success(t *testing.T) {
database.DeliveryStatusPending,
)
cfg := newHTTPTargetConfig(ts.URL)
fix := buildHTTPFixture(
dlv, event, targetID, "test-http", cfg, 1, 1,
dlv, event, targetID, "test-http", ts.URL, 1, 1,
)
e.ExportDeliverHTTP(
@@ -321,10 +306,9 @@ func TestDeliverHTTP_Failure(t *testing.T) {
database.DeliveryStatusPending,
)
cfg := newHTTPTargetConfig(ts.URL)
fix := buildHTTPFixture(
dlv, event, targetID,
"test-http-fail", cfg, 1, 1,
"test-http-fail", ts.URL, 1, 1,
)
e.ExportDeliverHTTP(
@@ -351,7 +335,7 @@ func TestDeliverDatabase_ImmediateSuccess(
// The database target archives for real, so the engine needs
// the target in the main database and a data directory.
env := setupArchiveTest(t)
tgt := env.seedDatabaseTarget(t, "")
tgt := env.seedDatabaseTarget(t)
event := seedEvent(t, db, `{"db":"target"}`)
d := seedDatabaseTargetDelivery(t, db, event, tgt)
@@ -452,10 +436,9 @@ func TestDeliverHTTP_WithRetries_Success(t *testing.T) {
database.DeliveryStatusPending,
)
cfg := newHTTPTargetConfig(ts.URL)
fix := buildHTTPFixture(
dlv, event, targetID,
"test-http-retry", cfg, 5, 1,
"test-http-retry", ts.URL, 5, 1,
)
e.ExportDeliverHTTP(
@@ -510,13 +493,12 @@ func TestDeliverHTTP_MaxAttemptsExhausted(t *testing.T) {
database.DeliveryStatusPending,
)
cfg := newHTTPTargetConfig(ts.URL)
sched := &recordingScheduler{}
for attempt := 1; attempt <= maxAttempts; attempt++ {
fix := buildHTTPFixture(
dlv, event, targetID,
"test-http-exhaust", cfg, maxAttempts, attempt,
"test-http-exhaust", ts.URL, maxAttempts, attempt,
)
e.ExportDeliverHTTPWithScheduler(
@@ -574,10 +556,9 @@ func TestDeliverHTTP_SingleAttemptSkipsCircuitBreaker(t *testing.T) {
database.DeliveryStatusPending,
)
cfg := newHTTPTargetConfig(ts.URL)
fix := buildHTTPFixture(
dlv, event, targetID,
"test-http-single", cfg, 1, 1,
"test-http-single", ts.URL, 1, 1,
)
sched := &recordingScheduler{}
@@ -624,10 +605,9 @@ func TestDeliverHTTP_SchedulesRetryOnFailure(
database.DeliveryStatusPending,
)
cfg := newHTTPTargetConfig(ts.URL)
fix := buildHTTPFixture(
dlv, event, targetID,
"test-http-schedule", cfg, 5, 1,
"test-http-schedule", ts.URL, 5, 1,
)
e.ExportDeliverHTTP(
@@ -789,7 +769,7 @@ func seedConcurrencyTasks(
t *testing.T,
db *gorm.DB,
n int,
targetCfg string,
targetURL string,
) ([]database.Delivery, []delivery.Task) {
t.Helper()
@@ -815,21 +795,21 @@ func seedConcurrencyTasks(
Target: database.Target{
Name: fmt.Sprintf("task-%d", i),
Type: database.TargetTypeHTTP,
Config: targetCfg,
URL: targetURL,
MaxAttempts: 1,
},
}
tasks[i].ID = dlv.ID
dtasks[i] = delivery.Task{
DeliveryID: tasks[i].ID,
EventID: tasks[i].EventID,
TargetID: tasks[i].TargetID,
TargetName: tasks[i].Target.Name,
TargetType: tasks[i].Target.Type,
TargetConfig: tasks[i].Target.Config,
MaxAttempts: 1,
AttemptNum: 1,
DeliveryID: tasks[i].ID,
EventID: tasks[i].EventID,
TargetID: tasks[i].TargetID,
TargetName: tasks[i].Target.Name,
TargetType: tasks[i].Target.Type,
TargetURL: tasks[i].Target.URL,
MaxAttempts: 1,
AttemptNum: 1,
}
}
@@ -865,9 +845,8 @@ func TestWorkerPool_BoundedConcurrency(t *testing.T) {
defer ts.Close()
e := testEngine(t, numWorkers)
cfg := newHTTPTargetConfig(ts.URL)
tasks, dtasks := seedConcurrencyTasks(
t, db, numTasks, cfg,
t, db, numTasks, ts.URL,
)
runConcurrencyWorkers(
@@ -948,12 +927,9 @@ func TestDeliverHTTP_CircuitBreakerBlocks(t *testing.T) {
database.DeliveryStatusPending,
)
cfg := newHTTPTargetConfig(
"http://will-not-be-called.invalid",
)
fix := buildHTTPFixture(
dlv, event, targetID,
"test-cb-block", cfg, 5, 1,
"test-cb-block", "http://will-not-be-called.invalid", 5, 1,
)
e.ExportDeliverHTTP(
@@ -1016,9 +992,8 @@ func TestDeliverHTTP_HalfOpenBreakerDelaysQueuedTasks(t *testing.T) {
require.True(t, cb.Allow(), "the probe delivery should go through")
require.Equal(t, delivery.CircuitHalfOpen, cb.State())
cfg := newHTTPTargetConfig(
"http://will-not-be-called.invalid",
)
const targetURL = "http://will-not-be-called.invalid"
sched := &recordingScheduler{}
const queued, passes = 3, 4
@@ -1040,7 +1015,7 @@ func TestDeliverHTTP_HalfOpenBreakerDelaysQueuedTasks(t *testing.T) {
fix := buildHTTPFixture(
row, event, targetID,
"test-cb-half-open", cfg, 5, 1,
"test-cb-half-open", targetURL, 5, 1,
)
e.ExportDeliverHTTPWithScheduler(
@@ -1155,62 +1130,6 @@ func TestStateAndCooldown_ReadsHTTPAndSlackBreakers(t *testing.T) {
assert.Zero(t, cooldown, "half-open")
}
func TestParseHTTPConfig_Valid(t *testing.T) {
t.Parallel()
e := testEngine(t, 1)
cfg, err := e.ExportParseHTTPConfig(
`{"url":"https://example.com/hook",` +
`"headers":{"X-Token":"secret"}}`,
)
require.NoError(t, err)
assert.Equal(t, "https://example.com/hook", cfg.URL)
assert.Equal(t, "secret", cfg.Headers["X-Token"])
}
func TestParseHTTPConfig_Empty(t *testing.T) {
t.Parallel()
e := testEngine(t, 1)
_, err := e.ExportParseHTTPConfig("")
assert.Error(t, err,
"empty config should return error",
)
}
func TestParseHTTPConfig_MissingURL(t *testing.T) {
t.Parallel()
e := testEngine(t, 1)
_, err := e.ExportParseHTTPConfig(
`{"headers":{"X-Token":"secret"}}`,
)
assert.Error(t, err,
"config without URL should return error",
)
}
func TestParseHTTPConfig_Undecodable(t *testing.T) {
t.Parallel()
e := testEngine(t, 1)
_, err := e.ExportParseHTTPConfig(
`{"url":"https://example.com/hook","timeout":"soon"}`,
)
assert.Error(t, err,
"config that does not decode should return error, "+
"even when the part that did names a URL",
)
}
func TestScheduleRetry_SendsToRetryChannel(
t *testing.T,
) {
@@ -1358,7 +1277,7 @@ func TestDoHTTPRequest_ForwardsHeaders(t *testing.T) {
e := testEngine(t, 1)
cfg := &delivery.HTTPTargetConfig{
target := &database.Target{
URL: ts.URL,
Headers: map[string]string{
"X-Target-Auth": "bearer xyz",
@@ -1373,7 +1292,7 @@ func TestDoHTTPRequest_ForwardsHeaders(t *testing.T) {
}
statusCode, _, _, err := e.ExportDoHTTPRequest(
context.TODO(), cfg, event,
context.TODO(), target, event,
)
require.NoError(t, err)
@@ -1415,7 +1334,7 @@ func TestDoHTTPRequest_CutShortResponseIsAnError(t *testing.T) {
_, body, _, err := e.ExportDoHTTPRequest(
context.TODO(),
&delivery.HTTPTargetConfig{URL: ts.URL},
&database.Target{URL: ts.URL},
&database.Event{},
)
@@ -1467,9 +1386,9 @@ func TestApplyRequestHeaders_SendsOneContentType(t *testing.T) {
})
require.NoError(t, err)
cfg := &delivery.HTTPTargetConfig{}
var configured map[string]string
if tc.configured != "" {
cfg.Headers = map[string]string{
configured = map[string]string{
headerContentType: tc.configured,
}
}
@@ -1488,7 +1407,7 @@ func TestApplyRequestHeaders_SendsOneContentType(t *testing.T) {
Headers: string(inbound),
ContentType: tc.event,
},
cfg,
configured,
"webhooker/dev",
)
@@ -1519,7 +1438,7 @@ func TestApplyRequestHeaders_UndecodableInboundForwardsNothing(
&database.Event{
Headers: `{"X-Custom":["value1"],"X-Broken":"not a list"}`,
},
&delivery.HTTPTargetConfig{},
nil,
"webhooker/dev",
)
@@ -1537,7 +1456,7 @@ func TestProcessDelivery_RoutesToCorrectHandler(
// The database target archives for real, so the engine needs
// the target in the main database and a data directory.
env := setupArchiveTest(t)
archive := env.seedDatabaseTarget(t, "")
archive := env.seedDatabaseTarget(t)
tests := []struct {
name string
@@ -1622,51 +1541,6 @@ func TestMaxInlineBodySize_Constant(t *testing.T) {
)
}
func TestParseSlackConfig_Valid(t *testing.T) {
t.Parallel()
e := testEngine(t, 1)
cfg, err := e.ExportParseSlackConfig(
`{"webhookUrl":"https://hooks.slack.com/services/T00/B00/xxx"}`,
)
require.NoError(t, err)
assert.Equal(t,
"https://hooks.slack.com/services/T00/B00/xxx",
cfg.WebhookURL,
)
}
func TestParseSlackConfig_Empty(t *testing.T) {
t.Parallel()
e := testEngine(t, 1)
_, err := e.ExportParseSlackConfig("")
assert.Error(t, err,
"empty config should return error",
)
}
func TestParseSlackConfig_MissingWebhookURL(
t *testing.T,
) {
t.Parallel()
e := testEngine(t, 1)
_, err := e.ExportParseSlackConfig(
`{"other":"field"}`,
)
assert.Error(t, err,
"config without webhook_url should return error",
)
}
func TestFormatSlackMessage_JSONBody(t *testing.T) {
t.Parallel()
@@ -1763,11 +1637,11 @@ func TestFormatSlackMessage_LargeJSONTruncated(
}
// buildSlackDelivery creates a database.Delivery
// with a Slack target config for testing.
// to a Slack target with the given webhook URL for testing.
func buildSlackDelivery(
dlv database.Delivery,
event database.Event,
targetID, name, slackCfg string,
targetID, name, webhookURL string,
) *database.Delivery {
d := &database.Delivery{
EventID: event.ID,
@@ -1777,7 +1651,7 @@ func buildSlackDelivery(
Target: database.Target{
Name: name,
Type: database.TargetTypeSlack,
Config: slackCfg,
URL: webhookURL,
MaxAttempts: 1,
},
}
@@ -1815,11 +1689,10 @@ func TestDeliverSlack_Success(t *testing.T) {
)
defer ts.Close()
e, targetID, slackCfg, event, dlv :=
setupSlackTest(t, db, ts.URL)
e, targetID, event, dlv := setupSlackTest(t, db)
d := buildSlackDelivery(
dlv, event, targetID, "test-slack", slackCfg,
dlv, event, targetID, "test-slack", ts.URL,
)
e.ExportDeliverSlack(context.TODO(), db, d)
@@ -1838,9 +1711,8 @@ func TestDeliverSlack_Success(t *testing.T) {
func setupSlackTest(
t *testing.T,
db *gorm.DB,
serverURL string,
) (
*delivery.Engine, string, string,
*delivery.Engine, string,
database.Event, database.Delivery,
) {
t.Helper()
@@ -1848,13 +1720,6 @@ func setupSlackTest(
e := testEngine(t, 1)
targetID := uuid.New().String()
cfgBytes, err := json.Marshal(
delivery.SlackTargetConfig{
WebhookURL: serverURL,
},
)
require.NoError(t, err)
event := seedEvent(
t, db, `{"action":"test","data":"value"}`,
)
@@ -1864,7 +1729,7 @@ func setupSlackTest(
database.DeliveryStatusPending,
)
return e, targetID, string(cfgBytes), event, dlv
return e, targetID, event, dlv
}
func assertSlackPayload(
@@ -1913,13 +1778,6 @@ func TestDeliverSlack_Failure(t *testing.T) {
e := testEngine(t, 1)
targetID := uuid.New().String()
slackCfg, err := json.Marshal(
delivery.SlackTargetConfig{
WebhookURL: ts.URL,
},
)
require.NoError(t, err)
event := seedEvent(t, db, `{"test":true}`)
dlv := seedDelivery(
@@ -1929,7 +1787,7 @@ func TestDeliverSlack_Failure(t *testing.T) {
d := buildSlackDelivery(
dlv, event, targetID,
"test-slack-fail", string(slackCfg),
"test-slack-fail", ts.URL,
)
e.ExportDeliverSlack(context.TODO(), db, d)
@@ -1943,7 +1801,10 @@ func TestDeliverSlack_Failure(t *testing.T) {
)
}
func TestDeliverSlack_InvalidConfig(t *testing.T) {
// TestDeliverSlack_NoURL proves a Slack target without a URL, which
// the forms never store, fails its delivery rather than reporting it
// delivered.
func TestDeliverSlack_NoURL(t *testing.T) {
t.Parallel()
db := testWebhookDB(t)
@@ -1964,7 +1825,6 @@ func TestDeliverSlack_InvalidConfig(t *testing.T) {
Target: database.Target{
Name: "test-slack-bad",
Type: database.TargetTypeSlack,
Config: `{"not_webhook_url":"missing"}`,
MaxAttempts: 1,
},
}
@@ -2002,13 +1862,6 @@ func TestProcessDelivery_RoutesToSlack(t *testing.T) {
e := testEngine(t, 1)
slackCfg, err := json.Marshal(
delivery.SlackTargetConfig{
WebhookURL: ts.URL,
},
)
require.NoError(t, err)
event := seedEvent(t, db, `{"route":"slack"}`)
dlv := seedDelivery(
@@ -2018,7 +1871,7 @@ func TestProcessDelivery_RoutesToSlack(t *testing.T) {
d := buildSlackDelivery(
dlv, event, dlv.TargetID,
"test-slack-route", string(slackCfg),
"test-slack-route", ts.URL,
)
task := &delivery.Task{
@@ -2133,7 +1986,7 @@ func TestDeliverLog_LogsFullContent(t *testing.T) {
func buildSlackRetryDelivery(
dlv database.Delivery,
event database.Event,
targetID, cfg string,
targetID, webhookURL string,
) *database.Delivery {
d := &database.Delivery{
EventID: event.ID,
@@ -2143,7 +1996,7 @@ func buildSlackRetryDelivery(
Target: database.Target{
Name: "test-slack-retry",
Type: database.TargetTypeSlack,
Config: cfg,
URL: webhookURL,
MaxAttempts: 5,
},
}
@@ -2162,11 +2015,6 @@ func TestDeliverSlack_WithRetries_SchedulesRetry(
e := testEngine(t, 1)
targetID := uuid.New().String()
slackCfg, err := json.Marshal(
delivery.SlackTargetConfig{WebhookURL: ts.URL},
)
require.NoError(t, err)
event := seedEvent(t, db, `{"slack":"retry"}`)
dlv := seedDelivery(
@@ -2175,7 +2023,7 @@ func TestDeliverSlack_WithRetries_SchedulesRetry(
)
d := buildSlackRetryDelivery(
dlv, event, targetID, string(slackCfg),
dlv, event, targetID, ts.URL,
)
task := &delivery.Task{
internal/delivery/event_db_isolation_test.go
+4 -6
View File
@@ -69,7 +69,6 @@ func TestEventDBHoldsNoTargetRows(t *testing.T) {
))
defer ts.Close()
cfg := iHTTPConfig(ts.URL)
targetID := uuid.New().String()
dbPath := s.DBMgr.DBPath(s.WebhookID)
@@ -85,7 +84,7 @@ func TestEventDBHoldsNoTargetRows(t *testing.T) {
)
task := iTask(
d, event, s.WebhookID, targetID,
"leaky-target", cfg, 5, 1, &body,
"leaky-target", ts.URL, 5, 1, &body,
)
s.Engine.ExportProcessNewTask(context.TODO(), &task)
@@ -102,7 +101,7 @@ func TestEventDBHoldsNoTargetRows(t *testing.T) {
// https://git.eeqj.de/sneak/webhooker/issues/107.
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "leaky-target",
database.TargetTypeHTTP, cfg, 5,
database.TargetTypeHTTP, ts.URL, 5,
)
rd := iSeedDelivery(
@@ -111,7 +110,7 @@ func TestEventDBHoldsNoTargetRows(t *testing.T) {
)
rTask := iTask(
rd, event, s.WebhookID, targetID,
"leaky-target", cfg, 5, 2, &body,
"leaky-target", ts.URL, 5, 2, &body,
)
s.Engine.ExportProcessRetryTask(context.TODO(), &rTask)
@@ -138,7 +137,6 @@ func TestEventDBHoldsNoTargetRowsOnFailedDelivery(t *testing.T) {
))
defer ts.Close()
cfg := iHTTPConfig(ts.URL)
targetID := uuid.New().String()
event := iSeedEvent(
@@ -152,7 +150,7 @@ func TestEventDBHoldsNoTargetRowsOnFailedDelivery(t *testing.T) {
)
task := iTask(
d, event, s.WebhookID, targetID,
"leaky-target", cfg, 1, 1, &body,
"leaky-target", ts.URL, 1, 1, &body,
)
s.Engine.ExportProcessNewTask(context.TODO(), &task)
internal/delivery/event_timestamp_test.go
+6 -20
View File
@@ -93,17 +93,6 @@ func (s *tsSink) text(t *testing.T) string {
}
}
func tsSlackConfig(t *testing.T, url string) string {
t.Helper()
data, err := json.Marshal(
delivery.SlackTargetConfig{WebhookURL: url},
)
require.NoError(t, err)
return string(data)
}
// tsSeedEvent writes an event whose CreatedAt is tsEventCreatedAt
// rather than the write time, so an assertion on the rendered
// timestamp cannot pass by accident against "roughly now".
@@ -142,7 +131,7 @@ func tsSeedEvent(
// tsSeedTarget writes the slack target row into the main database.
// The retry path confirms the target still exists before sending.
func tsSeedTarget(
t *testing.T, mainDB *gorm.DB, webhookID, config string,
t *testing.T, mainDB *gorm.DB, webhookID, webhookURL string,
) database.Target {
t.Helper()
@@ -150,7 +139,7 @@ func tsSeedTarget(
WebhookID: webhookID,
Name: "slack-sink",
Type: database.TargetTypeSlack,
Config: config,
URL: webhookURL,
Active: true,
MaxAttempts: 1,
}
@@ -176,7 +165,7 @@ func tsTask(
TargetID: target.ID,
TargetName: target.Name,
TargetType: database.TargetTypeSlack,
TargetConfig: target.Config,
TargetURL: target.URL,
MaxAttempts: 1,
Method: event.Method,
Headers: event.Headers,
@@ -226,8 +215,7 @@ func (c tsCase) run(t *testing.T) (iSetup, database.Delivery, string) {
s := newISetup(t)
sink := newTSSink(t)
cfg := tsSlackConfig(t, sink.URL)
target := tsSeedTarget(t, s.MainDB, s.WebhookID, cfg)
target := tsSeedTarget(t, s.MainDB, s.WebhookID, sink.URL)
event := tsSeedEvent(t, s.WebhookDB, s.WebhookID)
d := iSeedDelivery(
@@ -338,8 +326,7 @@ func TestFormatSlackMessageOverTaskReconstructedEvent(
s := newISetup(t)
cfg := tsSlackConfig(t, tsUndeliverableHook)
target := tsSeedTarget(t, s.MainDB, s.WebhookID, cfg)
target := tsSeedTarget(t, s.MainDB, s.WebhookID, tsUndeliverableHook)
event := tsSeedEvent(t, s.WebhookDB, s.WebhookID)
d := iSeedDelivery(
@@ -410,8 +397,7 @@ func TestEventReconstructionSurvivesAReapedRow(t *testing.T) {
s := newISetup(t)
cfg := tsSlackConfig(t, tsUndeliverableHook)
target := tsSeedTarget(t, s.MainDB, s.WebhookID, cfg)
target := tsSeedTarget(t, s.MainDB, s.WebhookID, tsUndeliverableHook)
event := tsSeedEvent(t, s.WebhookDB, s.WebhookID)
d := iSeedDelivery(
internal/delivery/export_test.go
+6 -20
View File
@@ -77,10 +77,10 @@ func ExportIsForwardableHeader(name string) bool {
func ExportApplyRequestHeaders(
req *http.Request,
event *database.Event,
cfg *HTTPTargetConfig,
configured map[string]string,
userAgent string,
) []string {
return applyRequestHeaders(req, event, cfg, userAgent)
return applyRequestHeaders(req, event, configured, userAgent)
}
// ExportTruncate exposes truncate for testing.
@@ -216,37 +216,23 @@ func (e *Engine) ExportSetSlackCircuitBreaker(
e.slackTarget.circuitBreakers.Store(targetID, cb)
}
// ExportParseHTTPConfig exposes parseHTTPConfig.
func (e *Engine) ExportParseHTTPConfig(
configJSON string,
) (*HTTPTargetConfig, error) {
return parseHTTPConfig(configJSON)
}
// ExportParseSlackConfig exposes parseSlackConfig.
func (e *Engine) ExportParseSlackConfig(
configJSON string,
) (*SlackTargetConfig, error) {
return parseSlackConfig(configJSON)
}
// ExportDoHTTPRequest exposes the http target's
// doHTTPRequest.
func (e *Engine) ExportDoHTTPRequest(
ctx context.Context,
cfg *HTTPTargetConfig,
target *database.Target,
event *database.Event,
) (int, string, int64, error) {
return e.httpTarget.doHTTPRequest(ctx, cfg, event)
return e.httpTarget.doHTTPRequest(ctx, target, event)
}
// ExportClientForRequest exposes the http target's
// clientForRequest.
func (e *Engine) ExportClientForRequest(
cfg *HTTPTargetConfig,
timeoutSeconds int,
originScoped []string,
) *http.Client {
return e.httpTarget.clientForRequest(cfg, originScoped)
return e.httpTarget.clientForRequest(timeoutSeconds, originScoped)
}
// ErrExportTooManyRedirects exposes the sentinel the redirect
internal/delivery/metrics_test.go
+12 -15
View File
@@ -192,7 +192,7 @@ func mDeliverOK(t *testing.T, s iSetup) {
body := event.Body
task := iTask(
d, event, s.WebhookID, targetID,
"metrics-ok", iHTTPConfig(ts.URL), 3, 1, &body,
"metrics-ok", ts.URL, 3, 1, &body,
)
s.Engine.ExportProcessNewTask(context.TODO(), &task)
@@ -226,18 +226,17 @@ func mExhaustRetries(t *testing.T, s iSetup) {
)
body := event.Body
cfg := iHTTPConfig(ts.URL)
// The retry below is only run if its target still exists; see
// https://git.eeqj.de/sneak/webhooker/issues/107.
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "metrics-fail",
database.TargetTypeHTTP, cfg, 2,
database.TargetTypeHTTP, ts.URL, 2,
)
first := iTask(
d, event, s.WebhookID, targetID,
"metrics-fail", cfg, 2, 1, &body,
"metrics-fail", ts.URL, 2, 1, &body,
)
s.Engine.ExportProcessNewTask(context.TODO(), &first)
@@ -251,7 +250,7 @@ func mExhaustRetries(t *testing.T, s iSetup) {
// deterministic and off the wall clock.
second := iTask(
d, event, s.WebhookID, targetID,
"metrics-fail", cfg, 2, 2, &body,
"metrics-fail", ts.URL, 2, 2, &body,
)
s.Engine.ExportProcessRetryTask(
@@ -289,7 +288,6 @@ func TestDeliveryMetrics_CircuitBreakerGauge(t *testing.T) {
)
body := event.Body
cfg := iHTTPConfig(ts.URL)
// An attempt budget above the failure threshold, so the breaker
// rather than the budget is what stops the delivery.
@@ -299,12 +297,12 @@ func TestDeliveryMetrics_CircuitBreakerGauge(t *testing.T) {
// see https://git.eeqj.de/sneak/webhooker/issues/107.
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "metrics-trip",
database.TargetTypeHTTP, cfg, maxAttempts,
database.TargetTypeHTTP, ts.URL, maxAttempts,
)
first := iTask(
d, event, s.WebhookID, targetID,
"metrics-trip", cfg, maxAttempts, 1, &body,
"metrics-trip", ts.URL, maxAttempts, 1, &body,
)
s.Engine.ExportProcessNewTask(context.TODO(), &first)
@@ -316,7 +314,7 @@ func TestDeliveryMetrics_CircuitBreakerGauge(t *testing.T) {
ExportDefaultFailureThreshold; attempt++ {
task := iTask(
d, event, s.WebhookID, targetID,
"metrics-trip", cfg, maxAttempts, attempt, &body,
"metrics-trip", ts.URL, maxAttempts, attempt, &body,
)
s.Engine.ExportProcessRetryTask(
@@ -363,17 +361,16 @@ func TestDeliveryMetrics_BreakerBlockedIsNotAnAttempt(
)
body := event.Body
cfg := iHTTPConfig(ts.URL)
maxAttempts := delivery.ExportDefaultFailureThreshold + 5
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "metrics-blocked",
database.TargetTypeHTTP, cfg, maxAttempts,
database.TargetTypeHTTP, ts.URL, maxAttempts,
)
first := iTask(
d, event, s.WebhookID, targetID,
"metrics-blocked", cfg, maxAttempts, 1, &body,
"metrics-blocked", ts.URL, maxAttempts, 1, &body,
)
s.Engine.ExportProcessNewTask(context.TODO(), &first)
@@ -382,7 +379,7 @@ func TestDeliveryMetrics_BreakerBlockedIsNotAnAttempt(
ExportDefaultFailureThreshold; attempt++ {
task := iTask(
d, event, s.WebhookID, targetID,
"metrics-blocked", cfg, maxAttempts, attempt, &body,
"metrics-blocked", ts.URL, maxAttempts, attempt, &body,
)
s.Engine.ExportProcessRetryTask(context.TODO(), &task)
@@ -405,7 +402,7 @@ func TestDeliveryMetrics_BreakerBlockedIsNotAnAttempt(
blocked := iTask(
d, event, s.WebhookID, targetID,
"metrics-blocked", cfg, maxAttempts,
"metrics-blocked", ts.URL, maxAttempts,
delivery.ExportDefaultFailureThreshold+1, &body,
)
@@ -474,7 +471,7 @@ func TestDeliveryMetrics_QueueDepthGauges(t *testing.T) {
iCreateTarget(t, s.MainDB, targetID, s.WebhookID,
"queue-depth-target", database.TargetTypeHTTP,
iHTTPConfig("https://example.com/hook"), 3,
"https://example.com/hook", 3,
)
event := iSeedEvent(
internal/delivery/queue_depth_gormlog_test.go
+1 -1
View File
@@ -102,7 +102,7 @@ func TestQueueDepthSample_LogsNoBoundValue(t *testing.T) {
iCreateTarget(t, mainDB.DB(), targetID, webhookID,
"queue-depth-gormlog-target", database.TargetTypeHTTP,
iHTTPConfig("https://example.com/hook"), 3,
"https://example.com/hook", 3,
)
event := iSeedEvent(
internal/delivery/recovery_durability_test.go
+2 -2
View File
@@ -357,7 +357,7 @@ func TestFailedResultWriteLeavesDeliveryRecoverable(
Target: database.Target{
Name: "unwritable",
Type: database.TargetTypeHTTP,
Config: iHTTPConfig(ts.URL),
URL: ts.URL,
MaxAttempts: 1,
},
}
@@ -448,7 +448,7 @@ func TestFailedResultWriteWithRetriesLeavesDeliveryRecoverable(
Target: database.Target{
Name: "unwritable",
Type: database.TargetTypeHTTP,
Config: iHTTPConfig(ts.URL),
URL: ts.URL,
MaxAttempts: 3,
},
}
internal/delivery/recovery_test.go
+50 -1
View File
@@ -56,7 +56,7 @@ func recoverySetup(t *testing.T) (iSetup, string) {
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "recovery",
database.TargetTypeHTTP,
iHTTPConfig("http://example.com/hook"), 20,
"http://example.com/hook", 20,
)
return s, targetID
@@ -205,6 +205,55 @@ func TestRecovery_PendingOlderThanTheAgeBound(t *testing.T) {
}
}
// TestRecovery_TaskCarriesEveryTargetSetting: the task the recovery
// routine queues carries every one of the target's settings, all six
// set here.
// They are copied into the task one field at a time, and a field left
// out is sent empty without any error.
func TestRecovery_TaskCarriesEveryTargetSetting(t *testing.T) {
t.Parallel()
for _, run := range recoveryRuns() {
t.Run(run.name, func(t *testing.T) {
t.Parallel()
s := newISetup(t)
iCreateWebhook(t, s.MainDB, s.WebhookID, "recovery")
target := database.Target{
WebhookID: s.WebhookID,
Name: "recovery",
Type: database.TargetTypeHTTP,
Active: true,
URL: "http://example.com/hook",
Headers: map[string]string{headerAuthorization: "Bearer t"},
TimeoutSeconds: 7,
ForwardQuery: true,
ArchiveExpiry: time.Hour,
ArchiveRotation: "daily",
MaxAttempts: 1,
}
require.NoError(t, s.MainDB.Create(&target).Error)
id := seedRecoveryDelivery(
t, s, target.ID, database.DeliveryStatusPending, 0, false,
)
rAgePending(t, s.WebhookDB, id)
s.Engine.ExportRecoverDeliveries(context.Background(), run.atStart)
task := requireQueued(t, s.Engine.ExportDeliveryCh())
assert.Equal(t, target.ID, task.TargetID)
assert.Equal(t, target.URL, task.TargetURL)
assert.Equal(t, target.Headers, task.TargetHeaders)
assert.Equal(t, target.TimeoutSeconds, task.TimeoutSeconds)
assert.Equal(t, target.ForwardQuery, task.ForwardQuery)
assert.Equal(t, target.ArchiveExpiry, task.ArchiveExpiry)
assert.Equal(t, target.ArchiveRotation, task.ArchiveRotation)
})
}
}
// TestRecovery_PendingYoungerThanTheAgeBound: a pending delivery touched
// a moment ago, as one a stop cut off is, is sent at start, which is how
// it reaches its target on the next start
internal/delivery/redirect_test.go
+9 -17
View File
@@ -85,19 +85,14 @@ func deliverWithProbeHeaders(
database.DeliveryStatusPending,
)
cfg, err := json.Marshal(delivery.HTTPTargetConfig{
URL: targetURL,
Headers: map[string]string{
probeHeaderName: probeHeaderValue,
},
})
require.NoError(t, err)
body := event.Body
task := iTask(
d, event, s.WebhookID, targetID,
"redirect-target", string(cfg), 1, 1, &body,
"redirect-target", targetURL, 1, 1, &body,
)
task.TargetHeaders = map[string]string{
probeHeaderName: probeHeaderValue,
}
s.Engine.ExportProcessNewTask(context.TODO(), &task)
@@ -309,8 +304,7 @@ func TestRedirectPolicy_StopsAtHopCap(t *testing.T) {
)
client := engine.ExportClientForRequest(
&delivery.HTTPTargetConfig{URL: srv.URL},
[]string{probeHeaderName},
0, []string{probeHeaderName},
)
require.NotNil(t, client.CheckRedirect)
@@ -350,7 +344,7 @@ func TestApplyRequestHeaders_ReportsOriginScopedNames(t *testing.T) {
inbound, err := json.Marshal(map[string][]string{
inboundHeaderName: {inboundHeaderValue},
"Content-Type": {testContentType},
"User-Agent": {"curl/8.7.1"},
headerUserAgent: {"curl/8.7.1"},
"Host": {"inbound.example.com"},
})
require.NoError(t, err)
@@ -369,11 +363,9 @@ func TestApplyRequestHeaders_ReportsOriginScopedNames(t *testing.T) {
Headers: string(inbound),
ContentType: testContentType,
},
&delivery.HTTPTargetConfig{
Headers: map[string]string{
probeHeaderName: probeHeaderValue,
"Content-Type": testContentType,
},
map[string]string{
probeHeaderName: probeHeaderValue,
"Content-Type": testContentType,
},
"webhooker/dev",
)
internal/delivery/target_config_edit.go
-137
View File
@@ -1,137 +0,0 @@
package delivery
import (
"encoding/json"
"errors"
"fmt"
"sneak.berlin/go/webhooker/internal/archive"
"sneak.berlin/go/webhooker/internal/database"
)
// errUnknownTargetTypeForEdit is returned when a stored target has a
// type the edit form has no field set for.
var errUnknownTargetTypeForEdit = errors.New(
"unknown target type",
)
// TargetConfigForm is the UNMASKED projection of a target's stored
// configuration, for pre-filling the target edit form.
//
// It is the deliberate exception to the rule TargetView enforces
// everywhere else: TargetView exists so that no template can render
// a target's stored blob, because a destination URL's path segments
// and a header value are both routinely the credential. An operator
// cannot correct a value they cannot see, so the edit form — and
// only the edit form — is shown the full value.
//
// Everything that keeps that exception narrow lives at the call
// site: the route is behind RequireAuth and the webhook's ownership
// check, and its group sets NoCache so the rendered secret is not
// written to a shared cache. Do not reach for this type from any
// other page.
type TargetConfigForm struct {
// URL is the destination for an HTTP target and the webhook
// URL for a Slack target.
URL string
// Headers is the HTTP target's configured headers in the
// textarea representation, one "Name: value" per line.
Headers string
// Timeout is the HTTP target's per-request timeout in seconds,
// empty when unset.
Timeout string
// ForwardQuery is the HTTP target's setting that passes each
// event's query string on to it.
ForwardQuery bool
// Expiry is the database (archive) target's row expiry.
Expiry string
// Rotation is the database (archive) target's rotation.
Rotation string
}
// NewTargetConfigForm parses a target's stored configuration into
// the edit form's fields.
//
// A configuration that does not parse is an error rather than a
// zero-valued form that silently looks like a target with no
// settings. The caller shows the operator that the stored value
// could not be read, so that saving the form is understood as
// replacing it rather than preserving it.
func NewTargetConfigForm(
t *database.Target,
) (TargetConfigForm, error) {
switch t.Type {
case database.TargetTypeHTTP:
cfg, err := parseHTTPConfig(t.Config)
if err != nil {
return TargetConfigForm{}, err
}
return TargetConfigForm{
URL: cfg.URL,
Headers: FormatTargetHeaders(cfg.Headers),
Timeout: FormatTargetTimeout(cfg.Timeout),
ForwardQuery: cfg.ForwardQuery,
}, nil
case database.TargetTypeSlack:
cfg, err := parseSlackConfig(t.Config)
if err != nil {
return TargetConfigForm{}, err
}
return TargetConfigForm{URL: cfg.WebhookURL}, nil
case database.TargetTypeDatabase:
return databaseConfigForm(t.Config)
case database.TargetTypeLog:
// The log target takes no configuration.
return TargetConfigForm{}, nil
default:
return TargetConfigForm{}, fmt.Errorf(
"%w: %q", errUnknownTargetTypeForEdit, t.Type,
)
}
}
// databaseConfigForm parses an archive target's optional expiry and
// rotation. An absent, empty or never expiry yields an empty expiry,
// on which the edit form starts at never; saving it unchanged stores
// never, which means the same as an empty expiry. An absent rotation
// is empty too, and the form starts at none. An expiry that is set
// but not a valid duration, or a rotation that is not one of the
// four, is an error, not a blank field.
func databaseConfigForm(
configJSON string,
) (TargetConfigForm, error) {
if configJSON == "" {
return TargetConfigForm{}, nil
}
var cfg archive.DatabaseTargetConfig
err := json.Unmarshal([]byte(configJSON), &cfg)
if err != nil {
return TargetConfigForm{}, fmt.Errorf(
"parsing config JSON: %w", err,
)
}
err = archive.ValidateRotation(cfg.Rotation)
if err != nil {
return TargetConfigForm{}, err
}
form := TargetConfigForm{Rotation: cfg.Rotation}
if cfg.Expiry == "" || cfg.Expiry == archive.ExpiryNever {
return form, nil
}
err = archive.ValidateExpiry(cfg.Expiry)
if err != nil {
return TargetConfigForm{}, err
}
form.Expiry = cfg.Expiry
return form, nil
}
internal/delivery/target_config_view.go
+33 -74
View File
@@ -9,16 +9,17 @@ import (
"sneak.berlin/go/webhooker/internal/database"
)
// configUnavailable is what a target's configuration renders
// as when it is absent, of an unknown type, or does not
// parse. The stored blob is never shown as a fallback: it can
// hold a credential (a Slack incoming webhook URL is a bearer
// token) and a UI that prints it leaks that credential into
// browser history, screenshots and screen shares.
// configUnavailable is what a target's settings render as when
// the target is of an unknown type, and what its URL renders as
// when it does not parse into a scheme and host. The stored URL is
// never shown as a fallback: it can be a credential (a Slack
// incoming webhook URL is a bearer token) and a UI that prints it
// leaks that credential into browser history, screenshots and
// screen shares.
const configUnavailable = "(unavailable)"
// ConfigField is one labelled, display-safe value derived
// from a target's stored configuration.
// from a target's settings.
type ConfigField struct {
Label string
Value string
@@ -32,9 +33,9 @@ type ConfigField struct {
const deletedNameSuffix = " (deleted)"
// TargetView is the display-safe projection of a target for
// the UI. It deliberately has no raw configuration field, so
// no template — present or future — can render the stored
// blob.
// the UI. It deliberately has no URL or header field, so no
// template — present or future — can render the stored
// credential.
type TargetView struct {
ID string
Name string
@@ -61,8 +62,7 @@ func (v TargetView) DisplayName() string {
}
// NewTargetViews projects targets for rendering, replacing
// each stored configuration blob with named, display-safe
// fields.
// each target's settings with named, display-safe fields.
//
// A soft-deleted row projects exactly as a live one does, minus
// the deleted marker on its name: masking is a property of the
@@ -91,8 +91,8 @@ func NewTargetViews(
}
// targetConfigFields returns the display-safe fields for a
// target's configuration. Anything it cannot parse becomes
// the neutral placeholder.
// target's settings. A target of an unknown type gets the
// neutral placeholder.
func targetConfigFields(
t *database.Target,
) []ConfigField {
@@ -102,36 +102,25 @@ func targetConfigFields(
case database.TargetTypeHTTP:
return httpConfigFields(t)
case database.TargetTypeDatabase:
return databaseConfigFields(t.Config)
return databaseConfigFields(t)
case database.TargetTypeLog:
// The log target takes no configuration.
// The log target takes no settings.
return nil
default:
return unavailableConfigFields()
return []ConfigField{{
Label: "Configuration",
Value: configUnavailable,
}}
}
}
// unavailableConfigFields is the neutral placeholder shown
// for a configuration that could not be presented.
func unavailableConfigFields() []ConfigField {
return []ConfigField{{
Label: "Configuration",
Value: configUnavailable,
}}
}
// slackConfigFields describes a Slack target: its masked
// webhook URL and its number of delivery attempts. Only the
// masked URL is shown; the full URL is the credential.
func slackConfigFields(t *database.Target) []ConfigField {
cfg, err := parseSlackConfig(t.Config)
if err != nil {
return unavailableConfigFields()
}
return []ConfigField{{
Label: "Webhook URL",
Value: cfg.MaskedWebhookURL(),
Value: MaskURL(t.URL),
}, maxAttemptsField(t)}
}
@@ -146,33 +135,28 @@ func slackConfigFields(t *database.Target) []ConfigField {
// whose path segments are the credential, and the field takes
// an arbitrary URL, so no segment can be assumed non-secret.
func httpConfigFields(t *database.Target) []ConfigField {
cfg, err := parseHTTPConfig(t.Config)
if err != nil {
return unavailableConfigFields()
}
fields := []ConfigField{{
Label: "Destination URL",
Value: MaskURL(cfg.URL),
Value: MaskURL(t.URL),
}}
if cfg.Timeout > 0 {
if t.TimeoutSeconds > 0 {
fields = append(fields, ConfigField{
Label: "Timeout",
Value: strconv.Itoa(cfg.Timeout) + "s",
Value: strconv.Itoa(t.TimeoutSeconds) + "s",
})
}
if len(cfg.Headers) > 0 {
if len(t.Headers) > 0 {
fields = append(fields, ConfigField{
Label: "Headers",
Value: fmt.Sprintf(
"%d configured", len(cfg.Headers),
"%d configured", len(t.Headers),
),
})
}
if cfg.ForwardQuery {
if t.ForwardQuery {
fields = append(fields, ConfigField{
Label: "Query string",
Value: "passed on to this target",
@@ -185,8 +169,7 @@ func httpConfigFields(t *database.Target) []ConfigField {
}
// maxAttemptsField describes a target's number of delivery
// attempts, which lives on the target row rather than in its
// configuration blob.
// attempts.
func maxAttemptsField(t *database.Target) ConfigField {
return ConfigField{
Label: "Delivery attempts",
@@ -196,24 +179,11 @@ func maxAttemptsField(t *database.Target) ConfigField {
// databaseConfigFields describes an archive target by its
// expiry in plain units, such as "30 days", or "never" when
// the archive is kept forever, and by its rotation. An expiry
// that is set but not a valid duration, or a rotation that is
// not one of the four, is reported as unavailable rather than
// echoed back.
func databaseConfigFields(configJSON string) []ConfigField {
expiry, err := archive.ParseExpiry(configJSON)
if err != nil {
return unavailableConfigFields()
}
rotation, err := archive.ParseRotation(configJSON)
if err != nil {
return unavailableConfigFields()
}
// the archive is kept forever, and by its rotation.
func databaseConfigFields(t *database.Target) []ConfigField {
value := archive.ExpiryNever
if expiry > 0 {
value = plainDuration(expiry)
if t.ArchiveExpiry > 0 {
value = plainDuration(t.ArchiveExpiry)
}
return []ConfigField{{
@@ -221,7 +191,7 @@ func databaseConfigFields(configJSON string) []ConfigField {
Value: value,
}, {
Label: "Archive rotation",
Value: rotation,
Value: t.ArchiveRotation,
}}
}
@@ -257,14 +227,3 @@ func plainDuration(d time.Duration) string {
return d.String()
}
// MaskedWebhookURL returns the Slack webhook URL reduced to
// its scheme and host, with the path, query and any userinfo
// elided. The path segments are the credential, so none of
// them is shown: the field accepts an arbitrary URL, so no
// segment can be assumed non-secret. A URL that does not
// parse into a scheme and host yields the neutral
// placeholder, never the raw string.
func (c *SlackTargetConfig) MaskedWebhookURL() string {
return MaskURL(c.WebhookURL)
}
internal/delivery/target_config_view_test.go
+58 -122
View File
@@ -35,18 +35,17 @@ const (
viewMaxAttempts = "Delivery attempts"
)
// The archive rotations a database target's settings can name, and
// the config of one whose rotation is not one of the four.
// The archive rotations a database target's settings can name.
const (
rotationNone = "none"
rotationMonthly = "monthly"
rotationDaily = "daily"
rotationHourly = "hourly"
weeklyConfig = `{"rotation":"weekly"}`
)
func TestMaskedWebhookURL(t *testing.T) {
// TestNewTargetViews_SlackMasksWebhookURL shows what a Slack target's
// URL renders as on the target list.
func TestNewTargetViews_SlackMasksWebhookURL(t *testing.T) {
t.Parallel()
tests := map[string]struct {
@@ -90,28 +89,24 @@ func TestMaskedWebhookURL(t *testing.T) {
t.Run(name, func(t *testing.T) {
t.Parallel()
cfg := &delivery.SlackTargetConfig{
WebhookURL: tc.url,
}
target := slackTarget()
target.URL = tc.url
assert.Equal(
t, tc.want, cfg.MaskedWebhookURL(),
t, tc.want,
fieldMap(viewFor(t, target).Config)["Webhook URL"],
)
})
}
}
// TestMaskedWebhookURL_NeverLeaksPath is the direct
// TestNewTargetViews_SlackNeverLeaksPath is the direct
// expression of the rule: whatever the input, the masked
// value never contains a path segment of it.
func TestMaskedWebhookURL_NeverLeaksPath(t *testing.T) {
func TestNewTargetViews_SlackNeverLeaksPath(t *testing.T) {
t.Parallel()
cfg := &delivery.SlackTargetConfig{
WebhookURL: slackWebhookURL,
}
masked := cfg.MaskedWebhookURL()
masked := fieldMap(viewFor(t, slackTarget()).Config)["Webhook URL"]
assert.NotContains(t, masked, "T00000000")
assert.NotContains(t, masked, "B00000000")
@@ -177,11 +172,10 @@ func TestNewTargetViews_DeletedTarget(t *testing.T) {
// share.
func slackTarget() database.Target {
return database.Target{
Name: slackTargetName,
Type: database.TargetTypeSlack,
Active: true,
Config: `{"webhookUrl":"` +
slackWebhookURL + `"}`,
Name: slackTargetName,
Type: database.TargetTypeSlack,
Active: true,
URL: slackWebhookURL,
MaxAttempts: 1,
}
}
@@ -233,12 +227,12 @@ func TestNewTargetViews_HTTP(t *testing.T) {
t.Parallel()
view := viewFor(t, database.Target{
Type: database.TargetTypeHTTP,
Config: `{"url":"` + viewExampleHook + `",` +
`"timeout":30,` +
`"headers":{"Authorization":"Bearer sekrit"},` +
`"forwardQuery":true}`,
MaxAttempts: 5,
Type: database.TargetTypeHTTP,
URL: viewExampleHook,
TimeoutSeconds: 30,
Headers: map[string]string{headerAuthorization: "Bearer sekrit"},
ForwardQuery: true,
MaxAttempts: 5,
})
fields := fieldMap(view.Config)
@@ -268,7 +262,7 @@ func TestNewTargetViews_HTTPSingleAttempt(t *testing.T) {
view := viewFor(t, database.Target{
Type: database.TargetTypeHTTP,
Config: `{"url":"` + viewExampleHook + `"}`,
URL: viewExampleHook,
MaxAttempts: 1,
})
@@ -291,7 +285,7 @@ func TestNewTargetViews_HTTPMasksDestinationURL(t *testing.T) {
view := viewFor(t, database.Target{
Type: database.TargetTypeHTTP,
Config: `{"url":"` + slackWebhookURL + `"}`,
URL: slackWebhookURL,
MaxAttempts: 1,
})
@@ -315,25 +309,20 @@ func TestNewTargetViews_Database(t *testing.T) {
t.Parallel()
tests := map[string]struct {
config string
expiry time.Duration
want string
}{
"empty config": {config: "", want: viewExpiryNever},
"empty expiry": {config: `{}`, want: viewExpiryNever},
"never literal": {
config: `{"expiry":"` + viewExpiryNever + `"}`,
want: viewExpiryNever,
},
"1h": {config: `{"expiry":"1h"}`, want: "1 hour"},
"12h": {config: `{"expiry":"12h"}`, want: "12 hours"},
"24h": {config: `{"expiry":"24h"}`, want: "1 day"},
"720h": {config: `{"expiry":"720h"}`, want: "30 days"},
"2160h": {config: `{"expiry":"2160h"}`, want: "90 days"},
"8760h": {config: `{"expiry":"8760h"}`, want: "365 days"},
"36h": {config: `{"expiry":"36h"}`, want: "36 hours"},
"1h30m": {config: `{"expiry":"1h30m"}`, want: "90 minutes"},
"45s": {config: `{"expiry":"45s"}`, want: "45 seconds"},
"1.5s": {config: `{"expiry":"1.5s"}`, want: "1.5s"},
"never": {expiry: 0, want: viewExpiryNever},
"1h": {expiry: time.Hour, want: "1 hour"},
"12h": {expiry: 12 * time.Hour, want: "12 hours"},
"24h": {expiry: 24 * time.Hour, want: "1 day"},
"720h": {expiry: 720 * time.Hour, want: "30 days"},
"2160h": {expiry: 2160 * time.Hour, want: "90 days"},
"8760h": {expiry: 8760 * time.Hour, want: "365 days"},
"36h": {expiry: 36 * time.Hour, want: "36 hours"},
"1h30m": {expiry: 90 * time.Minute, want: "90 minutes"},
"45s": {expiry: 45 * time.Second, want: "45 seconds"},
"1.5s": {expiry: 1500 * time.Millisecond, want: "1.5s"},
}
for name, tc := range tests {
@@ -341,8 +330,9 @@ func TestNewTargetViews_Database(t *testing.T) {
t.Parallel()
view := viewFor(t, database.Target{
Type: database.TargetTypeDatabase,
Config: tc.config,
Type: database.TargetTypeDatabase,
ArchiveExpiry: tc.expiry,
ArchiveRotation: rotationNone,
})
assert.Equal(
@@ -358,33 +348,23 @@ func TestNewTargetViews_Database(t *testing.T) {
}
// TestNewTargetViews_DatabaseRotation proves the target list shows a
// database target's rotation, none when it has none stored.
// database target's rotation.
func TestNewTargetViews_DatabaseRotation(t *testing.T) {
t.Parallel()
// Each stored config, and the rotation the list shows for it.
tests := map[string]string{
"": rotationNone,
`{"rotation":""}`: rotationNone,
}
for _, rotation := range []string{
rotationNone, rotationMonthly, rotationDaily, rotationHourly,
} {
tests[`{"rotation":"`+rotation+`"}`] = rotation
}
for config, want := range tests {
t.Run(config, func(t *testing.T) {
t.Run(rotation, func(t *testing.T) {
t.Parallel()
view := viewFor(t, database.Target{
Type: database.TargetTypeDatabase,
Config: config,
Type: database.TargetTypeDatabase,
ArchiveRotation: rotation,
})
assert.Equal(
t, want, fieldMap(view.Config)["Archive rotation"],
t, rotation, fieldMap(view.Config)["Archive rotation"],
)
})
}
@@ -394,72 +374,28 @@ func TestNewTargetViews_Log(t *testing.T) {
t.Parallel()
view := viewFor(t, database.Target{
Type: database.TargetTypeLog,
Config: "",
Type: database.TargetTypeLog,
})
assert.Empty(t, view.Config)
}
// TestNewTargetViews_Unpresentable proves that no config the
// view cannot present falls back to the stored blob.
func TestNewTargetViews_Unpresentable(t *testing.T) {
// TestNewTargetViews_UnknownType proves that a target of a type the
// view does not know shows the neutral placeholder, and never its
// URL.
func TestNewTargetViews_UnknownType(t *testing.T) {
t.Parallel()
const blob = `{"webhookUrl":"https://hooks.slack.com` +
slackSecretPath + `"`
view := viewFor(t, database.Target{
Type: database.TargetType("carrier-pigeon"),
URL: slackWebhookURL,
})
tests := map[string]database.Target{
"unknown target type": {
Type: database.TargetType("carrier-pigeon"),
Config: blob,
assert.Equal(
t,
map[string]string{
"Configuration": viewUnavailable,
},
"unparseable json": {
Type: database.TargetTypeSlack,
Config: blob,
MaxAttempts: 1,
},
"empty slack config": {
Type: database.TargetTypeSlack,
MaxAttempts: 1,
},
"slack config without url": {
Type: database.TargetTypeSlack,
Config: `{}`,
MaxAttempts: 1,
},
"unparseable http json": {
Type: database.TargetTypeHTTP,
Config: `{"url":`,
MaxAttempts: 1,
},
"unparseable archive json": {
Type: database.TargetTypeDatabase,
Config: `{"expiry":`,
},
"invalid archive expiry": {
Type: database.TargetTypeDatabase,
Config: `{"expiry":"a fortnight"}`,
},
"invalid archive rotation": {
Type: database.TargetTypeDatabase,
Config: weeklyConfig,
},
}
for name, target := range tests {
t.Run(name, func(t *testing.T) {
t.Parallel()
view := viewFor(t, target)
assert.Equal(
t,
map[string]string{
"Configuration": viewUnavailable,
},
fieldMap(view.Config),
)
})
}
fieldMap(view.Config),
)
}
internal/delivery/target_database_test.go
+108 -19
View File
@@ -1,6 +1,7 @@
package delivery_test
import (
"context"
"database/sql"
"fmt"
"log/slog"
@@ -20,6 +21,7 @@ import (
"sneak.berlin/go/webhooker/internal/archive"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/database/databasetest"
"sneak.berlin/go/webhooker/internal/db/migrations"
"sneak.berlin/go/webhooker/internal/delivery"
"sneak.berlin/go/webhooker/internal/gormlog"
)
@@ -78,10 +80,11 @@ func (env *archiveEnv) archivePath(tgt *database.Target) string {
)
}
// seedDatabaseTarget creates a webhook with one database target
// carrying the given target config JSON, and returns the target.
// seedDatabaseTarget creates a webhook with one database target,
// which keeps its archive forever in one file, and returns the
// target.
func (env *archiveEnv) seedDatabaseTarget(
t *testing.T, configJSON string,
t *testing.T,
) *database.Target {
t.Helper()
@@ -97,12 +100,12 @@ func (env *archiveEnv) seedDatabaseTarget(
)
tgt := &database.Target{
WebhookID: wh.ID,
Name: "Archive",
Type: database.TargetTypeDatabase,
Active: true,
Config: configJSON,
MaxAttempts: 1,
WebhookID: wh.ID,
Name: "Archive",
Type: database.TargetTypeDatabase,
Active: true,
ArchiveRotation: "none",
MaxAttempts: 1,
}
require.NoError(
t,
@@ -136,9 +139,10 @@ func seedDatabaseTargetDelivery(
Status: database.DeliveryStatusPending,
Event: event,
Target: database.Target{
Name: tgt.Name,
Type: database.TargetTypeDatabase,
Config: tgt.Config,
Name: tgt.Name,
Type: database.TargetTypeDatabase,
ArchiveExpiry: tgt.ArchiveExpiry,
ArchiveRotation: tgt.ArchiveRotation,
},
}
d.ID = dlv.ID
@@ -218,7 +222,7 @@ func TestDeliverDatabase_ArchivesEvent(t *testing.T) {
t.Parallel()
env := setupArchiveTest(t)
tgt := env.seedDatabaseTarget(t, "")
tgt := env.seedDatabaseTarget(t)
webhookDB := testWebhookDB(t)
event := seedEvent(t, webhookDB, `{"archived":true}`)
@@ -256,9 +260,9 @@ func TestDeliverDatabase_ArchivesEvent(t *testing.T) {
}
// TestDeliverDatabase_ArchiveFailureFailsDelivery verifies that
// an archive error (here: an unparseable expiry in the target
// config) fails the delivery loudly: the attempt is recorded as
// failed with the error and the delivery is marked failed, not
// an archive error (here: an event with no webhook id to record)
// fails the delivery loudly: the attempt is recorded as failed
// with the error and the delivery is marked failed, not
// delivered.
func TestDeliverDatabase_ArchiveFailureFailsDelivery(
t *testing.T,
@@ -266,11 +270,12 @@ func TestDeliverDatabase_ArchiveFailureFailsDelivery(
t.Parallel()
env := setupArchiveTest(t)
tgt := env.seedDatabaseTarget(t, `{"expiry":"nonsense"}`)
tgt := env.seedDatabaseTarget(t)
webhookDB := testWebhookDB(t)
event := seedEvent(t, webhookDB, `{"archived":false}`)
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
d.Event.WebhookID = ""
env.eng.ExportDeliverDatabase(webhookDB, d)
@@ -295,12 +300,96 @@ func TestDeliverDatabase_ArchiveFailureFailsDelivery(
"the attempt must be recorded as failed",
)
assert.Contains(t,
results[0].Error, "nonsense",
results[0].Error, "without a webhook id",
"the archive error must be recorded on the attempt",
)
assert.NoFileExists(t,
env.archivePath(tgt),
"no archive file should exist for a failed config",
"no archive file should exist for a failed write",
)
}
// seedArchivedRow creates the archive file at path holding one row, for
// eventID, archived at archivedAt.
func seedArchivedRow(
t *testing.T, path, eventID string, archivedAt time.Time,
) {
t.Helper()
sqlDB, err := sql.Open(
"sqlite", fmt.Sprintf("file:%s?mode=rwc", path),
)
require.NoError(t, err)
defer func() { _ = sqlDB.Close() }()
gdb, err := gorm.Open(
sqlite.Dialector{Conn: sqlDB},
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
)
require.NoError(t, err)
require.NoError(t, migrations.Apply(gdb, migrations.Archive))
require.NoError(t, gdb.Exec(
"INSERT INTO archived_events (event_id, archived_at) VALUES (?, ?)",
eventID, archivedAt,
).Error)
}
// TestDeliverDatabase_TaskArchiveSettings proves a task's ArchiveExpiry
// and ArchiveRotation reach the archive through the engine. The
// target's row has neither, so only the task can bring them: with daily
// rotation the event goes to the file for its day, and opening that
// file with a one-hour expiry prunes the row it already held, archived
// two hours ago.
func TestDeliverDatabase_TaskArchiveSettings(t *testing.T) {
t.Parallel()
s := newISetup(t)
webhook := database.Webhook{UserID: uuid.NewString(), Name: "settings"}
webhook.ID = s.WebhookID
require.NoError(t, s.MainDB.Create(&webhook).Error)
target := database.Target{
WebhookID: s.WebhookID,
Name: "archive",
Type: database.TargetTypeDatabase,
Active: true,
MaxAttempts: 1,
}
require.NoError(t, s.MainDB.Create(&target).Error)
event := iSeedEvent(t, s.WebhookDB, s.WebhookID, `{"archived":true}`)
d := iSeedDelivery(
t, s.WebhookDB, event.ID, target.ID,
database.DeliveryStatusPending,
)
task := iTask(
d, event, s.WebhookID, target.ID, target.Name, "", 1, 1,
&event.Body,
)
task.TargetType = database.TargetTypeDatabase
task.ArchiveExpiry = time.Hour
task.ArchiveRotation = "daily"
daily := target
daily.ArchiveRotation = task.ArchiveRotation
dailyFile := archive.PathAt(s.DBMgr, &webhook, &daily, event.CreatedAt)
seedArchivedRow(t, dailyFile, "expired", time.Now().Add(-2*time.Hour))
s.Engine.ExportProcessNewTask(context.TODO(), &task)
iAssertStatus(t, s.WebhookDB, d.ID, database.DeliveryStatusDelivered)
var rows []archivedRow
require.NoError(t, openArchiveDBForRead(t, dailyFile).
Table("archived_events").Find(&rows).Error)
require.Len(t, rows, 1)
assert.Equal(t, event.ID, rows[0].EventID,
"the event goes to its day's file and the expired row is pruned",
)
}
internal/delivery/target_headers.go
+7 -9
View File
@@ -17,7 +17,7 @@ import (
const MaxTargetTimeoutSeconds = 300
// Errors returned when a target's header or timeout form input
// cannot be turned into a configuration.
// cannot be turned into the target's setting.
//
// None of these ever quotes a header VALUE. A target header value
// is routinely an authorization token, and these messages are shown
@@ -48,8 +48,8 @@ var (
)
// isReservedTargetHeader reports whether name (canonicalised) is a
// header a target configuration may not set, because the delivery
// path or net/http itself writes it regardless.
// header a target may not set, because the delivery path or
// net/http itself writes it regardless.
//
// These are rejected rather than accepted-and-ignored. Storing a
// header that provably never reaches the wire tells the operator
@@ -77,13 +77,11 @@ func isReservedTargetHeader(name string) bool {
// ParseTargetHeaders turns the target form's headers field — one
// "Name: value" pair per line, blank lines ignored — into the map
// stored in HTTPTargetConfig.Headers. Names are canonicalised, so a
// stored in database.Target.Headers. Names are canonicalised, so a
// name repeated in a different case is still a duplicate rather than
// one pair silently overwriting the other.
//
// An input with no pairs yields an empty map, which omitempty drops
// from the stored config, so a target configured with no headers
// stores no headers key at all.
// An input with no pairs yields an empty map.
func ParseTargetHeaders(raw string) (map[string]string, error) {
headers := make(map[string]string)
@@ -223,8 +221,8 @@ func FormatTargetHeaders(headers map[string]string) string {
// ParseTargetTimeout interprets the target form's timeout field as
// a whole number of seconds. An empty field means "unset" and yields
// 0, which omitempty drops from the stored config and which the
// delivery path reads as "use the shared client's timeout".
// 0, which the delivery path reads as "use the shared client's
// timeout".
//
// Anything else that is not a whole number in range is an error, not
// a silently substituted default: a target whose timeout was typed
internal/delivery/target_headers_test.go
+4 -99
View File
@@ -1,23 +1,21 @@
package delivery_test
import (
"encoding/json"
"strconv"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/delivery"
)
// Literals these tests repeat, named so that the header names and the
// keep-forever archive config each have one definition.
// Literals these tests repeat, named so that each header name has one
// definition.
const (
headerAuthorization = "Authorization"
headerContentType = "Content-Type"
headerUserAgent = "User-Agent"
bearerValue = "Bearer abc"
archiveConfigNever = "{\"expiry\":\"never\"}"
)
func TestParseTargetHeaders_AcceptsPairs(t *testing.T) {
@@ -41,23 +39,13 @@ func TestParseTargetHeaders_AcceptsPairs(t *testing.T) {
)
}
// A configuration with no headers must stay indistinguishable from
// one written before the field existed, so omitempty drops the key.
// An input of blank lines is no headers, not an error.
func TestParseTargetHeaders_EmptyInputYieldsNoHeaders(t *testing.T) {
t.Parallel()
got, err := delivery.ParseTargetHeaders("\n \n")
require.NoError(t, err)
assert.Empty(t, got)
encoded, err := json.Marshal(delivery.HTTPTargetConfig{
URL: "https://example.com/h",
Headers: got,
})
require.NoError(t, err)
assert.JSONEq(
t, `{"url":"https://example.com/h"}`, string(encoded),
)
}
func TestParseTargetHeaders_Rejects(t *testing.T) {
@@ -181,89 +169,6 @@ func TestFormatTargetTimeout(t *testing.T) {
assert.Empty(t, delivery.FormatTargetTimeout(-1))
}
func TestNewTargetConfigForm(t *testing.T) {
t.Parallel()
form, err := delivery.NewTargetConfigForm(&database.Target{
Type: database.TargetTypeHTTP,
Config: `{"url":"https://example.com/h",` +
`"headers":{"Authorization":"Bearer abc"},` +
`"timeout":9}`,
MaxAttempts: 1,
})
require.NoError(t, err)
assert.Equal(t, "https://example.com/h", form.URL)
assert.Equal(t, "Authorization: Bearer abc\n", form.Headers)
assert.Equal(t, "9", form.Timeout)
form, err = delivery.NewTargetConfigForm(&database.Target{
Type: database.TargetTypeSlack,
Config: `{"webhookUrl":"https://hooks.example/s"}`,
MaxAttempts: 1,
})
require.NoError(t, err)
assert.Equal(t, "https://hooks.example/s", form.URL)
form, err = delivery.NewTargetConfigForm(&database.Target{
Type: database.TargetTypeDatabase,
Config: `{"expiry":"720h","rotation":"daily"}`,
})
require.NoError(t, err)
assert.Equal(t, "720h", form.Expiry)
assert.Equal(t, rotationDaily, form.Rotation)
form, err = delivery.NewTargetConfigForm(&database.Target{
Type: database.TargetTypeLog,
})
require.NoError(t, err)
assert.Empty(t, form.URL)
}
// A keep-forever archive target yields an empty expiry, so the edit
// form starts on never; saving it unchanged stores never, which means
// the same as an empty expiry.
func TestNewTargetConfigForm_DatabaseNeverIsBlank(t *testing.T) {
t.Parallel()
for _, cfg := range []string{"", `{}`, archiveConfigNever} {
form, err := delivery.NewTargetConfigForm(
&database.Target{
Type: database.TargetTypeDatabase,
Config: cfg,
},
)
require.NoError(t, err, cfg)
assert.Empty(t, form.Expiry, cfg)
}
}
// An unreadable stored config is an error rather than a blank form
// that looks like a target with no settings, so the caller can tell
// the operator that saving replaces the stored value.
func TestNewTargetConfigForm_UnreadableConfigErrors(t *testing.T) {
t.Parallel()
cases := []*database.Target{
{Type: database.TargetTypeHTTP, Config: "not json", MaxAttempts: 1},
{Type: database.TargetTypeHTTP, Config: `{}`, MaxAttempts: 1},
{Type: database.TargetTypeSlack, Config: "", MaxAttempts: 1},
{
Type: database.TargetTypeDatabase,
Config: `{"expiry":"soon"}`,
},
{
Type: database.TargetTypeDatabase,
Config: weeklyConfig,
},
{Type: database.TargetType("nope")},
}
for _, target := range cases {
_, err := delivery.NewTargetConfigForm(target)
require.Error(t, err, target.Type)
}
}
// The ceiling exists so one misconfigured target cannot hold a
// delivery worker indefinitely, and it is inclusive.
func TestParseTargetTimeout_CeilingIsInclusive(t *testing.T) {
internal/delivery/target_http.go
+23 -97
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -17,29 +16,6 @@ import (
"sneak.berlin/go/webhooker/internal/database"
)
// Sentinel errors returned by the config parsers.
var (
errEmptyTargetConfig = errors.New(
"empty target config",
)
errMissingTargetURL = errors.New(
"target URL is required",
)
)
// HTTPTargetConfig holds configuration for http target
// types.
type HTTPTargetConfig struct {
URL string `json:"url"`
Headers map[string]string `json:"headers,omitempty"`
Timeout int `json:"timeout,omitempty"`
// ForwardQuery passes each event's query string on to the target,
// appended to URL. Off, the target URL is sent exactly as
// configured.
ForwardQuery bool `json:"forwardQuery,omitempty"`
}
// httpCore holds the retry, backoff, and circuit-breaker
// machinery shared by the HTTP and Slack targets. Each of
// those targets owns its own httpCore instance (and thus its
@@ -385,34 +361,8 @@ func (t *httpTarget) Deliver(
task *Task,
sched Scheduler,
) {
cfg, err := parseHTTPConfig(d.Target.Config)
if err != nil {
t.eng.log.Error(
"invalid HTTP target config",
"target_id", d.TargetID,
"error", err,
)
recErr := t.eng.recordResult(
webhookDB, d, task.AttemptNum,
false, 0, "", err.Error(), 0,
)
if recErr != nil {
t.eng.bookkeepingFailed(d, recErr)
return
}
t.eng.settleStatus(
webhookDB, d, d.Target.Type,
database.DeliveryStatusFailed,
)
return
}
attempt := func() attemptResult {
return t.attempt(ctx, cfg, &d.Event)
return t.attempt(ctx, &d.Target, &d.Event)
}
t.deliver(
@@ -427,11 +377,11 @@ func (t *httpTarget) Deliver(
// string; only a transport-level error does.
func (t *httpTarget) attempt(
ctx context.Context,
cfg *HTTPTargetConfig,
target *database.Target,
event *database.Event,
) attemptResult {
statusCode, respBody, duration, reqErr :=
t.doHTTPRequest(ctx, cfg, event)
t.doHTTPRequest(ctx, target, event)
success := reqErr == nil &&
statusCode >= httpSuccessMin &&
@@ -453,7 +403,7 @@ func (t *httpTarget) attempt(
func (t *httpTarget) doHTTPRequest(
ctx context.Context,
cfg *HTTPTargetConfig,
target *database.Target,
event *database.Event,
) (int, string, int64, error) {
start := time.Now()
@@ -461,7 +411,7 @@ func (t *httpTarget) doHTTPRequest(
req, reqErr := http.NewRequestWithContext(
ctx,
http.MethodPost,
cfg.URL,
target.URL,
bytes.NewReader([]byte(event.Body)),
)
if reqErr != nil {
@@ -471,15 +421,15 @@ func (t *httpTarget) doHTTPRequest(
)
}
if cfg.ForwardQuery {
if target.ForwardQuery {
appendQuery(req.URL, event.RawQuery)
}
originScoped := applyRequestHeaders(
req, event, cfg, t.eng.userAgent(),
req, event, target.Headers, t.eng.userAgent(),
)
client := t.clientForRequest(cfg, originScoped)
client := t.clientForRequest(target.TimeoutSeconds, originScoped)
resp, doErr := executeHTTPRequest(client, req)
@@ -519,17 +469,17 @@ func appendQuery(u *url.URL, rawQuery string) {
}
// clientForRequest returns the client for one delivery attempt.
// originScoped is the header set applyRequestHeaders built for that
// attempt; a request with neither a per-target timeout nor an
// origin-scoped header gets the shared client, because there is
// then nothing for the redirect policy to strip and net/http's
// default policy already withholds Authorization and Cookie across
// hosts.
// timeoutSeconds is the target's TimeoutSeconds, and originScoped is
// the header set applyRequestHeaders built for that attempt; a
// request with neither a per-target timeout nor an origin-scoped
// header gets the shared client, because there is then nothing for
// the redirect policy to strip and net/http's default policy already
// withholds Authorization and Cookie across hosts.
func (t *httpTarget) clientForRequest(
cfg *HTTPTargetConfig,
timeoutSeconds int,
originScoped []string,
) *http.Client {
if cfg.Timeout <= 0 && len(originScoped) == 0 {
if timeoutSeconds <= 0 && len(originScoped) == 0 {
return t.client
}
@@ -542,9 +492,9 @@ func (t *httpTarget) clientForRequest(
Transport: t.client.Transport,
}
if cfg.Timeout > 0 {
if timeoutSeconds > 0 {
client.Timeout = time.Duration(
cfg.Timeout,
timeoutSeconds,
) * time.Second
}
@@ -555,31 +505,6 @@ func (t *httpTarget) clientForRequest(
return client
}
func parseHTTPConfig(
configJSON string,
) (*HTTPTargetConfig, error) {
if configJSON == "" {
return nil, errEmptyTargetConfig
}
var cfg HTTPTargetConfig
err := json.Unmarshal(
[]byte(configJSON), &cfg,
)
if err != nil {
return nil, fmt.Errorf(
"parsing config JSON: %w", err,
)
}
if cfg.URL == "" {
return nil, errMissingTargetURL
}
return &cfg, nil
}
// isForwardableHeader returns true if the header should
// be forwarded to targets.
func isForwardableHeader(name string) bool {
@@ -611,11 +536,12 @@ func isForwardableHeader(name string) bool {
// wins, otherwise the event's ContentType, otherwise none. The inbound
// Content-Type in the event's headers is never forwarded.
//
// userAgent is set last, over any configured or inbound User-Agent.
// configured is the target's Headers. userAgent is set last, over
// any configured or inbound User-Agent.
func applyRequestHeaders(
req *http.Request,
event *database.Event,
cfg *HTTPTargetConfig,
configured map[string]string,
userAgent string,
) []string {
if event.ContentType != "" {
@@ -626,7 +552,7 @@ func applyRequestHeaders(
originScoped := forwardEventHeaders(req, event)
for k, v := range cfg.Headers {
for k, v := range configured {
req.Header.Set(k, v)
originScoped[http.CanonicalHeaderKey(k)] = struct{}{}
}
@@ -691,7 +617,7 @@ func forwardEventHeaders(
}
// executeHTTPRequest sends an HTTP request using the provided
// client. URLs are validated by the config parsers and the
// client. URLs are validated by the target forms and the
// SSRF-safe transport before reaching here.
//
// Transport failures are masked here, at the single point
internal/delivery/target_http_query_test.go
+18 -28
View File
@@ -2,7 +2,6 @@ package delivery_test
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
@@ -18,25 +17,13 @@ import (
// with.
const eventQuery = "a=1&b=2"
// httpTargetConfig is the stored configuration of an HTTP target at
// targetURL.
func httpTargetConfig(
t *testing.T, targetURL string, forwardQuery bool,
) string {
t.Helper()
cfg, err := json.Marshal(delivery.HTTPTargetConfig{
URL: targetURL, ForwardQuery: forwardQuery,
})
require.NoError(t, err)
return string(cfg)
}
// deliverWithQuery sends one event that arrived with eventQuery to an
// HTTP target configured with cfg, through the path a received event's
// delivery takes, and returns the attempt it recorded.
func deliverWithQuery(t *testing.T, cfg string) database.DeliveryResult {
// HTTP target at targetURL, with its ForwardQuery setting as given,
// through the path a received event's delivery takes, and returns the
// attempt it recorded.
func deliverWithQuery(
t *testing.T, targetURL string, forwardQuery bool,
) database.DeliveryResult {
t.Helper()
s := newISetup(t)
@@ -47,8 +34,10 @@ func deliverWithQuery(t *testing.T, cfg string) database.DeliveryResult {
database.DeliveryStatusPending,
)
task := iTask(
d, event, s.WebhookID, d.TargetID, "query", cfg, 1, 1, &event.Body,
d, event, s.WebhookID, d.TargetID, "query", targetURL, 1, 1,
&event.Body,
)
task.ForwardQuery = forwardQuery
task.RawQuery = eventQuery
s.Engine.ExportProcessNewTask(context.TODO(), &task)
@@ -111,9 +100,9 @@ func TestDeliverHTTP_ForwardQuery(t *testing.T) {
))
t.Cleanup(ts.Close)
result := deliverWithQuery(t, httpTargetConfig(
result := deliverWithQuery(
t, ts.URL+tc.path, tc.forwardQuery,
))
)
assert.True(t, result.Success)
require.Len(t, received, 1)
@@ -144,12 +133,13 @@ func TestDeliverHTTP_ForwardedQueryKeepsTheTargetURLMasked(t *testing.T) {
t.Cleanup(ts.Close)
target := &database.Target{
Type: database.TargetTypeHTTP,
Config: httpTargetConfig(t, ts.URL+"/in?token="+secret, true),
MaxAttempts: 1,
Type: database.TargetTypeHTTP,
URL: ts.URL + "/in?token=" + secret,
ForwardQuery: true,
MaxAttempts: 1,
}
deliverWithQuery(t, target.Config)
deliverWithQuery(t, target.URL, target.ForwardQuery)
require.Len(t, received, 1)
sent := <-received
@@ -164,9 +154,9 @@ func TestDeliverHTTP_ForwardedQueryKeepsTheTargetURLMasked(t *testing.T) {
}
// Nothing listens on port 1.
failed := deliverWithQuery(t, httpTargetConfig(
failed := deliverWithQuery(
t, "http://127.0.0.1:1/in?token="+secret, true,
))
)
require.NotEmpty(t, failed.Error)
assert.NotContains(t, failed.Error, secret)
assert.NotContains(t, failed.Error, eventQuery)
internal/delivery/target_redact.go
+6 -27
View File
@@ -18,7 +18,7 @@ const RedactionMarker = "(redacted)"
// line behind the delivery path's own URL masking.
//
// It removes byte-identical echoes of strings taken from the
// target's stored configuration, and nothing else. Anything
// target's URL and headers, and nothing else. Anything
// the remote re-encodes survives: JSON "\/" escaping (what
// PHP's json_encode emits by default), percent-encoding, HTML
// entities, and an echo of only part of a path. It cannot
@@ -115,7 +115,7 @@ func (r Redactor) secretPrefixSuffix(s string) int {
}
// targetSecrets returns the credential-bearing strings a
// target's configuration carries.
// target's settings carry.
//
// The destination URL contributes. Its path, query and
// userinfo are the credential for both target types that have
@@ -129,36 +129,15 @@ func (r Redactor) secretPrefixSuffix(s string) int {
// picks out. That is the same class-based rule applied to
// URLs: an echoed Accept or User-Agent still renders, an
// echoed Authorization does not.
//
// A setting a type does not use is empty and contributes
// nothing, so a database or log target has nothing to redact.
func targetSecrets(t *database.Target) []string {
if t == nil {
return nil
}
switch t.Type {
case database.TargetTypeSlack:
cfg, err := parseSlackConfig(t.Config)
if err != nil {
return nil
}
return urlSecrets(cfg.WebhookURL)
case database.TargetTypeHTTP:
cfg, err := parseHTTPConfig(t.Config)
if err != nil {
return nil
}
return append(
urlSecrets(cfg.URL),
headerSecrets(cfg.Headers)...,
)
case database.TargetTypeDatabase, database.TargetTypeLog:
// Neither has a destination URL, so neither has
// anything to redact.
return nil
default:
return nil
}
return append(urlSecrets(t.URL), headerSecrets(t.Headers)...)
}
// urlSecrets returns the substrings of a destination URL that
internal/delivery/target_redact_test.go
+29 -29
View File
@@ -23,7 +23,7 @@ func TestRedactor_RemovesSlackWebhookURL(t *testing.T) {
r := delivery.NewRedactor(&database.Target{
Type: database.TargetTypeSlack,
Config: `{"webhookUrl":"` + redactWebhookURL + `"}`,
URL: redactWebhookURL,
MaxAttempts: 1,
})
@@ -54,7 +54,7 @@ func TestRedactor_RemovesSecretSeveredByACut(t *testing.T) {
r := delivery.NewRedactor(&database.Target{
Type: database.TargetTypeSlack,
Config: `{"webhookUrl":"` + redactWebhookURL + `"}`,
URL: redactWebhookURL,
MaxAttempts: 1,
})
@@ -86,18 +86,19 @@ func TestRedactor_RedactsCredentialShapedHeaderValues(
r := delivery.NewRedactor(&database.Target{
Type: database.TargetTypeHTTP,
Config: `{"url":"https://example.com/in",` +
`"headers":{` +
`"Authorization":"Bearer AAAAAAAAAAAA",` +
`"Cookie":"session=BBBBBBBBBBBB",` +
`"X-Api-Key":"CCCCCCCCCCCC",` +
`"X-Hub-Signature":"sha256=DDDDDDDDDDDD",` +
`"X-Sig":"EEEEEEEEEEEE",` +
`"X-Pass":"FFFFFFFFFFFF",` +
`"X-HMAC":"GGGGGGGGGGGG",` +
`"X-Credential":"HHHHHHHHHHHH",` +
`"Accept":"application/json",` +
`"User-Agent":"webhooker/1.0"}}`,
URL: "https://example.com/in",
Headers: map[string]string{
headerAuthorization: "Bearer AAAAAAAAAAAA",
"Cookie": "session=BBBBBBBBBBBB",
probeHeaderName: "CCCCCCCCCCCC",
"X-Hub-Signature": "sha256=DDDDDDDDDDDD",
"X-Sig": "EEEEEEEEEEEE",
"X-Pass": "FFFFFFFFFFFF",
"X-HMAC": "GGGGGGGGGGGG",
"X-Credential": "HHHHHHHHHHHH",
"Accept": "application/json",
headerUserAgent: "webhooker/1.0",
},
MaxAttempts: 1,
})
@@ -136,9 +137,9 @@ func TestRedactor_IgnoresVeryShortHeaderValues(t *testing.T) {
t.Parallel()
r := delivery.NewRedactor(&database.Target{
Type: database.TargetTypeHTTP,
Config: `{"url":"https://example.com/in",` +
`"headers":{"X-Api-Key":"ab"}}`,
Type: database.TargetTypeHTTP,
URL: "https://example.com/in",
Headers: map[string]string{probeHeaderName: "ab"},
MaxAttempts: 1,
})
@@ -155,7 +156,7 @@ func TestRedactor_RemovesBarePath(t *testing.T) {
r := delivery.NewRedactor(&database.Target{
Type: database.TargetTypeSlack,
Config: `{"webhookUrl":"` + redactWebhookURL + `"}`,
URL: redactWebhookURL,
MaxAttempts: 1,
})
@@ -189,7 +190,7 @@ func TestRedactor_RemovesHTTPURLQueryAndUserinfo(t *testing.T) {
r := delivery.NewRedactor(&database.Target{
Type: database.TargetTypeHTTP,
Config: `{"url":"` + raw + `"}`,
URL: raw,
MaxAttempts: 1,
})
@@ -239,7 +240,7 @@ func TestRedactor_RemovesEchoedQueryOfURLWithoutPath(t *testing.T) {
} {
r := delivery.NewRedactor(&database.Target{
Type: database.TargetTypeHTTP,
Config: `{"url":"` + dest + `"}`,
URL: dest,
MaxAttempts: 1,
})
@@ -261,7 +262,7 @@ func TestRedactor_LeavesUnrelatedTextAlone(t *testing.T) {
r := delivery.NewRedactor(&database.Target{
Type: database.TargetTypeSlack,
Config: `{"webhookUrl":"` + redactWebhookURL + `"}`,
URL: redactWebhookURL,
MaxAttempts: 1,
})
@@ -288,7 +289,7 @@ func TestRedactor_EmptyUserinfoDoesNotShredTheBody(t *testing.T) {
r := delivery.NewRedactor(&database.Target{
Type: database.TargetTypeHTTP,
Config: `{"url":"` + dest + `"}`,
URL: dest,
MaxAttempts: 1,
})
@@ -306,11 +307,10 @@ func TestRedactor_EmptyUserinfoDoesNotShredTheBody(t *testing.T) {
)
}
// TestRedactor_ZeroValueAndConfiglessTargets pins that a
// caller with no target, an unparseable config, or a target
// type with no destination URL gets a redactor that changes
// nothing rather than one that panics.
func TestRedactor_ZeroValueAndConfiglessTargets(t *testing.T) {
// TestRedactor_ZeroValueAndURLlessTargets pins that a caller
// with no target, or a target with no destination URL, gets a
// redactor that changes nothing rather than one that panics.
func TestRedactor_ZeroValueAndURLlessTargets(t *testing.T) {
t.Parallel()
const text = "some response body"
@@ -323,8 +323,8 @@ func TestRedactor_ZeroValueAndConfiglessTargets(t *testing.T) {
for _, tgt := range []database.Target{
{Type: database.TargetTypeLog},
{Type: database.TargetTypeDatabase},
{Type: database.TargetTypeSlack, Config: "not json", MaxAttempts: 1},
{Type: database.TargetTypeHTTP, Config: "", MaxAttempts: 1},
{Type: database.TargetTypeSlack, MaxAttempts: 1},
{Type: database.TargetTypeHTTP, MaxAttempts: 1},
} {
assert.Equal(
t, text,
internal/delivery/target_slack.go
+19 -81
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -15,18 +14,6 @@ import (
"sneak.berlin/go/webhooker/internal/database"
)
// errMissingWebhookURL is returned when a Slack target config
// omits its webhook URL.
var errMissingWebhookURL = errors.New(
"webhook_url is required",
)
// SlackTargetConfig holds configuration for slack target
// types.
type SlackTargetConfig struct {
WebhookURL string `json:"webhookUrl"`
}
// slackTarget delivers events to Slack incoming webhooks. It
// formats the event into a Slack message and posts it as
// JSON. It shares the retry core with the HTTP target: a
@@ -46,19 +33,6 @@ func (t *slackTarget) Deliver(
task *Task,
sched Scheduler,
) {
cfg, err := parseSlackConfig(d.Target.Config)
if err != nil {
t.eng.log.Error(
"invalid Slack target config",
"target_id", d.TargetID,
"error", err,
)
t.failConfig(webhookDB, d, err)
return
}
msg := FormatSlackMessage(&d.Event)
payload, err := json.Marshal(
@@ -71,13 +45,26 @@ func (t *slackTarget) Deliver(
"error", err,
)
t.failConfig(webhookDB, d, err)
recErr := t.eng.recordResult(
webhookDB, d, 1,
false, 0, "", err.Error(), 0,
)
if recErr != nil {
t.eng.bookkeepingFailed(d, recErr)
return
}
t.eng.settleStatus(
webhookDB, d, d.Target.Type,
database.DeliveryStatusFailed,
)
return
}
attempt := func() attemptResult {
return t.attempt(ctx, cfg, payload)
return t.attempt(ctx, d.Target.URL, payload)
}
t.deliver(
@@ -86,37 +73,13 @@ func (t *slackTarget) Deliver(
)
}
// failConfig records a first-attempt failure for a delivery
// that could not be prepared (bad config or unmarshalable
// payload) and marks it failed.
func (t *slackTarget) failConfig(
webhookDB *gorm.DB,
d *database.Delivery,
err error,
) {
recErr := t.eng.recordResult(
webhookDB, d, 1,
false, 0, "", err.Error(), 0,
)
if recErr != nil {
t.eng.bookkeepingFailed(d, recErr)
return
}
t.eng.settleStatus(
webhookDB, d, d.Target.Type,
database.DeliveryStatusFailed,
)
}
// attempt performs a single Slack POST and derives its
// outcome, preserving the engine's original semantics: a
// attempt performs a single Slack POST to targetURL and derives
// its outcome, preserving the engine's original semantics: a
// non-2xx response records an "HTTP <code>" error string and
// a transport error records a "sending request" error.
func (t *slackTarget) attempt(
ctx context.Context,
cfg *SlackTargetConfig,
targetURL string,
payload []byte,
) attemptResult {
start := time.Now()
@@ -124,7 +87,7 @@ func (t *slackTarget) attempt(
req, err := http.NewRequestWithContext(
ctx,
http.MethodPost,
cfg.WebhookURL,
targetURL,
bytes.NewReader(payload),
)
if err != nil {
@@ -186,31 +149,6 @@ func (t *slackTarget) readSlackResponse(
}
}
func parseSlackConfig(
configJSON string,
) (*SlackTargetConfig, error) {
if configJSON == "" {
return nil, errEmptyTargetConfig
}
var cfg SlackTargetConfig
err := json.Unmarshal(
[]byte(configJSON), &cfg,
)
if err != nil {
return nil, fmt.Errorf(
"parsing config JSON: %w", err,
)
}
if cfg.WebhookURL == "" {
return nil, errMissingWebhookURL
}
return &cfg, nil
}
// FormatSlackMessage builds a Slack-compatible message
// string from a webhook event.
func FormatSlackMessage(
internal/delivery/terminal_state_test.go
+8 -9
View File
@@ -45,7 +45,7 @@ func tSeedDeletedTarget(
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, name,
database.TargetTypeHTTP, iHTTPConfig(url), 5,
database.TargetTypeHTTP, url, 5,
)
event := iSeedEvent(
@@ -122,9 +122,9 @@ func TestProcessDelivery_UnknownTargetType_RecordsWhy(
)
target := database.Target{
Name: "mystery",
Type: tUnknownType,
Config: iHTTPConfig("http://example.com/hook"),
Name: "mystery",
Type: tUnknownType,
URL: "http://example.com/hook",
}
target.ID = targetID
@@ -140,7 +140,7 @@ func TestProcessDelivery_UnknownTargetType_RecordsWhy(
body := event.Body
task := iTask(
seeded, event, s.WebhookID, targetID, "mystery",
target.Config, 0, 1, &body,
target.URL, 0, 1, &body,
)
task.TargetType = tUnknownType
@@ -346,11 +346,10 @@ func tRetryChainSetup(
iCreateWebhook(t, s.MainDB, s.WebhookID, name)
targetID := uuid.New().String()
cfg := iHTTPConfig(ts.URL)
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, name,
database.TargetTypeHTTP, cfg, 5,
database.TargetTypeHTTP, ts.URL, 5,
)
event := iSeedEvent(
@@ -367,7 +366,7 @@ func tRetryChainSetup(
body := event.Body
return iTask(
d, event, s.WebhookID, targetID, name, cfg, 5, 2, &body,
d, event, s.WebhookID, targetID, name, ts.URL, 5, 2, &body,
), targetID
}
@@ -528,7 +527,7 @@ func TestRecoverSingleRetry_TargetUnreadable_LeavesDeliveryAlone(
iCreateTarget(
t, s.MainDB, targetID, s.WebhookID, "healthy",
database.TargetTypeHTTP,
iHTTPConfig("http://example.com/hook"), 5,
"http://example.com/hook", 5,
)
event := iSeedEvent(
internal/delivery/url_mask_test.go
+4 -16
View File
@@ -2,7 +2,6 @@ package delivery_test
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
@@ -71,13 +70,6 @@ func deliverSlackTo(
e := testEngine(t, 1)
targetID := uuid.New().String()
slackCfg, err := json.Marshal(
delivery.SlackTargetConfig{
WebhookURL: webhookURL,
},
)
require.NoError(t, err)
event := seedEvent(t, db, `{"test":true}`)
dlv := seedDelivery(
@@ -87,7 +79,7 @@ func deliverSlackTo(
d := buildSlackDelivery(
dlv, event, targetID,
"test-slack-mask", string(slackCfg),
"test-slack-mask", webhookURL,
)
e.ExportDeliverSlack(context.TODO(), db, d)
@@ -160,13 +152,9 @@ func TestDoHTTPRequest_TransportErrorMasksURL(t *testing.T) {
e := testEngine(t, 1)
cfg, err := e.ExportParseHTTPConfig(
newHTTPTargetConfig(host + maskSecretPath),
)
require.NoError(t, err)
statusCode, _, _, reqErr := e.ExportDoHTTPRequest(
context.TODO(), cfg,
context.TODO(),
&database.Target{URL: host + maskSecretPath},
&database.Event{Body: `{"test":true}`},
)
require.Error(t, reqErr)
@@ -188,7 +176,7 @@ func TestDoHTTPRequest_UnparsableURLIsMasked(t *testing.T) {
statusCode, _, _, reqErr := e.ExportDoHTTPRequest(
context.TODO(),
&delivery.HTTPTargetConfig{
&database.Target{
URL: "https://hooks.example.com" + maskSecretPath + "\n",
},
&database.Event{},
internal/delivery/user_agent_test.go
+3 -9
View File
@@ -2,7 +2,6 @@ package delivery_test
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"net/netip"
@@ -59,9 +58,9 @@ func TestUserAgent_IsTheBuildVersion(t *testing.T) {
statusCode, _, _, err := e.ExportDoHTTPRequest(
context.Background(),
&delivery.HTTPTargetConfig{
&database.Target{
URL: ts.URL,
Headers: map[string]string{"User-Agent": "configured/1"},
Headers: map[string]string{headerUserAgent: "configured/1"},
},
&database.Event{Headers: `{"User-Agent":["curl/8"]}`},
)
@@ -73,18 +72,13 @@ func TestUserAgent_IsTheBuildVersion(t *testing.T) {
db := testWebhookDB(t)
targetID := uuid.New().String()
slackCfg, err := json.Marshal(
delivery.SlackTargetConfig{WebhookURL: ts.URL},
)
require.NoError(t, err)
event := seedEvent(t, db, `{"action":"test"}`)
dlv := seedDelivery(
t, db, event.ID, targetID, database.DeliveryStatusPending,
)
e.ExportDeliverSlack(context.Background(), db, buildSlackDelivery(
dlv, event, targetID, "test-slack", string(slackCfg),
dlv, event, targetID, "test-slack", ts.URL,
))
require.Len(t, userAgents, 1, "the slack target sent no request")
assert.Equal(t, want, <-userAgents, "slack target")