Deploy main eacf1e7 to prod

Replaces #559, which conflicted.

Model: opus-5-5

internal/delivery/target_http.go

@@ -4,10 +4,10 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"sort"
 	"sync"
 	"time"
@@ -16,24 +16,6 @@ import (
 	"sneak.berlin/go/webhooker/internal/database"
 )
 
-// Sentinel errors returned by the config parsers.
-var (
-	errEmptyTargetConfig = errors.New(
-		"empty target config",
-	)
-	errMissingTargetURL = errors.New(
-		"target URL is required",
-	)
-)
-
-// HTTPTargetConfig holds configuration for http target
-// types.
-type HTTPTargetConfig struct {
-	URL     string            `json:"url"`
-	Headers map[string]string `json:"headers,omitempty"`
-	Timeout int               `json:"timeout,omitempty"`
-}
-
 // httpCore holds the retry, backoff, and circuit-breaker
 // machinery shared by the HTTP and Slack targets. Each of
 // those targets owns its own httpCore instance (and thus its
@@ -47,34 +29,72 @@ type httpCore struct {
 }
 
 // deliver runs one delivery attempt through the retry core.
-// A maxRetries of 0 is fire-and-forget: a single attempt is
-// recorded and no circuit breaker is consulted. A positive
-// maxRetries gates the attempt on the circuit breaker and
-// schedules a backed-off retry on failure.
+// A maxAttempts of 1 is a single attempt, recorded without
+// consulting a circuit breaker. A higher maxAttempts gates the
+// attempt on the circuit breaker and schedules a backed-off retry
+// on failure, until maxAttempts attempts have been made.
+//
+// ctx is the context attempt sends its request with. Stopping the
+// engine cancels it; see interruptedByStop.
 func (c *httpCore) deliver(
+	ctx context.Context,
 	webhookDB *gorm.DB,
 	d *database.Delivery,
 	task *Task,
 	sched Scheduler,
-	maxRetries int,
+	maxAttempts int,
 	attempt func() attemptResult,
 ) {
-	if maxRetries == 0 {
-		c.fireAndForget(webhookDB, d, attempt())
+	if maxAttempts == 1 {
+		c.fireAndForget(ctx, webhookDB, d, attempt())
 
 		return
 	}
 
 	c.withRetry(
-		webhookDB, d, task, sched, maxRetries, attempt,
+		ctx, webhookDB, d, task, sched, maxAttempts, attempt,
 	)
 }
 
+// interruptedByStop reports whether an attempt failed because the
+// engine is stopping, which cancels the context its request is made
+// with. Such an attempt is not recorded: no DeliveryResult, no circuit
+// breaker outcome, no status change, no count in the attempt metrics.
+// The delivery stays pending or retrying, and the recovery routine
+// sends it again on the next start.
+//
+// A target's own timeout leaves that context alone, so its attempt is
+// still recorded as the failure it is.
+func (c *httpCore) interruptedByStop(
+	ctx context.Context,
+	d *database.Delivery,
+	res attemptResult,
+) bool {
+	if res.success || ctx.Err() == nil {
+		return false
+	}
+
+	c.eng.log.Warn(
+		"delivery attempt interrupted by shutdown; not recorded, "+
+			"it is sent again on the next start",
+		"delivery_id", d.ID,
+		"event_id", d.EventID,
+		"target_id", d.TargetID,
+	)
+
+	return true
+}
+
 func (c *httpCore) fireAndForget(
+	ctx context.Context,
 	webhookDB *gorm.DB,
 	d *database.Delivery,
 	res attemptResult,
 ) {
+	if c.interruptedByStop(ctx, d, res) {
+		return
+	}
+
 	c.eng.observeAttempt(d.Target.Type, res.elapsed())
 
 	err := c.eng.recordResult(
@@ -104,11 +124,12 @@ func (c *httpCore) fireAndForget(
 }
 
 func (c *httpCore) withRetry(
+	ctx context.Context,
 	webhookDB *gorm.DB,
 	d *database.Delivery,
 	task *Task,
 	sched Scheduler,
-	maxRetries int,
+	maxAttempts int,
 	attempt func() attemptResult,
 ) {
 	cb := c.getCircuitBreaker(task.TargetID)
@@ -125,6 +146,10 @@ func (c *httpCore) withRetry(
 
 	res := attempt()
 
+	if c.interruptedByStop(ctx, d, res) {
+		return
+	}
+
 	c.eng.observeAttempt(d.Target.Type, res.elapsed())
 
 	err := c.eng.recordResult(
@@ -156,7 +181,7 @@ func (c *httpCore) withRetry(
 	cb.RecordFailure()
 
 	c.handleRetry(
-		webhookDB, d, task, sched, maxRetries, attemptNum,
+		webhookDB, d, task, sched, maxAttempts, attemptNum,
 	)
 }
 
@@ -217,10 +242,10 @@ func (c *httpCore) handleRetry(
 	d *database.Delivery,
 	task *Task,
 	sched Scheduler,
-	maxRetries int,
+	maxAttempts int,
 	attemptNum int,
 ) {
-	if attemptNum >= maxRetries {
+	if attemptNum >= maxAttempts {
 		c.eng.settleStatus(
 			webhookDB, d, d.Target.Type,
 			database.DeliveryStatusFailed,
@@ -234,7 +259,7 @@ func (c *httpCore) handleRetry(
 		database.DeliveryStatusRetrying,
 	)
 
-	backoff := calcBackoff(attemptNum)
+	backoff := Backoff(attemptNum)
 
 	retryTask := *task
 	retryTask.AttemptNum = attemptNum + 1
@@ -285,7 +310,8 @@ func (c *httpCore) publishCircuitState(
 
 // remainingBackoff returns how long remains of the backoff
 // window for the last attempt of a recovered retrying
-// delivery. It implements rescheduler.
+// delivery: zero once it has passed, or when the last attempt
+// cannot be read. It implements rescheduler.
 func (c *httpCore) remainingBackoff(
 	webhookDB *gorm.DB,
 	deliveryID string,
@@ -301,37 +327,16 @@ func (c *httpCore) remainingBackoff(
 		return 0
 	}
 
-	backoff := calcBackoff(attemptNum)
+	backoff := Backoff(attemptNum)
 	elapsed := time.Since(lastResult.CreatedAt)
 	remaining := backoff - elapsed
 
 	return max(remaining, 0)
 }
 
-// backoffElapsed reports whether the backoff window for the
-// last attempt of a retrying delivery has passed. It
-// implements rescheduler.
-func (c *httpCore) backoffElapsed(
-	webhookDB *gorm.DB,
-	deliveryID string,
-	attemptNum int,
-) bool {
-	var lastResult database.DeliveryResult
-
-	err := webhookDB.
-		Where("delivery_id = ?", deliveryID).
-		Order("created_at DESC").
-		First(&lastResult).Error
-	if err != nil {
-		return true
-	}
-
-	backoff := calcBackoff(attemptNum)
-
-	return time.Since(lastResult.CreatedAt) >= backoff
-}
-
-func calcBackoff(attemptNum int) time.Duration {
+// Backoff is how long an http or slack target with retries waits after
+// a delivery's failed attempt attemptNum before trying it again.
+func Backoff(attemptNum int) time.Duration {
 	shift := max(attemptNum-1, 0)
 	shift = min(shift, maxBackoffShift)
 
@@ -356,39 +361,13 @@ func (t *httpTarget) Deliver(
 	task *Task,
 	sched Scheduler,
 ) {
-	cfg, err := parseHTTPConfig(d.Target.Config)
-	if err != nil {
-		t.eng.log.Error(
-			"invalid HTTP target config",
-			"target_id", d.TargetID,
-			"error", err,
-		)
-
-		recErr := t.eng.recordResult(
-			webhookDB, d, task.AttemptNum,
-			false, 0, "", err.Error(), 0,
-		)
-		if recErr != nil {
-			t.eng.bookkeepingFailed(d, recErr)
-
-			return
-		}
-
-		t.eng.settleStatus(
-			webhookDB, d, d.Target.Type,
-			database.DeliveryStatusFailed,
-		)
-
-		return
-	}
-
 	attempt := func() attemptResult {
-		return t.attempt(ctx, cfg, &d.Event)
+		return t.attempt(ctx, &d.Target, &d.Event)
 	}
 
 	t.deliver(
-		webhookDB, d, task, sched,
-		d.Target.MaxRetries, attempt,
+		ctx, webhookDB, d, task, sched,
+		d.Target.MaxAttempts, attempt,
 	)
 }
 
@@ -398,11 +377,11 @@ func (t *httpTarget) Deliver(
 // string; only a transport-level error does.
 func (t *httpTarget) attempt(
 	ctx context.Context,
-	cfg *HTTPTargetConfig,
+	target *database.Target,
 	event *database.Event,
 ) attemptResult {
 	statusCode, respBody, duration, reqErr :=
-		t.doHTTPRequest(ctx, cfg, event)
+		t.doHTTPRequest(ctx, target, event)
 
 	success := reqErr == nil &&
 		statusCode >= httpSuccessMin &&
@@ -424,7 +403,7 @@ func (t *httpTarget) attempt(
 
 func (t *httpTarget) doHTTPRequest(
 	ctx context.Context,
-	cfg *HTTPTargetConfig,
+	target *database.Target,
 	event *database.Event,
 ) (int, string, int64, error) {
 	start := time.Now()
@@ -432,7 +411,7 @@ func (t *httpTarget) doHTTPRequest(
 	req, reqErr := http.NewRequestWithContext(
 		ctx,
 		http.MethodPost,
-		cfg.URL,
+		target.URL,
 		bytes.NewReader([]byte(event.Body)),
 	)
 	if reqErr != nil {
@@ -442,9 +421,15 @@ func (t *httpTarget) doHTTPRequest(
 		)
 	}
 
-	originScoped := applyRequestHeaders(req, event, cfg)
+	if target.ForwardQuery {
+		appendQuery(req.URL, event.RawQuery)
+	}
 
-	client := t.clientForRequest(cfg, originScoped)
+	originScoped := applyRequestHeaders(
+		req, event, target.Headers, t.eng.userAgent(),
+	)
+
+	client := t.clientForRequest(target.TimeoutSeconds, originScoped)
 
 	resp, doErr := executeHTTPRequest(client, req)
 
@@ -470,18 +455,31 @@ func (t *httpTarget) doHTTPRequest(
 	return resp.StatusCode, string(body), dur, nil
 }
 
+// appendQuery adds an event's query string to a delivery's URL, joined
+// with "&" to any query string the target URL already has.
+func appendQuery(u *url.URL, rawQuery string) {
+	switch {
+	case rawQuery == "":
+		return
+	case u.RawQuery == "":
+		u.RawQuery = rawQuery
+	default:
+		u.RawQuery += "&" + rawQuery
+	}
+}
+
 // clientForRequest returns the client for one delivery attempt.
-// originScoped is the header set applyRequestHeaders built for that
-// attempt; a request with neither a per-target timeout nor an
-// origin-scoped header gets the shared client, because there is
-// then nothing for the redirect policy to strip and net/http's
-// default policy already withholds Authorization and Cookie across
-// hosts.
+// timeoutSeconds is the target's TimeoutSeconds, and originScoped is
+// the header set applyRequestHeaders built for that attempt; a
+// request with neither a per-target timeout nor an origin-scoped
+// header gets the shared client, because there is then nothing for
+// the redirect policy to strip and net/http's default policy already
+// withholds Authorization and Cookie across hosts.
 func (t *httpTarget) clientForRequest(
-	cfg *HTTPTargetConfig,
+	timeoutSeconds int,
 	originScoped []string,
 ) *http.Client {
-	if cfg.Timeout <= 0 && len(originScoped) == 0 {
+	if timeoutSeconds <= 0 && len(originScoped) == 0 {
 		return t.client
 	}
 
@@ -494,9 +492,9 @@ func (t *httpTarget) clientForRequest(
 		Transport: t.client.Transport,
 	}
 
-	if cfg.Timeout > 0 {
+	if timeoutSeconds > 0 {
 		client.Timeout = time.Duration(
-			cfg.Timeout,
+			timeoutSeconds,
 		) * time.Second
 	}
 
@@ -507,31 +505,6 @@ func (t *httpTarget) clientForRequest(
 	return client
 }
 
-func parseHTTPConfig(
-	configJSON string,
-) (*HTTPTargetConfig, error) {
-	if configJSON == "" {
-		return nil, errEmptyTargetConfig
-	}
-
-	var cfg HTTPTargetConfig
-
-	err := json.Unmarshal(
-		[]byte(configJSON), &cfg,
-	)
-	if err != nil {
-		return nil, fmt.Errorf(
-			"parsing config JSON: %w", err,
-		)
-	}
-
-	if cfg.URL == "" {
-		return nil, errMissingTargetURL
-	}
-
-	return &cfg, nil
-}
-
 // isForwardableHeader returns true if the header should
 // be forwarded to targets.
 func isForwardableHeader(name string) bool {
@@ -562,10 +535,14 @@ func isForwardableHeader(name string) bool {
 // Content-Type goes out once: a Content-Type configured on the target
 // wins, otherwise the event's ContentType, otherwise none. The inbound
 // Content-Type in the event's headers is never forwarded.
+//
+// configured is the target's Headers. userAgent is set last, over
+// any configured or inbound User-Agent.
 func applyRequestHeaders(
 	req *http.Request,
 	event *database.Event,
-	cfg *HTTPTargetConfig,
+	configured map[string]string,
+	userAgent string,
 ) []string {
 	if event.ContentType != "" {
 		req.Header.Set(
@@ -575,12 +552,12 @@ func applyRequestHeaders(
 
 	originScoped := forwardEventHeaders(req, event)
 
-	for k, v := range cfg.Headers {
+	for k, v := range configured {
 		req.Header.Set(k, v)
 		originScoped[http.CanonicalHeaderKey(k)] = struct{}{}
 	}
 
-	req.Header.Set("User-Agent", "webhooker/1.0")
+	req.Header.Set("User-Agent", userAgent)
 
 	// A Content-Type configured on the target describes the body
 	// being sent rather than the sender. A 307/308 preserves the
@@ -640,7 +617,7 @@ func forwardEventHeaders(
 }
 
 // executeHTTPRequest sends an HTTP request using the provided
-// client. URLs are validated by the config parsers and the
+// client. URLs are validated by the target forms and the
 // SSRF-safe transport before reaching here.
 //
 // Transport failures are masked here, at the single point