package delivery import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net/http" "sort" "sync" "time" "gorm.io/gorm" "sneak.berlin/go/webhooker/internal/database" ) // Sentinel errors returned by the config parsers. var ( errEmptyTargetConfig = errors.New( "empty target config", ) errMissingTargetURL = errors.New( "target URL is required", ) ) // HTTPTargetConfig holds configuration for http target // types. type HTTPTargetConfig struct { URL string `json:"url"` Headers map[string]string `json:"headers,omitempty"` Timeout int `json:"timeout,omitempty"` } // httpCore holds the retry, backoff, and circuit-breaker // machinery shared by the HTTP and Slack targets. Each of // those targets owns its own httpCore instance (and thus its // own circuit breakers); the per-attempt request differs // between them and is supplied as a closure. type httpCore struct { eng *Engine // circuitBreakers stores a *CircuitBreaker per target ID. circuitBreakers sync.Map } // deliver runs one delivery attempt through the retry core. // A maxRetries of 0 is fire-and-forget: a single attempt is // recorded and no circuit breaker is consulted. A positive // maxRetries gates the attempt on the circuit breaker and // schedules a backed-off retry on failure. func (c *httpCore) deliver( webhookDB *gorm.DB, d *database.Delivery, task *Task, sched Scheduler, maxRetries int, attempt func() attemptResult, ) { if maxRetries == 0 { c.fireAndForget(webhookDB, d, attempt()) return } c.withRetry( webhookDB, d, task, sched, maxRetries, attempt, ) } func (c *httpCore) fireAndForget( webhookDB *gorm.DB, d *database.Delivery, res attemptResult, ) { c.eng.observeAttempt(d.Target.Type, res.elapsed()) c.eng.recordResult( webhookDB, d, 1, res.success, res.statusCode, res.respBody, res.errMsg, res.duration, ) if res.success { c.eng.updateDeliveryStatus( webhookDB, d, d.Target.Type, database.DeliveryStatusDelivered, ) return } c.eng.updateDeliveryStatus( webhookDB, d, d.Target.Type, database.DeliveryStatusFailed, ) } func (c *httpCore) withRetry( webhookDB *gorm.DB, d *database.Delivery, task *Task, sched Scheduler, maxRetries int, attempt func() attemptResult, ) { cb := c.getCircuitBreaker(task.TargetID) if c.circuitBreakerBlock(webhookDB, d, task, sched, cb) { return } // Allow may have moved the breaker to half-open, and the // attempt below may open or close it, so the gauge is // republished on every exit from here. defer c.publishCircuitState(d.Target.Type) attemptNum := task.AttemptNum res := attempt() c.eng.observeAttempt(d.Target.Type, res.elapsed()) c.eng.recordResult( webhookDB, d, attemptNum, res.success, res.statusCode, res.respBody, res.errMsg, res.duration, ) if res.success { cb.RecordSuccess() c.eng.updateDeliveryStatus( webhookDB, d, d.Target.Type, database.DeliveryStatusDelivered, ) return } cb.RecordFailure() c.handleRetry( webhookDB, d, task, sched, maxRetries, attemptNum, ) } func (c *httpCore) circuitBreakerBlock( webhookDB *gorm.DB, d *database.Delivery, task *Task, sched Scheduler, cb *CircuitBreaker, ) bool { if cb.Allow() { return false } defer c.publishCircuitState(d.Target.Type) remaining := cb.CooldownRemaining() c.eng.log.Info( "circuit breaker open, skipping delivery", "target_id", task.TargetID, "target_name", task.TargetName, "delivery_id", d.ID, "cooldown_remaining", remaining, ) c.eng.updateDeliveryStatus( webhookDB, d, d.Target.Type, database.DeliveryStatusRetrying, ) retryTask := *task sched.ScheduleRetry(retryTask, remaining) return true } func (c *httpCore) handleRetry( webhookDB *gorm.DB, d *database.Delivery, task *Task, sched Scheduler, maxRetries int, attemptNum int, ) { if attemptNum >= maxRetries { c.eng.updateDeliveryStatus( webhookDB, d, d.Target.Type, database.DeliveryStatusFailed, ) return } c.eng.updateDeliveryStatus( webhookDB, d, d.Target.Type, database.DeliveryStatusRetrying, ) backoff := calcBackoff(attemptNum) retryTask := *task retryTask.AttemptNum = attemptNum + 1 sched.ScheduleRetry(retryTask, backoff) } func (c *httpCore) getCircuitBreaker( targetID string, ) *CircuitBreaker { if val, ok := c.circuitBreakers.Load(targetID); ok { cb, _ := val.(*CircuitBreaker) return cb } fresh := NewCircuitBreaker() actual, _ := c.circuitBreakers.LoadOrStore( targetID, fresh, ) cb, _ := actual.(*CircuitBreaker) return cb } // publishCircuitState recounts this core's open breakers and // publishes the gauge. Each core holds the breakers of exactly one // target type, so the recount is over that type's targets alone. // Counting rather than adjusting a delta keeps the gauge honest // however a breaker changed state. func (c *httpCore) publishCircuitState( targetType database.TargetType, ) { open := 0 c.circuitBreakers.Range(func(_, val any) bool { cb, ok := val.(*CircuitBreaker) if ok && cb.State() == CircuitOpen { open++ } return true }) c.eng.mtr.SetCircuitBreakersOpen(targetType, open) } // remainingBackoff returns how long remains of the backoff // window for the last attempt of a recovered retrying // delivery. It implements rescheduler. func (c *httpCore) remainingBackoff( webhookDB *gorm.DB, deliveryID string, attemptNum int, ) time.Duration { var lastResult database.DeliveryResult err := webhookDB. Where("delivery_id = ?", deliveryID). Order("created_at DESC"). First(&lastResult).Error if err != nil { return 0 } backoff := calcBackoff(attemptNum) elapsed := time.Since(lastResult.CreatedAt) remaining := backoff - elapsed return max(remaining, 0) } // backoffElapsed reports whether the backoff window for the // last attempt of a retrying delivery has passed. It // implements rescheduler. func (c *httpCore) backoffElapsed( webhookDB *gorm.DB, deliveryID string, attemptNum int, ) bool { var lastResult database.DeliveryResult err := webhookDB. Where("delivery_id = ?", deliveryID). Order("created_at DESC"). First(&lastResult).Error if err != nil { return true } backoff := calcBackoff(attemptNum) return time.Since(lastResult.CreatedAt) >= backoff } func calcBackoff(attemptNum int) time.Duration { shift := max(attemptNum-1, 0) shift = min(shift, maxBackoffShift) return time.Duration(1<= httpSuccessMin && statusCode < httpSuccessMax errMsg := "" if reqErr != nil { errMsg = reqErr.Error() } return attemptResult{ statusCode: statusCode, respBody: respBody, duration: duration, success: success, errMsg: errMsg, } } func (t *httpTarget) doHTTPRequest( ctx context.Context, cfg *HTTPTargetConfig, event *database.Event, ) (int, string, int64, error) { start := time.Now() req, reqErr := http.NewRequestWithContext( ctx, http.MethodPost, cfg.URL, bytes.NewReader([]byte(event.Body)), ) if reqErr != nil { return 0, "", 0, fmt.Errorf( "creating request: %w", maskURLError(reqErr), ) } originScoped := applyRequestHeaders(req, event, cfg) client := t.clientForRequest(cfg, originScoped) resp, doErr := executeHTTPRequest(client, req) dur := time.Since(start).Milliseconds() if doErr != nil { return 0, "", dur, fmt.Errorf( "sending request: %w", doErr, ) } defer func() { _ = resp.Body.Close() }() body, readErr := io.ReadAll( io.LimitReader(resp.Body, maxBodyLog), ) if readErr != nil { return resp.StatusCode, "", dur, fmt.Errorf( "reading response body: %w", readErr, ) } return resp.StatusCode, string(body), dur, nil } // clientForRequest returns the client for one delivery attempt. // originScoped is the header set applyRequestHeaders built for that // attempt; a request with neither a per-target timeout nor an // origin-scoped header gets the shared client, because there is // then nothing for the redirect policy to strip and net/http's // default policy already withholds Authorization and Cookie across // hosts. func (t *httpTarget) clientForRequest( cfg *HTTPTargetConfig, originScoped []string, ) *http.Client { if cfg.Timeout <= 0 && len(originScoped) == 0 { return t.client } // Reuse the shared client's SSRF-safe transport so neither a // per-target timeout nor the redirect policy drops the // request-time private-IP guard — which, being a dial hook, // also covers every redirect hop. client := &http.Client{ Timeout: t.client.Timeout, Transport: t.client.Transport, } if cfg.Timeout > 0 { client.Timeout = time.Duration( cfg.Timeout, ) * time.Second } if len(originScoped) > 0 { client.CheckRedirect = offOriginHeaderPolicy(originScoped) } return client } func parseHTTPConfig( configJSON string, ) (*HTTPTargetConfig, error) { if configJSON == "" { return nil, errEmptyTargetConfig } var cfg HTTPTargetConfig err := json.Unmarshal( []byte(configJSON), &cfg, ) if err != nil { return nil, fmt.Errorf( "parsing config JSON: %w", err, ) } if cfg.URL == "" { return nil, errMissingTargetURL } return &cfg, nil } // isForwardableHeader returns true if the header should // be forwarded to targets. func isForwardableHeader(name string) bool { switch http.CanonicalHeaderKey(name) { case "Host", "Connection", "Keep-Alive", "Transfer-Encoding", "Te", "Trailer", "Upgrade", "Proxy-Authorization", "Proxy-Connection", "Content-Length": return false default: return true } } // applyRequestHeaders builds one outbound delivery's header set and // returns the canonical names of every header in it that is scoped // to the configured origin: the inbound event headers this delivery // forwarded, plus the operator's configured headers. The redirect // policy strips exactly that set on a hop that leaves the origin, // so the forward set is decided here and only here — a header added // to it is covered off-origin without a second edit elsewhere. func applyRequestHeaders( req *http.Request, event *database.Event, cfg *HTTPTargetConfig, ) []string { if event.ContentType != "" { req.Header.Set( "Content-Type", event.ContentType, ) } originScoped := forwardEventHeaders(req, event) for k, v := range cfg.Headers { req.Header.Set(k, v) originScoped[http.CanonicalHeaderKey(k)] = struct{}{} } req.Header.Set("User-Agent", "webhooker/1.0") // Content-Type describes the body being sent rather than the // sender, and the delivery path sets it from the event itself. // A 307/308 preserves the body across hosts, so stripping it // would send that body untyped. delete(originScoped, "Content-Type") // User-Agent is overwritten just above, so an inbound one never // reaches the wire and the value that does identifies this // delivery path rather than the sender. Reporting it would strip // it off-origin and leave net/http's own default in its place. delete(originScoped, "User-Agent") names := make([]string, 0, len(originScoped)) for name := range originScoped { names = append(names, name) } sort.Strings(names) return names } // forwardEventHeaders copies the inbound event's forwardable // headers onto the outbound request and returns the canonical names // it forwarded. Headers the event never carried are absent from the // result, so the redirect policy strips what was actually sent. func forwardEventHeaders( req *http.Request, event *database.Event, ) map[string]struct{} { forwarded := make(map[string]struct{}) if event.Headers == "" { return forwarded } var inbound map[string][]string if json.Unmarshal([]byte(event.Headers), &inbound) != nil { return forwarded } for k, vals := range inbound { if !isForwardableHeader(k) || len(vals) == 0 { continue } for _, v := range vals { req.Header.Add(k, v) } forwarded[http.CanonicalHeaderKey(k)] = struct{}{} } return forwarded } // executeHTTPRequest sends an HTTP request using the provided // client. URLs are validated by the config parsers and the // SSRF-safe transport before reaching here. // // Transport failures are masked here, at the single point // where every target's request errors are born, because the // caller stores them in DeliveryResult.Error: an unmasked // *url.Error would write the target URL — the credential for // a Slack incoming webhook — into the per-webhook database. func executeHTTPRequest( client *http.Client, req *http.Request, ) (*http.Response, error) { resp, err := client.Do(req) //#nosec G704 -- validated URL, SSRF-safe transport if err != nil { return nil, maskURLError(err) } return resp, nil }