All checks were successful
check / check (push) Successful in 3m10s
/metrics carried only the inbound HTTP surface, so a destination failing for an hour, a growing retry backlog and a stuck-open circuit breaker were all invisible: the receive side stays healthy in each case because it is. New internal/metrics registers, on the existing default registry that the go-http-metrics recorder and the promhttp handler already share: - webhooker_events_received_total - webhooker_delivery_attempts_total - webhooker_deliveries_succeeded_total - webhooker_deliveries_failed_total - webhooker_delivery_retries_total - webhooker_delivery_duration_seconds - webhooker_deliveries_pending / _retrying - webhooker_circuit_breakers_open The route mounting is untouched. Every delivery metric carries one label, target_type, whose domain is the four target-type constants; anything outside it collapses to "unknown" so no series can be minted from a UUID. Target ids, event ids and entrypoint ids are deliberately not labels. An attempt is counted, and its duration observed, only where one was actually dispatched — the target's own result path, which is also where the DeliveryResult is written. A delivery an open circuit breaker refuses sends nothing and records no result row; counting it would climb the attempts counter with no traffic behind it and pull the duration quantiles down for as long as the breaker stayed open, moving the metric the wrong way during the outage it exists to reveal. The log and database targets now time their own work, so their result rows carry a real duration too. The outcome counters move after the status row is written rather than before, so a transition the database rejected is never reported as an outcome that happened. The queue-depth gauges are counted out of the per-webhook databases by a 30s sampler rather than tracked as deltas, which would need seeding at startup and would drift on any transition that failed to persist. They publish an "unknown" series from registration: deliveries queued against a target that has since been deleted resolve to the empty type and are folded there, because a backlog behind a deleted target is precisely the one nobody is watching. The open-breaker gauge is recounted from the target's breaker registry on every state change. The orphaned-retry terminal path takes the target type as an argument rather than attaching the loaded target to the delivery. That path loads the delivery without its target relation on purpose: a populated Delivery.Target makes GORM's SaveBeforeAssociations upsert the whole target row on the status UPDATE, writing the plaintext target config — the credential, for a slack target — into the per-webhook events database. A test asserts that path leaves the targets table empty.
301 lines
5.6 KiB
Go
301 lines
5.6 KiB
Go
package delivery
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"gorm.io/gorm"
|
|
"sneak.berlin/go/webhooker/internal/database"
|
|
)
|
|
|
|
// errMissingWebhookURL is returned when a Slack target config
|
|
// omits its webhook URL.
|
|
var errMissingWebhookURL = errors.New(
|
|
"webhook_url is required",
|
|
)
|
|
|
|
// SlackTargetConfig holds configuration for slack target
|
|
// types.
|
|
type SlackTargetConfig struct {
|
|
WebhookURL string `json:"webhookUrl"`
|
|
}
|
|
|
|
// slackTarget delivers events to Slack incoming webhooks. It
|
|
// formats the event into a Slack message and posts it as
|
|
// JSON. It shares the retry core with the HTTP target: a
|
|
// MaxRetries of 0 stays single-attempt fire-and-forget
|
|
// (preserving existing Slack targets), while a positive
|
|
// MaxRetries adds backoff and circuit breaking.
|
|
type slackTarget struct {
|
|
*httpCore
|
|
|
|
client *http.Client
|
|
}
|
|
|
|
// Deliver implements Target.
|
|
func (t *slackTarget) Deliver(
|
|
ctx context.Context,
|
|
webhookDB *gorm.DB,
|
|
d *database.Delivery,
|
|
task *Task,
|
|
sched Scheduler,
|
|
) {
|
|
cfg, err := parseSlackConfig(d.Target.Config)
|
|
if err != nil {
|
|
t.eng.log.Error(
|
|
"invalid Slack target config",
|
|
"target_id", d.TargetID,
|
|
"error", err,
|
|
)
|
|
|
|
t.failConfig(webhookDB, d, err)
|
|
|
|
return
|
|
}
|
|
|
|
msg := FormatSlackMessage(&d.Event)
|
|
|
|
payload, err := json.Marshal(
|
|
map[string]string{"text": msg},
|
|
)
|
|
if err != nil {
|
|
t.eng.log.Error(
|
|
"failed to marshal Slack payload",
|
|
"target_id", d.TargetID,
|
|
"error", err,
|
|
)
|
|
|
|
t.failConfig(webhookDB, d, err)
|
|
|
|
return
|
|
}
|
|
|
|
attempt := func() attemptResult {
|
|
return t.attempt(ctx, cfg, payload)
|
|
}
|
|
|
|
t.deliver(
|
|
webhookDB, d, task, sched,
|
|
d.Target.MaxRetries, attempt,
|
|
)
|
|
}
|
|
|
|
// failConfig records a first-attempt failure for a delivery
|
|
// that could not be prepared (bad config or unmarshalable
|
|
// payload) and marks it failed.
|
|
func (t *slackTarget) failConfig(
|
|
webhookDB *gorm.DB,
|
|
d *database.Delivery,
|
|
err error,
|
|
) {
|
|
t.eng.recordResult(
|
|
webhookDB, d, 1,
|
|
false, 0, "", err.Error(), 0,
|
|
)
|
|
|
|
t.eng.updateDeliveryStatus(
|
|
webhookDB, d, d.Target.Type,
|
|
database.DeliveryStatusFailed,
|
|
)
|
|
}
|
|
|
|
// attempt performs a single Slack POST and derives its
|
|
// outcome, preserving the engine's original semantics: a
|
|
// non-2xx response records an "HTTP <code>" error string and
|
|
// a transport error records a "sending request" error.
|
|
func (t *slackTarget) attempt(
|
|
ctx context.Context,
|
|
cfg *SlackTargetConfig,
|
|
payload []byte,
|
|
) attemptResult {
|
|
start := time.Now()
|
|
|
|
req, err := http.NewRequestWithContext(
|
|
ctx,
|
|
http.MethodPost,
|
|
cfg.WebhookURL,
|
|
bytes.NewReader(payload),
|
|
)
|
|
if err != nil {
|
|
return attemptResult{
|
|
success: false,
|
|
errMsg: maskURLError(err).Error(),
|
|
}
|
|
}
|
|
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("User-Agent", "webhooker/1.0")
|
|
|
|
resp, doErr := executeHTTPRequest(t.client, req)
|
|
durationMs := time.Since(start).Milliseconds()
|
|
|
|
if doErr != nil {
|
|
return attemptResult{
|
|
success: false,
|
|
duration: durationMs,
|
|
errMsg: fmt.Errorf(
|
|
"sending request: %w", doErr,
|
|
).Error(),
|
|
}
|
|
}
|
|
|
|
defer func() { _ = resp.Body.Close() }()
|
|
|
|
return t.readSlackResponse(resp, durationMs)
|
|
}
|
|
|
|
func (t *slackTarget) readSlackResponse(
|
|
resp *http.Response,
|
|
durationMs int64,
|
|
) attemptResult {
|
|
body, readErr := io.ReadAll(
|
|
io.LimitReader(resp.Body, maxBodyLog),
|
|
)
|
|
if readErr != nil {
|
|
t.eng.log.Error(
|
|
"failed to read Slack response body",
|
|
"error", readErr,
|
|
)
|
|
}
|
|
|
|
success := resp.StatusCode >= httpSuccessMin &&
|
|
resp.StatusCode < httpSuccessMax
|
|
|
|
errMsg := ""
|
|
if !success {
|
|
errMsg = fmt.Sprintf("HTTP %d", resp.StatusCode)
|
|
}
|
|
|
|
return attemptResult{
|
|
statusCode: resp.StatusCode,
|
|
respBody: string(body),
|
|
duration: durationMs,
|
|
success: success,
|
|
errMsg: errMsg,
|
|
}
|
|
}
|
|
|
|
func parseSlackConfig(
|
|
configJSON string,
|
|
) (*SlackTargetConfig, error) {
|
|
if configJSON == "" {
|
|
return nil, errEmptyTargetConfig
|
|
}
|
|
|
|
var cfg SlackTargetConfig
|
|
|
|
err := json.Unmarshal(
|
|
[]byte(configJSON), &cfg,
|
|
)
|
|
if err != nil {
|
|
return nil, fmt.Errorf(
|
|
"parsing config JSON: %w", err,
|
|
)
|
|
}
|
|
|
|
if cfg.WebhookURL == "" {
|
|
return nil, errMissingWebhookURL
|
|
}
|
|
|
|
return &cfg, nil
|
|
}
|
|
|
|
// FormatSlackMessage builds a Slack-compatible message
|
|
// string from a webhook event.
|
|
func FormatSlackMessage(
|
|
event *database.Event,
|
|
) string {
|
|
var b strings.Builder
|
|
|
|
b.WriteString("*Webhook Event Received*\n")
|
|
|
|
fmt.Fprintf(
|
|
&b, "*Method:* `%s`\n", event.Method,
|
|
)
|
|
|
|
fmt.Fprintf(
|
|
&b,
|
|
"*Content-Type:* `%s`\n",
|
|
event.ContentType,
|
|
)
|
|
|
|
fmt.Fprintf(
|
|
&b,
|
|
"*Timestamp:* `%s`\n",
|
|
event.CreatedAt.UTC().Format(time.RFC3339),
|
|
)
|
|
|
|
fmt.Fprintf(
|
|
&b,
|
|
"*Body Size:* %d bytes\n",
|
|
len(event.Body),
|
|
)
|
|
|
|
if event.Body == "" {
|
|
b.WriteString("\n_(empty body)_\n")
|
|
|
|
return b.String()
|
|
}
|
|
|
|
if formatted := formatJSONBody(event.Body); formatted != "" {
|
|
b.WriteString(formatted)
|
|
|
|
return b.String()
|
|
}
|
|
|
|
formatRawBody(&b, event.Body)
|
|
|
|
return b.String()
|
|
}
|
|
|
|
func formatJSONBody(body string) string {
|
|
var parsed json.RawMessage
|
|
if json.Unmarshal([]byte(body), &parsed) != nil {
|
|
return ""
|
|
}
|
|
|
|
var pretty bytes.Buffer
|
|
if json.Indent(&pretty, parsed, "", " ") != nil {
|
|
return ""
|
|
}
|
|
|
|
var b strings.Builder
|
|
|
|
b.WriteString("\n```\n")
|
|
|
|
prettyStr := pretty.String()
|
|
|
|
const maxPayloadDisplay = 3500
|
|
if len(prettyStr) > maxPayloadDisplay {
|
|
b.WriteString(prettyStr[:maxPayloadDisplay])
|
|
b.WriteString("\n... (truncated)")
|
|
} else {
|
|
b.WriteString(prettyStr)
|
|
}
|
|
|
|
b.WriteString("\n```\n")
|
|
|
|
return b.String()
|
|
}
|
|
|
|
func formatRawBody(b *strings.Builder, body string) {
|
|
b.WriteString("\n```\n")
|
|
|
|
const maxRawDisplay = 3500
|
|
if len(body) > maxRawDisplay {
|
|
b.WriteString(body[:maxRawDisplay])
|
|
b.WriteString("\n... (truncated)")
|
|
} else {
|
|
b.WriteString(body)
|
|
}
|
|
|
|
b.WriteString("\n```\n")
|
|
}
|