Files
webhooker/internal/delivery/target_slack.go
clawbot 9a70afb8b7
All checks were successful
check / check (push) Successful in 3m3s
Make SQLite durable under concurrent readers and stop re-delivering stranded webhooks (closes #256)
An operator running `sqlite3 <db> .dump` against their own per-webhook
database wedged it: inbound webhooks rejected with HTTP 500, delivered
webhooks stranded at `pending`, and every one of them POSTed a second
time on the next restart while the event log recorded a single attempt.

Durability. Every SQLite file — main, per-webhook, and archive — now
opens through one path, `internal/database/sqlite_open.go`, in WAL
journal mode with a 10-second busy timeout, `BEGIN IMMEDIATE`
transactions, and a bounded connection pool. WAL is what stops a reader
blocking writers at all. `_txlock=immediate` is what stops a `COMMIT`
failing while its transaction stays open on a pooled connection, which
is how four `database is locked` errors became 593 `cannot start a
transaction within a transaction`. `cache=shared` is gone, because
under it an in-process conflict is SQLITE_LOCKED, which the busy
handler does not retry. The busy timeout is applied before
journal_mode: the driver runs DSN pragmas in order on every new
connection, and `PRAGMA journal_mode` takes a lock, so the reverse
order leaves the one pragma that can block uncovered by the handler
meant to cover it.

Eligibility. `internal/delivery/inflight.go` holds the set of
deliveries the engine owns — taken when a task is queued, when a
target schedules a retry, and by every recovery path before it
re-dispatches; dropped when the worker that ran the task returns.
Recovery and both sweep arms re-dispatch only what the set does not
hold. Nothing decides that from a row's age: a delivery waiting in a
10000-deep channel is arbitrarily old and perfectly healthy, and
reasoning from age re-sends it. `takeForRedispatch` is the single gate
every re-dispatch goes through — ownership first, then a conditional
update confirming the row is still in the status the batch read.

Bookkeeping. `recordResult` and `updateDeliveryStatus` return their
errors instead of logging and dropping them, and a caller whose
bookkeeping write failed writes nothing at all: the delivery keeps
whichever non-terminal status it already held, and the sweeps recover
it. Every recovery path — pending and retrying alike — first settles
any delivery that already holds a successful `DeliveryResult` rather
than sending it again. Recovery continues each delivery's own attempt
numbering instead of restarting at 1. The sweep gains a
`pending`-with-age-bound arm, so a stranded delivery no longer waits
for a restart.

Docs. WAL produces `-wal`/`-shm` sidecars, so the backup and restore
procedures in README.md are corrected against measurement: both
documented procedures were re-run against a live instance, a `-wal`
left by a crash carries data the `.db` alone does not, and an archive
file normally holds its rows in a `-wal` rather than in the `.db`.
2026-08-24 01:02:01 +00:00

306 lines
5.7 KiB
Go

package delivery
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
"gorm.io/gorm"
"sneak.berlin/go/webhooker/internal/database"
)
// errMissingWebhookURL is returned when a Slack target config
// omits its webhook URL.
var errMissingWebhookURL = errors.New(
"webhook_url is required",
)
// SlackTargetConfig holds configuration for slack target
// types.
type SlackTargetConfig struct {
WebhookURL string `json:"webhookUrl"`
}
// slackTarget delivers events to Slack incoming webhooks. It
// formats the event into a Slack message and posts it as
// JSON. It shares the retry core with the HTTP target: a
// MaxRetries of 0 stays single-attempt fire-and-forget
// (preserving existing Slack targets), while a positive
// MaxRetries adds backoff and circuit breaking.
type slackTarget struct {
*httpCore
client *http.Client
}
// Deliver implements Target.
func (t *slackTarget) Deliver(
ctx context.Context,
webhookDB *gorm.DB,
d *database.Delivery,
task *Task,
sched Scheduler,
) {
cfg, err := parseSlackConfig(d.Target.Config)
if err != nil {
t.eng.log.Error(
"invalid Slack target config",
"target_id", d.TargetID,
"error", err,
)
t.failConfig(webhookDB, d, err)
return
}
msg := FormatSlackMessage(&d.Event)
payload, err := json.Marshal(
map[string]string{"text": msg},
)
if err != nil {
t.eng.log.Error(
"failed to marshal Slack payload",
"target_id", d.TargetID,
"error", err,
)
t.failConfig(webhookDB, d, err)
return
}
attempt := func() attemptResult {
return t.attempt(ctx, cfg, payload)
}
t.deliver(
webhookDB, d, task, sched,
d.Target.MaxRetries, attempt,
)
}
// failConfig records a first-attempt failure for a delivery
// that could not be prepared (bad config or unmarshalable
// payload) and marks it failed.
func (t *slackTarget) failConfig(
webhookDB *gorm.DB,
d *database.Delivery,
err error,
) {
recErr := t.eng.recordResult(
webhookDB, d, 1,
false, 0, "", err.Error(), 0,
)
if recErr != nil {
t.eng.bookkeepingFailed(d, recErr)
return
}
t.eng.settleStatus(
webhookDB, d, d.Target.Type,
database.DeliveryStatusFailed,
)
}
// attempt performs a single Slack POST and derives its
// outcome, preserving the engine's original semantics: a
// non-2xx response records an "HTTP <code>" error string and
// a transport error records a "sending request" error.
func (t *slackTarget) attempt(
ctx context.Context,
cfg *SlackTargetConfig,
payload []byte,
) attemptResult {
start := time.Now()
req, err := http.NewRequestWithContext(
ctx,
http.MethodPost,
cfg.WebhookURL,
bytes.NewReader(payload),
)
if err != nil {
return attemptResult{
success: false,
errMsg: maskURLError(err).Error(),
}
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", "webhooker/1.0")
resp, doErr := executeHTTPRequest(t.client, req)
durationMs := time.Since(start).Milliseconds()
if doErr != nil {
return attemptResult{
success: false,
duration: durationMs,
errMsg: fmt.Errorf(
"sending request: %w", doErr,
).Error(),
}
}
defer func() { _ = resp.Body.Close() }()
return t.readSlackResponse(resp, durationMs)
}
func (t *slackTarget) readSlackResponse(
resp *http.Response,
durationMs int64,
) attemptResult {
body, readErr := io.ReadAll(
io.LimitReader(resp.Body, maxBodyLog),
)
if readErr != nil {
t.eng.log.Error(
"failed to read Slack response body",
"error", readErr,
)
}
success := resp.StatusCode >= httpSuccessMin &&
resp.StatusCode < httpSuccessMax
errMsg := ""
if !success {
errMsg = fmt.Sprintf("HTTP %d", resp.StatusCode)
}
return attemptResult{
statusCode: resp.StatusCode,
respBody: string(body),
duration: durationMs,
success: success,
errMsg: errMsg,
}
}
func parseSlackConfig(
configJSON string,
) (*SlackTargetConfig, error) {
if configJSON == "" {
return nil, errEmptyTargetConfig
}
var cfg SlackTargetConfig
err := json.Unmarshal(
[]byte(configJSON), &cfg,
)
if err != nil {
return nil, fmt.Errorf(
"parsing config JSON: %w", err,
)
}
if cfg.WebhookURL == "" {
return nil, errMissingWebhookURL
}
return &cfg, nil
}
// FormatSlackMessage builds a Slack-compatible message
// string from a webhook event.
func FormatSlackMessage(
event *database.Event,
) string {
var b strings.Builder
b.WriteString("*Webhook Event Received*\n")
fmt.Fprintf(
&b, "*Method:* `%s`\n", event.Method,
)
fmt.Fprintf(
&b,
"*Content-Type:* `%s`\n",
event.ContentType,
)
fmt.Fprintf(
&b,
"*Timestamp:* `%s`\n",
event.CreatedAt.UTC().Format(time.RFC3339),
)
fmt.Fprintf(
&b,
"*Body Size:* %d bytes\n",
len(event.Body),
)
if event.Body == "" {
b.WriteString("\n_(empty body)_\n")
return b.String()
}
if formatted := formatJSONBody(event.Body); formatted != "" {
b.WriteString(formatted)
return b.String()
}
formatRawBody(&b, event.Body)
return b.String()
}
func formatJSONBody(body string) string {
var parsed json.RawMessage
if json.Unmarshal([]byte(body), &parsed) != nil {
return ""
}
var pretty bytes.Buffer
if json.Indent(&pretty, parsed, "", " ") != nil {
return ""
}
var b strings.Builder
b.WriteString("\n```\n")
prettyStr := pretty.String()
const maxPayloadDisplay = 3500
if len(prettyStr) > maxPayloadDisplay {
b.WriteString(prettyStr[:maxPayloadDisplay])
b.WriteString("\n... (truncated)")
} else {
b.WriteString(prettyStr)
}
b.WriteString("\n```\n")
return b.String()
}
func formatRawBody(b *strings.Builder, body string) {
b.WriteString("\n```\n")
const maxRawDisplay = 3500
if len(body) > maxRawDisplay {
b.WriteString(body[:maxRawDisplay])
b.WriteString("\n... (truncated)")
} else {
b.WriteString(body)
}
b.WriteString("\n```\n")
}