Render delivery attempt detail in the event log (closes #202) (#219)

delivery_results stored status_code, response_body, error, duration and
attempt_num, and no template rendered any of it, so a failure read as
"target: failed" and diagnosing it meant opening the per-webhook SQLite
file by hand.

An expanded delivery now lists its attempts with attempt number, status
code, duration, error and response body. The body is bounded in the query
rather than read whole and truncated in Go (#135), and a body the engine
itself cut is no longer presented as complete.

The response body and error are untrusted remote content, so target
credentials are removed before rendering. Two cases needed care: a secret
severed by the 4096-byte cut matches nothing as a whole string, and the
engine's io.LimitReader cuts at the same constant the renderer uses, so
the guard keys on the body reaching the cap rather than on the stored size
exceeding it. Empty secrets are filtered where the secret list is built,
because an empty string passed to strings.ReplaceAll inserts the marker at
every byte boundary.

loadTargetMap builds the redactor half unscoped, so a soft-deleted
target's historical deliveries still render redacted.

Also regenerates static/css/tailwind.css, which had drifted from the
templates: hover:text-red-700, text-red-500, underline and w-28 were in
use but absent from the served stylesheet (#236).

internal/handlers/source_management.go

@@ -4,11 +4,13 @@ import (
 	"encoding/json"
 	"errors"
 	"net/http"
+	"slices"
 	"strconv"
 	"strings"
 
 	"github.com/go-chi/chi"
 	"github.com/google/uuid"
+	"gorm.io/gorm"
 	"sneak.berlin/go/webhooker/internal/database"
 	"sneak.berlin/go/webhooker/internal/delivery"
 	"sneak.berlin/go/webhooker/internal/signature"
@@ -101,6 +103,32 @@ type DeliveryView struct {
 	ID     string
 	Status database.DeliveryStatus
 	Target delivery.TargetView
+
+	// Results is this delivery's attempts in attempt order,
+	// bounded by maxRenderedAttempts. Without them a failure
+	// renders as the status word alone and says nothing about
+	// why.
+	Results []DeliveryResultView
+
+	// AttemptCount is how many attempts were recorded, which
+	// is more than len(Results) once the middle was dropped.
+	AttemptCount int
+
+	// AttemptsOmitted is how many attempts were dropped from
+	// the middle of Results. The page must show it, or the
+	// bound would hide history rather than fold it.
+	AttemptsOmitted int
+}
+
+// eventLogTarget is what the event log needs to know about
+// one target: the display-safe view its template renders, and
+// the redactor that keeps that target's own credential out of
+// the text its remote peer chose. The two are kept together
+// so a caller cannot pick up one without the other, and apart
+// from TargetView so the secrets never reach a template.
+type eventLogTarget struct {
+	View     delivery.TargetView
+	Redactor delivery.Redactor
 }
 
 // HandleSourceList shows a list of user's webhooks.
@@ -769,12 +797,24 @@ func (h *Handlers) HandleSourceLogs() http.HandlerFunc {
 			return
 		}
 
-		targets := h.loadTargetMap(webhook.ID)
+		targets, err := h.loadTargetMap(webhook.ID)
+		if err != nil {
+			// Without the map every delivery renders through a
+			// zero redactor, so failing the page is the only
+			// safe answer.
+			h.serverError(w, "failed to load targets", err)
+
+			return
+		}
+
 		page := h.parsePage(r)
 
-		evts, total := h.loadEventsWithDeliveries(
+		evts, total, ok := h.loadEventsWithDeliveries(
 			w, webhook, targets, page,
 		)
+		if !ok {
+			return
+		}
 
 		totalPages := int(total) / paginationPerPage
 		if int(total)%paginationPerPage != 0 {
@@ -807,29 +847,54 @@ func (h *Handlers) HandleSourceLogs() http.HandlerFunc {
 }
 
 // loadTargetMap loads targets into a map of display-safe
-// views keyed by target ID. The projection happens here so
-// that no caller can hand a raw target, configuration blob
-// and all, to a template.
+// views keyed by target ID, each paired with its redactor.
+// The projection happens here so that no caller can hand a
+// raw target, configuration blob and all, to a template: the
+// raw rows do not leave this function.
+//
+// The load is Unscoped because deleting a target only soft
+// deletes the row while its deliveries survive in the
+// per-webhook database: a scoped load leaves those deliveries
+// with a zero redactor, which renders their response bodies
+// unredacted. Only the redactor half of the map is built from
+// deleted rows. The view half, which is what the page lists,
+// stays scoped.
 func (h *Handlers) loadTargetMap(
 	webhookID string,
-) map[string]delivery.TargetView {
+) (map[string]eventLogTarget, error) {
 	var targets []database.Target
 
-	h.db.DB().Where(
+	err := h.db.DB().Unscoped().Where(
 		"webhook_id = ?", webhookID,
-	).Find(&targets)
-
-	views := delivery.NewTargetViews(targets)
-
-	targetMap := make(
-		map[string]delivery.TargetView, len(views),
-	)
-
-	for _, v := range views {
-		targetMap[v.ID] = v
+	).Find(&targets).Error
+	if err != nil {
+		return nil, err
 	}
 
-	return targetMap
+	targetMap := make(
+		map[string]eventLogTarget, len(targets),
+	)
+	live := make([]database.Target, 0, len(targets))
+
+	for i := range targets {
+		targetMap[targets[i].ID] = eventLogTarget{
+			Redactor: delivery.NewRedactor(&targets[i]),
+		}
+
+		if !targets[i].DeletedAt.Valid {
+			live = append(live, targets[i])
+		}
+	}
+
+	// The views come from NewTargetViews rather than being
+	// rebuilt here, so the masking rules stay in one place.
+	for _, v := range delivery.NewTargetViews(live) {
+		entry := targetMap[v.ID]
+		entry.View = v
+		targetMap[v.ID] = entry
+	}
+
+	return targetMap, nil
 }
 
 // parsePage extracts a page number from the query string.
@@ -850,18 +915,22 @@ func (h *Handlers) parsePage(r *http.Request) int {
 // deliveries from the per-webhook database. Events come back
 // as capped projections rather than database.Event rows: see
 // eventLogColumns for why the cut happens in SQL.
+//
+// The bool reports whether the load succeeded. It is false
+// once this has answered the request with an error, and the
+// caller must then render nothing further.
 func (h *Handlers) loadEventsWithDeliveries(
 	w http.ResponseWriter,
 	webhook database.Webhook,
-	targetMap map[string]delivery.TargetView,
+	targetMap map[string]eventLogTarget,
 	page int,
-) ([]EventLogView, int64) {
+) ([]EventLogView, int64, bool) {
 	var totalEvents int64
 
 	var result []EventLogView
 
 	if !h.dbMgr.DBExists(webhook.ID) {
-		return result, totalEvents
+		return result, totalEvents, true
 	}
 
 	webhookDB, err := h.dbMgr.GetDB(webhook.ID)
@@ -870,7 +939,7 @@ func (h *Handlers) loadEventsWithDeliveries(
 			w, "failed to get webhook database", err,
 		)
 
-		return nil, 0
+		return nil, 0, false
 	}
 
 	webhookDB.Model(&database.Event{}).Where(
@@ -890,43 +959,170 @@ func (h *Handlers) loadEventsWithDeliveries(
 	).Find(&rows)
 
 	result = make([]EventLogView, len(rows))
+	eventDeliveries := make([][]database.Delivery, len(rows))
+
+	var deliveryIDs []string
 
 	for i := range rows {
 		result[i] = rows[i].view()
 
-		var deliveries []database.Delivery
-
 		webhookDB.Where(
 			"event_id = ?", rows[i].ID,
-		).Find(&deliveries)
+		).Find(&eventDeliveries[i])
 
+		for j := range eventDeliveries[i] {
+			deliveryIDs = append(
+				deliveryIDs, eventDeliveries[i][j].ID,
+			)
+		}
+	}
+
+	attempts, err := h.loadDeliveryResults(
+		webhookDB, deliveryIDs,
+	)
+	if err != nil {
+		h.serverError(
+			w, "failed to load delivery attempts", err,
+		)
+
+		return nil, 0, false
+	}
+
+	for i := range rows {
 		result[i].Deliveries = newDeliveryViews(
-			deliveries, targetMap,
+			eventDeliveries[i], targetMap, attempts,
 		)
 	}
 
-	return result, totalEvents
+	return result, totalEvents, true
+}
+
+// deliveryIDChunkSize bounds how many delivery IDs go into one
+// IN clause. SQLite refuses a statement carrying more than
+// SQLITE_MAX_VARIABLE_NUMBER (32766) bound parameters, and a
+// page holds one delivery per target per event, so a webhook
+// with enough targets would turn the whole query into an error
+// and the page into zero attempts.
+const deliveryIDChunkSize = 500
+
+// loadDeliveryResults loads the recorded attempts for the
+// page's deliveries, keyed by delivery ID.
+//
+// Each response body is cut by SQLite rather than in Go, for
+// the reason deliveryResultColumns gives. How many attempts a
+// delivery has is the target's MaxRetries, which the
+// authenticated operator sets; how many of them reach the page
+// is bounded again by maxRenderedAttempts.
+func (h *Handlers) loadDeliveryResults(
+	webhookDB *gorm.DB,
+	deliveryIDs []string,
+) (map[string][]deliveryResultRow, error) {
+	byDelivery := make(map[string][]deliveryResultRow)
+
+	for chunk := range slices.Chunk(
+		deliveryIDs, deliveryIDChunkSize,
+	) {
+		var rows []deliveryResultRow
+
+		err := webhookDB.Model(
+			&database.DeliveryResult{},
+		).Select(
+			deliveryResultColumns, maxRenderedResponseBytes,
+		).Where(
+			"delivery_id IN ?", chunk,
+		).Order("attempt_num ASC").Find(&rows).Error
+		if err != nil {
+			// Returning what was loaded so far renders the
+			// deliveries in the failed chunk as never having run,
+			// which is indistinguishable from ones that really
+			// never ran. The page fails instead.
+			return nil, err
+		}
+
+		for i := range rows {
+			byDelivery[rows[i].DeliveryID] = append(
+				byDelivery[rows[i].DeliveryID], rows[i],
+			)
+		}
+	}
+
+	return byDelivery, nil
 }
 
 // newDeliveryViews projects deliveries for rendering,
-// resolving each one's target to its display-safe view.
+// resolving each one's target to its display-safe view and
+// each one's attempts through that target's redactor.
 func newDeliveryViews(
 	deliveries []database.Delivery,
-	targetMap map[string]delivery.TargetView,
+	targetMap map[string]eventLogTarget,
+	attempts map[string][]deliveryResultRow,
 ) []DeliveryView {
 	views := make([]DeliveryView, len(deliveries))
 
 	for i := range deliveries {
+		target := targetMap[deliveries[i].TargetID]
+		rows := attempts[deliveries[i].ID]
+
+		results, omitted := renderedAttempts(
+			rows, target.Redactor,
+		)
+
 		views[i] = DeliveryView{
-			ID:     deliveries[i].ID,
-			Status: deliveries[i].Status,
-			Target: targetMap[deliveries[i].TargetID],
+			ID:              deliveries[i].ID,
+			Status:          deliveries[i].Status,
+			Target:          target.View,
+			Results:         results,
+			AttemptCount:    len(rows),
+			AttemptsOmitted: omitted,
 		}
 	}
 
 	return views
 }
 
+// maxRenderedAttempts bounds how many of one delivery's
+// attempts the page renders. Past it the middle is dropped and
+// counted, keeping the first attempts and the last ones: how
+// the delivery started failing and how it ended are what a
+// reader needs, and the count says plainly that the rest was
+// dropped rather than never recorded.
+const (
+	renderedAttemptsHead = 10
+	renderedAttemptsTail = 10
+	maxRenderedAttempts  = renderedAttemptsHead +
+		renderedAttemptsTail
+)
+
+// renderedAttempts projects a delivery's attempts through the
+// target's redactor, at most maxRenderedAttempts of them, and
+// reports how many it dropped.
+func renderedAttempts(
+	rows []deliveryResultRow,
+	redactor delivery.Redactor,
+) ([]DeliveryResultView, int) {
+	omitted := 0
+
+	if len(rows) > maxRenderedAttempts {
+		omitted = len(rows) - maxRenderedAttempts
+
+		kept := make(
+			[]deliveryResultRow, 0, maxRenderedAttempts,
+		)
+		kept = append(kept, rows[:renderedAttemptsHead]...)
+		kept = append(
+			kept, rows[len(rows)-renderedAttemptsTail:]...,
+		)
+		rows = kept
+	}
+
+	views := make([]DeliveryResultView, len(rows))
+	for i := range rows {
+		views[i] = rows[i].view(redactor)
+	}
+
+	return views, omitted
+}
+
 // HandleEntrypointCreate handles adding a new entrypoint.
 func (h *Handlers) HandleEntrypointCreate() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {