Replaces #559, which conflicted. Model: opus-5-5
This commit is contained in:
@@ -0,0 +1,288 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"gorm.io/gorm"
|
||||
"sneak.berlin/go/webhooker/internal/archive"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/delivery"
|
||||
)
|
||||
|
||||
// TargetRowView is one row of the target list on a webhook's page.
|
||||
type TargetRowView struct {
|
||||
delivery.TargetView
|
||||
|
||||
// Deliveries counts the target's delivered and failed deliveries,
|
||||
// and is nil when the webhook's event database could not be read.
|
||||
Deliveries *TargetDeliveries
|
||||
|
||||
// Archive is a database target's archive files, and nil for a
|
||||
// target of any other type.
|
||||
Archive *ArchiveFileView
|
||||
|
||||
// Paused is set while the target's circuit breaker is turning its
|
||||
// deliveries away, and nil otherwise.
|
||||
Paused *PausedView
|
||||
}
|
||||
|
||||
// PausedView is a target's circuit breaker turning deliveries away.
|
||||
// While the breaker is open, Until is a time in UTC, and Relative how
|
||||
// long that is from now: on the target's row, when the cooldown ends;
|
||||
// on a delivery, the earliest it can be tried next. While it is
|
||||
// half-open both are empty: the cooldown has ended, and the target's
|
||||
// deliveries are held while one delivery tests whether the target has
|
||||
// recovered.
|
||||
type PausedView struct {
|
||||
Until string
|
||||
Relative string
|
||||
}
|
||||
|
||||
// pausedView reads the target's circuit breaker for its row, and
|
||||
// returns nil when the breaker lets the target's deliveries through.
|
||||
func (h *Handlers) pausedView(targetID string) *PausedView {
|
||||
state, cooldown := h.breakers.StateAndCooldown(targetID)
|
||||
|
||||
switch {
|
||||
case state == delivery.CircuitHalfOpen:
|
||||
return &PausedView{}
|
||||
case state == delivery.CircuitOpen && cooldown > 0:
|
||||
return newPausedView(time.Now().Add(cooldown))
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// deliveryPausedView reads the circuit breaker of a retrying delivery's
|
||||
// target. While it is open, it says the earliest the delivery can be
|
||||
// tried next: the later of the cooldown's end and the end of the
|
||||
// delivery's own backoff after its last attempt. It is only the
|
||||
// earliest: when the cooldown ends, one of the target's waiting
|
||||
// deliveries is sent to test it while the others wait at least one more
|
||||
// cooldown. Otherwise it returns nil, half-open included, since the
|
||||
// delivery may then be the one being sent to test the target.
|
||||
func (h *Handlers) deliveryPausedView(
|
||||
targetID string, attempts []deliveryResultRow,
|
||||
) *PausedView {
|
||||
state, cooldown := h.breakers.StateAndCooldown(targetID)
|
||||
if state != delivery.CircuitOpen || cooldown <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
next := time.Now().Add(cooldown)
|
||||
|
||||
if len(attempts) > 0 {
|
||||
last := attempts[len(attempts)-1]
|
||||
|
||||
backoffEnd := last.CreatedAt.Add(delivery.Backoff(last.AttemptNum))
|
||||
if backoffEnd.After(next) {
|
||||
next = backoffEnd
|
||||
}
|
||||
}
|
||||
|
||||
return newPausedView(next)
|
||||
}
|
||||
|
||||
// newPausedView is a PausedView of deliveries paused until the given
|
||||
// time. A time not on the current UTC day is written with its date.
|
||||
func newPausedView(until time.Time) *PausedView {
|
||||
until = until.UTC()
|
||||
|
||||
layout := time.TimeOnly
|
||||
if until.Format(time.DateOnly) != time.Now().UTC().Format(time.DateOnly) {
|
||||
layout = time.DateTime
|
||||
}
|
||||
|
||||
return &PausedView{
|
||||
Until: until.Format(layout) + " UTC",
|
||||
Relative: humanize.Time(until),
|
||||
}
|
||||
}
|
||||
|
||||
// TargetDeliveries is how many of a target's deliveries became
|
||||
// delivered and how many failed: in total, which retention does not
|
||||
// reduce, and in the last 24 hours. Deliveries still pending or
|
||||
// retrying count in neither.
|
||||
type TargetDeliveries struct {
|
||||
Delivered int64
|
||||
Failed int64
|
||||
|
||||
DeliveredLast24Hours int64
|
||||
FailedLast24Hours int64
|
||||
}
|
||||
|
||||
// ArchiveFileView is what a database target's row shows about its
|
||||
// archive files.
|
||||
type ArchiveFileView struct {
|
||||
// Name is the name of the file an event received now goes to.
|
||||
Name string
|
||||
|
||||
// Note says that file does not exist yet, or that the files could
|
||||
// not be read, and is empty otherwise.
|
||||
Note string
|
||||
|
||||
// Files counts the target's archive files, and is 0 when there are
|
||||
// none, or they could not be read, and so no size or last write to
|
||||
// show.
|
||||
Files int
|
||||
|
||||
// Size is the size on disk of all the files together. Written is
|
||||
// how long ago the latest of them was last written, and WrittenUTC
|
||||
// the full time the page shows on hover.
|
||||
Size string
|
||||
Written string
|
||||
WrittenUTC string
|
||||
}
|
||||
|
||||
// targetRows projects a webhook's targets for the target list on its
|
||||
// page.
|
||||
func (h *Handlers) targetRows(
|
||||
webhook *database.Webhook, targets []database.Target,
|
||||
) []TargetRowView {
|
||||
views := delivery.NewTargetViews(targets)
|
||||
rows := make([]TargetRowView, len(views))
|
||||
now := time.Now()
|
||||
|
||||
deliveries, err := h.loadTargetDeliveries(webhook.ID)
|
||||
if err != nil {
|
||||
h.log.Error(
|
||||
"failed to read target delivery counts",
|
||||
"webhook_id", webhook.ID,
|
||||
"error", err,
|
||||
)
|
||||
}
|
||||
|
||||
// NewTargetViews returns one view per target, in order.
|
||||
for i := range views {
|
||||
rows[i].TargetView = views[i]
|
||||
|
||||
if err == nil {
|
||||
counts := deliveries[targets[i].ID]
|
||||
rows[i].Deliveries = &counts
|
||||
}
|
||||
|
||||
if targets[i].Type == database.TargetTypeDatabase {
|
||||
rows[i].Archive = h.archiveFileView(webhook, &targets[i], now)
|
||||
}
|
||||
|
||||
rows[i].Paused = h.pausedView(targets[i].ID)
|
||||
}
|
||||
|
||||
return rows
|
||||
}
|
||||
|
||||
// loadTargetDeliveries reads the delivery counts of a webhook's targets
|
||||
// from its event database, keyed by target. A target with no deliveries
|
||||
// is left out, and so is every target when the event database does not
|
||||
// exist yet, since opening it would create it.
|
||||
func (h *Handlers) loadTargetDeliveries(
|
||||
webhookID string,
|
||||
) (map[string]TargetDeliveries, error) {
|
||||
if !h.dbMgr.DBExists(webhookID) {
|
||||
return map[string]TargetDeliveries{}, nil
|
||||
}
|
||||
|
||||
webhookDB, err := h.dbMgr.GetDB(webhookID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return readTargetDeliveries(webhookDB, time.Now())
|
||||
}
|
||||
|
||||
// readTargetDeliveries counts each target's deliveries that became
|
||||
// delivered and those that failed: in total from the targets' running
|
||||
// totals, and in the 24 hours before now from the deliveries' status
|
||||
// index. Each is one query for all the targets, and neither reads every
|
||||
// stored delivery.
|
||||
func readTargetDeliveries(
|
||||
db *gorm.DB, now time.Time,
|
||||
) (map[string]TargetDeliveries, error) {
|
||||
var totals []database.TargetTotals
|
||||
|
||||
err := db.Find(&totals).Error
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading target totals: %w", err)
|
||||
}
|
||||
|
||||
lastDay, err := finishedByTarget(db, now.Add(-longWindow))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
byTarget := make(map[string]TargetDeliveries, len(totals))
|
||||
|
||||
for _, total := range totals {
|
||||
byTarget[total.TargetID] = TargetDeliveries{
|
||||
Delivered: total.Delivered,
|
||||
Failed: total.Failed,
|
||||
}
|
||||
}
|
||||
|
||||
for _, finished := range lastDay {
|
||||
counts := byTarget[finished.TargetID]
|
||||
counts.DeliveredLast24Hours = finished.Delivered
|
||||
counts.FailedLast24Hours = finished.Failed
|
||||
byTarget[finished.TargetID] = counts
|
||||
}
|
||||
|
||||
return byTarget, nil
|
||||
}
|
||||
|
||||
// archiveFileView describes a database target's archive files from
|
||||
// their metadata alone; the archive is never opened. It names the file
|
||||
// an event received at now goes to. The files are found by the name
|
||||
// the archive writer uses, so they follow a rename of the webhook or
|
||||
// the target.
|
||||
func (h *Handlers) archiveFileView(
|
||||
webhook *database.Webhook, target *database.Target, now time.Time,
|
||||
) *ArchiveFileView {
|
||||
current := archive.PathAt(h.dbMgr, webhook, target, now)
|
||||
|
||||
view := &ArchiveFileView{Name: filepath.Base(current)}
|
||||
|
||||
_, statErr := os.Stat(current)
|
||||
if errors.Is(statErr, fs.ErrNotExist) {
|
||||
view.Note = "not created yet"
|
||||
}
|
||||
|
||||
files, err := archive.Stat(
|
||||
archive.Path(h.dbMgr, webhook, target),
|
||||
)
|
||||
|
||||
switch {
|
||||
case errors.Is(err, fs.ErrNotExist):
|
||||
// No files: no size or last write to show.
|
||||
case err != nil:
|
||||
return h.archiveUnreadable(view, target, err)
|
||||
default:
|
||||
view.Files = files.Files
|
||||
view.Size = humanize.Bytes(uint64(files.Size)) //nolint:gosec // never negative
|
||||
view.Written = humanize.Time(files.Written)
|
||||
view.WrittenUTC = files.Written.UTC().Format(time.DateTime) + " UTC"
|
||||
}
|
||||
|
||||
return view
|
||||
}
|
||||
|
||||
// archiveUnreadable logs why a database target's archive files could
|
||||
// not be described, and returns view saying so.
|
||||
func (h *Handlers) archiveUnreadable(
|
||||
view *ArchiveFileView, target *database.Target, err error,
|
||||
) *ArchiveFileView {
|
||||
h.log.Error(
|
||||
"failed to read archive file metadata",
|
||||
"target_id", target.ID,
|
||||
"error", err,
|
||||
)
|
||||
|
||||
view.Note = "could not be read"
|
||||
|
||||
return view
|
||||
}
|
||||
Reference in New Issue
Block a user