2 Commits
Author SHA1 Message Date
sneak ba6ce94860 Check the log charge against every code point (closes #172)
check / check (push) Successful in 3m17s
TestEncodedBytes_ChargesAtLeastWhatTheHandlersEmit now covers all of
Unicode, surrogates aside, in place of a dense range plus a sample.
Code points are logged 4,096 to a value, and each batch's summed
charge must cover the bytes the real handler writes for it, measured
as a line carrying the batch twice less one carrying it once. A
failing batch is then measured one code point at a time, so the
failure names how many are undercharged and the first of them.

Model: opus-5-5
2026-10-02 13:43:24 +00:00
clawbot 88b961c115 Keep each model's parent out of its JSON (closes #177)
check / check (push) Successful in 3m16s
Every reference from a model to the record it belongs to (a target's or entrypoint's webhook, a webhook's or API key's user, an event's webhook and entrypoint, a delivery's event and target, a delivery result's delivery) is now tagged json:"-", so a preloaded model can be marshalled without the encoder recursing between parent and child. References to child records stay. Tests marshal each of the nine references set, preloaded where it matters, and check the parent's id is absent, so deleting or restoring any one tag fails a test; preloading still fills each reference.

Model: opus-5-5
2026-10-02 15:43:06 +02:00
9 changed files with 239 additions and 75 deletions

No files matched your search

+3 -2
View File
@@ -15,6 +15,7 @@ type APIKey struct {
Description string `json:"description"` Description string `json:"description"`
LastUsedAt *time.Time `json:"lastUsedAt,omitempty"` LastUsedAt *time.Time `json:"lastUsedAt,omitempty"`
// Relations // Relations. No model marshals the record it belongs to:
User User `json:"user,omitzero"` // User.APIKeys leads back here, and the JSON could loop.
User User `json:"-"`
} }
+5 -3
View File
@@ -56,8 +56,10 @@ type Delivery struct {
// the index. // the index.
FinishedAt *time.Time `gorm:"index:idx_deliveries_status,priority:3" json:"finishedAt,omitempty"` FinishedAt *time.Time `gorm:"index:idx_deliveries_status,priority:3" json:"finishedAt,omitempty"`
// Relations // Relations. No model marshals the record it belongs to:
Event Event `json:"event,omitzero"` // Event.Deliveries and Target.Deliveries lead back here, and the
Target Target `json:"target,omitzero"` // JSON could loop.
Event Event `json:"-"`
Target Target `json:"-"`
DeliveryResults []DeliveryResult `json:"deliveryResults,omitempty"` DeliveryResults []DeliveryResult `json:"deliveryResults,omitempty"`
} }
+3 -2
View File
@@ -23,6 +23,7 @@ type DeliveryResult struct {
Error string `json:"error,omitempty"` Error string `json:"error,omitempty"`
Duration int64 `json:"durationMs"` // Duration in milliseconds Duration int64 `json:"durationMs"` // Duration in milliseconds
// Relations // Relations. No model marshals the record it belongs to:
Delivery Delivery `json:"delivery,omitzero"` // Delivery.DeliveryResults leads back here, and the JSON could loop.
Delivery Delivery `json:"-"`
} }
+3 -2
View File
@@ -15,6 +15,7 @@ type Entrypoint struct {
Description string `json:"description"` Description string `json:"description"`
Active bool `gorm:"default:true" json:"active"` Active bool `gorm:"default:true" json:"active"`
// Relations // Relations. No model marshals the record it belongs to:
Webhook Webhook `json:"webhook,omitzero"` // Webhook.Entrypoints leads back here, and the JSON could loop.
Webhook Webhook `json:"-"`
} }
+4 -3
View File
@@ -44,8 +44,9 @@ type Event struct {
// kept as the record of where the copy came from either way. // kept as the record of where the copy came from either way.
ResubmittedFromID *string `gorm:"type:uuid;index" json:"resubmittedFromId,omitempty"` ResubmittedFromID *string `gorm:"type:uuid;index" json:"resubmittedFromId,omitempty"`
// Relations // Relations. No model marshals the record it belongs to, so
Webhook Webhook `json:"webhook,omitzero"` // Webhook and Entrypoint are left out of the JSON.
Entrypoint Entrypoint `json:"entrypoint,omitzero"` Webhook Webhook `json:"-"`
Entrypoint Entrypoint `json:"-"`
Deliveries []Delivery `json:"deliveries,omitempty"` Deliveries []Delivery `json:"deliveries,omitempty"`
} }
+126
View File
@@ -0,0 +1,126 @@
package database_test
import (
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/webhooker/internal/database"
)
// TestPreloadedModelsMarshalWithoutTheirParent pins that a child's
// reference to the record it belongs to is left out of the JSON, so a
// webhook and its targets cannot marshal each other in a loop, and that
// GORM still preloads that reference, since it ignores json tags.
func TestPreloadedModelsMarshalWithoutTheirParent(t *testing.T) {
t.Parallel()
db := startedTestDB(t)
stored := database.Webhook{
UserID: uuid.New().String(),
Name: testWebhookName,
Entrypoints: []database.Entrypoint{{Path: uuid.New().String()}},
Targets: []database.Target{{
Name: "log",
Type: database.TargetTypeLog,
}},
}
require.NoError(t, db.Create(&stored).Error)
entrypointID := stored.Entrypoints[0].ID
targetID := stored.Targets[0].ID
var webhook database.Webhook
require.NoError(t, db.
Preload("Entrypoints.Webhook").
Preload("Targets.Webhook").
First(&webhook, "id = ?", stored.ID).Error)
require.Len(t, webhook.Entrypoints, 1)
require.Len(t, webhook.Targets, 1)
assert.Equal(t, stored.ID, webhook.Entrypoints[0].Webhook.ID)
assert.Equal(t, stored.ID, webhook.Targets[0].Webhook.ID)
encoded := marshalModel(t, webhook)
assert.Contains(t, encoded, entrypointID)
assert.Contains(t, encoded, targetID)
// Each child holds the parent's id as its webhookId, so the parent
// is looked for by its own id field.
parentIDField := `"id":"` + stored.ID + `"`
assert.NotContains(t, marshalModel(t, webhook.Entrypoints[0]), parentIDField)
assert.NotContains(t, marshalModel(t, webhook.Targets[0]), parentIDField)
var target database.Target
require.NoError(t, db.
Preload("Webhook").
First(&target, "id = ?", targetID).Error)
assert.Equal(t, stored.ID, target.Webhook.ID)
encoded = marshalModel(t, target)
assert.Contains(t, encoded, stored.ID)
assert.NotContains(t, encoded, parentIDField)
}
// TestModelsMarshalWithoutTheirParent covers the other references to a
// parent: each model is built with its parent set, and the parent's id
// must not appear in the JSON.
func TestModelsMarshalWithoutTheirParent(t *testing.T) {
t.Parallel()
parent := database.BaseModel{ID: uuid.New().String()}
cases := []struct {
name string
model any
}{
{
name: "Webhook.User",
model: database.Webhook{User: database.User{BaseModel: parent}},
},
{
name: "APIKey.User",
model: database.APIKey{User: database.User{BaseModel: parent}},
},
{
name: "Delivery.Event",
model: database.Delivery{Event: database.Event{BaseModel: parent}},
},
{
name: "Delivery.Target",
model: database.Delivery{Target: database.Target{BaseModel: parent}},
},
{
name: "DeliveryResult.Delivery",
model: database.DeliveryResult{
Delivery: database.Delivery{BaseModel: parent},
},
},
{
name: "Event.Webhook",
model: database.Event{Webhook: database.Webhook{BaseModel: parent}},
},
{
name: "Event.Entrypoint",
model: database.Event{
Entrypoint: database.Entrypoint{BaseModel: parent},
},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
assert.NotContains(t, marshalModel(t, tc.model), parent.ID)
})
}
}
+3 -2
View File
@@ -34,7 +34,8 @@ type Target struct {
MaxRetries int `json:"maxRetries,omitempty"` MaxRetries int `json:"maxRetries,omitempty"`
MaxQueueSize int `json:"maxQueueSize,omitempty"` MaxQueueSize int `json:"maxQueueSize,omitempty"`
// Relations // Relations. No model marshals the record it belongs to:
Webhook Webhook `json:"webhook,omitzero"` // Webhook.Targets leads back here, and the JSON could loop.
Webhook Webhook `json:"-"`
Deliveries []Delivery `json:"deliveries,omitempty"` Deliveries []Delivery `json:"deliveries,omitempty"`
} }
+3 -2
View File
@@ -66,8 +66,9 @@ type Webhook struct {
// must equal DefaultRetentionDays. // must equal DefaultRetentionDays.
RetentionDays int `gorm:"default:30" json:"retentionDays"` RetentionDays int `gorm:"default:30" json:"retentionDays"`
// Relations // Relations. No model marshals the record it belongs to:
User User `json:"user,omitzero"` // User.Webhooks leads back here, and the JSON could loop.
User User `json:"-"`
Entrypoints []Entrypoint `json:"entrypoints,omitempty"` Entrypoints []Entrypoint `json:"entrypoints,omitempty"`
Targets []Target `json:"targets,omitempty"` Targets []Target `json:"targets,omitempty"`
} }
+89 -59
View File
@@ -5,6 +5,7 @@ import (
"io" "io"
"log/slog" "log/slog"
"strings" "strings"
"sync"
"testing" "testing"
"unicode/utf8" "unicode/utf8"
@@ -18,15 +19,11 @@ import (
// width. // width.
const budget = 64 const budget = 64
// sampleRunes is how many runes wide the values in the charge test // batchRunes is how many consecutive code points the charge test logs
// are. The handlers add a constant per field — a pair of quotes when // in one value. Logging every code point on its own line is too slow
// the value needs quoting — so the per-rune charge is only visible // for the suite under the race detector; 4,096 at a time is 272 lines
// once it is amortised over a run of them. // per handler for all of Unicode.
const sampleRunes = 64 const batchRunes = 4096
// quotingSlack is that constant: the pair of quotes a handler adds to
// a value that needs them and omits from one that does not.
const quotingSlack = 2
// newHandlers are the two handlers internal/logger can install. Time // newHandlers are the two handlers internal/logger can install. Time
// is dropped so a line's width is a function of its value alone — // is dropped so a line's width is a function of its value alone —
@@ -66,46 +63,42 @@ func renderedWidth(
return buf.Len() return buf.Len()
} }
// chargeTestRunes is the set of code points the charge test measures: // emittedBytes is what a handler writes for the runes of s alone: the
// every rune in the first two planes' worth of the BMP that the // width of a line carrying s twice, less that of a line carrying it
// handlers are most likely to treat specially, the separators that // once. Both values hold the same runes, so the text handler quotes
// only slog's JSON handler escapes, and a stratified sample across // both or neither, and the quotes cancel along with everything else on
// the rest of Unicode so the astral charge is exercised on more than // the line.
// one hand-picked rune. func emittedBytes(
func chargeTestRunes() []rune { newHandler func(io.Writer) slog.Handler,
const ( s string,
denseCeiling = 0x800 ) int {
stride = 1021 return renderedWidth(newHandler, s+s) - renderedWidth(newHandler, s)
surrogateLo = 0xD800 }
surrogateHi = 0xDFFF
)
var runes []rune // firstUndercharged returns the first rune in s that the handler
// writes in more bytes than EncodedBytes charges for it, and how many
// runes in s are undercharged that way. It measures one rune per line,
// so the charge test calls it only on a batch that has already failed,
// to name the code points rather than just their range.
func firstUndercharged(
newHandler func(io.Writer) slog.Handler,
s string,
) (rune, int) {
first, count := rune(-1), 0
keep := func(r rune) { for _, r := range s {
if r >= surrogateLo && r <= surrogateHi { if emittedBytes(newHandler, string(r)) <= logfield.EncodedBytes(r) {
return continue
} }
runes = append(runes, r) if count == 0 {
first = r
} }
for r := range rune(denseCeiling) { count++
keep(r)
} }
for _, r := range []rune{ return first, count
0x2028, 0x2029, 0x200B, 0x4E00, 0xE000, 0xFFFD,
0x1000C, 0x1F600, 0xE0001, 0x10FFFF,
} {
keep(r)
}
for r := rune(denseCeiling); r <= utf8.MaxRune; r += stride {
keep(r)
}
return runes
} }
// TestEncodedBytes_ChargesAtLeastWhatTheHandlersEmit is the property // TestEncodedBytes_ChargesAtLeastWhatTheHandlersEmit is the property
@@ -114,33 +107,70 @@ func chargeTestRunes() []rune {
// how a stated ceiling becomes false without any test noticing, so // how a stated ceiling becomes false without any test noticing, so
// the charge is measured against what the handlers actually write // the charge is measured against what the handlers actually write
// rather than against the escaping rules as read. // rather than against the escaping rules as read.
//
// Every code point is covered, batchRunes at a time: each batch's
// summed charge must cover what the handler writes for the whole
// batch. A sum can hide an undercharged rune behind overcharged ones
// in the same batch, which is the price of fitting all of Unicode
// into the suite.
func TestEncodedBytes_ChargesAtLeastWhatTheHandlersEmit(t *testing.T) { func TestEncodedBytes_ChargesAtLeastWhatTheHandlersEmit(t *testing.T) {
t.Parallel() t.Parallel()
var batches []string
for lo := rune(0); lo <= utf8.MaxRune; lo += batchRunes {
var batch strings.Builder
for r := lo; r < lo+batchRunes; r++ {
// Surrogate halves are not runes a string can carry.
if utf8.ValidRune(r) {
batch.WriteRune(r)
}
}
batches = append(batches, batch.String())
}
// What EncodedBytes charges for each batch. Under -race -cover this
// takes longer than logging the batches, so it is worked out once,
// by whichever handler finishes logging first, while the other is
// still logging.
charged := sync.OnceValue(func() []int {
costs := make([]int, len(batches))
for i, batch := range batches {
for _, r := range batch {
costs[i] += logfield.EncodedBytes(r)
}
}
return costs
})
for name, newHandler := range newHandlers() { for name, newHandler := range newHandlers() {
t.Run(name, func(t *testing.T) { t.Run(name, func(t *testing.T) {
t.Parallel() t.Parallel()
// 'a' is a printable ASCII rune, charged exactly one emitted := make([]int, len(batches))
// byte, so it is the zero point the other runes are for i, batch := range batches {
// measured against. emitted[i] = emittedBytes(newHandler, batch)
base := renderedWidth( }
newHandler, strings.Repeat("a", sampleRunes),
)
for _, r := range chargeTestRunes() { for i, cost := range charged() {
got := renderedWidth( if emitted[i] <= cost {
newHandler, continue
strings.Repeat(string(r), sampleRunes), }
)
charged := sampleRunes *
(logfield.EncodedBytes(r) - 1)
require.LessOrEqual( lo := rune(i * batchRunes)
t, got-base, charged+quotingSlack, first, count := firstUndercharged(
"U+%04X costs more on the line than "+ newHandler, batches[i],
"EncodedBytes charges for it", )
r, t.Errorf(
"U+%04X to U+%04X emit %d bytes but are "+
"charged %d; %d of them cost more than "+
"EncodedBytes charges, the first U+%04X",
lo, lo+batchRunes-1, emitted[i], cost,
count, first,
) )
} }
}) })