Bound the event log's rendered bodies in the query (closes #135)
All checks were successful
check / check (push) Successful in 3m0s
All checks were successful
check / check (push) Successful in 3m0s
The event log rendered stored bodies untruncated. Since buffered rendering landed (#123) that became resident memory per concurrent viewer, up to tens of MB, driven by payloads unauthenticated clients supply to the public receiver. Bound in the query rather than the template, via substr(cast(body as blob), 1, ?) plus length(cast(body as blob)), so an oversized body never becomes a Go string at all. Adds an EventLogView projection carrying the true byte count, and trims a partial UTF-8 tail without rewriting bodies that are merely invalid UTF-8. Independently reviewed. The generated SQL was dumped under GORM DryRun to confirm the cap is a bound parameter, both casts are present, and no other path selects the full column; soft-delete scope, ordering and pagination are unchanged. Correction to the PR body: its quoted mutation output was produced by removing the bound from eventLogColumns, not by raising the cap to 1<<30 as the text claimed. The reviewer reproduced the real mutation and confirmed the tests do catch removal of the bound. Follow-up #157 restores in-app retrieval of bodies above the cap.
This commit was merged in pull request #158.
This commit is contained in:
258
internal/handlers/event_log_view_test.go
Normal file
258
internal/handlers/event_log_view_test.go
Normal file
@@ -0,0 +1,258 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm/clause"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/handlers"
|
||||
"sneak.berlin/go/webhooker/internal/session"
|
||||
)
|
||||
|
||||
// bodyCap is the number of body bytes the event log page is
|
||||
// allowed to render for one event.
|
||||
const bodyCap = handlers.MaxRenderedBodyBytesForTest
|
||||
|
||||
// snowman is a three-byte rune, so a body of them straddles the
|
||||
// byte-wise cut: bodyCap is not a multiple of three.
|
||||
const snowman = "☃"
|
||||
|
||||
// seedEventWithBody records one event with the given body in the
|
||||
// webhook's own database.
|
||||
func seedEventWithBody(
|
||||
t *testing.T,
|
||||
dbMgr *database.WebhookDBManager,
|
||||
webhookID string,
|
||||
body string,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
webhookDB, err := dbMgr.GetDB(webhookID)
|
||||
require.NoError(t, err)
|
||||
|
||||
event := &database.Event{
|
||||
WebhookID: webhookID,
|
||||
Method: http.MethodPost,
|
||||
Body: body,
|
||||
ContentType: "application/octet-stream",
|
||||
}
|
||||
|
||||
require.NoError(t, webhookDB.Omit(
|
||||
clause.Associations,
|
||||
).Create(event).Error)
|
||||
}
|
||||
|
||||
// seedAndProject stores one body and returns the projection the
|
||||
// event log page would be handed for it.
|
||||
func seedAndProject(
|
||||
t *testing.T,
|
||||
body string,
|
||||
) handlers.EventLogView {
|
||||
t.Helper()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
dbMgr *database.WebhookDBManager
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
wh := seedWebhook(t, db)
|
||||
seedEventWithBody(t, dbMgr, wh.ID, body)
|
||||
|
||||
views := h.LoadEventLogViewsForTest(
|
||||
httptest.NewRecorder(), *wh, 1,
|
||||
)
|
||||
require.Len(t, views, 1)
|
||||
|
||||
return views[0]
|
||||
}
|
||||
|
||||
// TestHandleSourceLogs_BoundsOversizeBody proves the rendered
|
||||
// page is bounded by the cap rather than by the stored payload:
|
||||
// the body here is 64 times the cap, and the ingest path would
|
||||
// accept twice as much again.
|
||||
func TestHandleSourceLogs_BoundsOversizeBody(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
dbMgr *database.WebhookDBManager
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
const (
|
||||
sentinel = "TAIL-SENTINEL-1f4a9c"
|
||||
storedBytes = 512 * 1024
|
||||
)
|
||||
|
||||
wh := seedWebhook(t, db)
|
||||
seedEventWithBody(
|
||||
t, dbMgr, wh.ID,
|
||||
strings.Repeat("A", storedBytes-len(sentinel))+sentinel,
|
||||
)
|
||||
|
||||
page := renderSourceLogsPage(t, h, sess, wh.ID)
|
||||
|
||||
// Nothing past the cap reaches the page, and the whole page
|
||||
// stays far below the stored body it is reporting on.
|
||||
assert.NotContains(t, page, sentinel)
|
||||
assert.Less(t, len(page), 4*bodyCap)
|
||||
|
||||
// The marker states the true stored size, not the cut one.
|
||||
assert.Contains(
|
||||
t, page,
|
||||
"showing "+strconv.Itoa(bodyCap)+
|
||||
" of "+strconv.Itoa(storedBytes)+" bytes",
|
||||
)
|
||||
}
|
||||
|
||||
// TestHandleSourceLogs_SmallBodyRendersWhole guards the other
|
||||
// side of the cap: a body under it is shown in full and carries
|
||||
// no truncation marker.
|
||||
func TestHandleSourceLogs_SmallBodyRendersWhole(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
dbMgr *database.WebhookDBManager
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
wh := seedWebhook(t, db)
|
||||
seedEventWithBody(t, dbMgr, wh.ID, `{"kept":"whole"}`)
|
||||
|
||||
page := renderSourceLogsPage(t, h, sess, wh.ID)
|
||||
|
||||
assert.Contains(t, page, ""kept"")
|
||||
assert.NotContains(t, page, "Body truncated for display")
|
||||
}
|
||||
|
||||
// TestEventLogView_CutMidRune proves a multi-byte rune severed
|
||||
// by the byte-wise cut is dropped rather than surfaced as a
|
||||
// mojibake tail.
|
||||
func TestEventLogView_CutMidRune(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
body := strings.Repeat(snowman, 4096)
|
||||
view := seedAndProject(t, body)
|
||||
|
||||
// bodyCap bytes hold bodyCap/3 whole snowmen and two bytes
|
||||
// of the next one; those two are dropped.
|
||||
whole := bodyCap / len(snowman)
|
||||
|
||||
assert.True(t, view.BodyTruncated)
|
||||
assert.Equal(t, int64(len(body)), view.BodyBytes)
|
||||
assert.Equal(t, strings.Repeat(snowman, whole), view.Body)
|
||||
assert.True(t, utf8.ValidString(view.Body))
|
||||
assert.LessOrEqual(t, len(view.Body), bodyCap)
|
||||
}
|
||||
|
||||
// TestEventLogView_BinaryBodyLeftAsStored proves a binary
|
||||
// payload is passed through byte for byte. Its tail is invalid
|
||||
// UTF-8 however the cut falls, so repairing it would misreport
|
||||
// what the sender delivered.
|
||||
func TestEventLogView_BinaryBodyLeftAsStored(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
raw := make([]byte, bodyCap+808)
|
||||
for i := range raw {
|
||||
// 0x80..0xBF: continuation bytes, never a rune start.
|
||||
raw[i] = 0x80 | byte(i%0x40)
|
||||
}
|
||||
|
||||
view := seedAndProject(t, string(raw))
|
||||
|
||||
assert.True(t, view.BodyTruncated)
|
||||
assert.Equal(t, int64(len(raw)), view.BodyBytes)
|
||||
assert.Equal(t, string(raw[:bodyCap]), view.Body)
|
||||
assert.False(t, utf8.ValidString(view.Body))
|
||||
}
|
||||
|
||||
// TestTrimPartialRune covers the distinction the cut repair
|
||||
// turns on: an incomplete but valid sequence is dropped, while
|
||||
// bytes that are merely invalid UTF-8 are left alone.
|
||||
func TestTrimPartialRune(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
in []byte
|
||||
want []byte
|
||||
}{{
|
||||
name: "complete ascii",
|
||||
in: []byte("abc"),
|
||||
want: []byte("abc"),
|
||||
}, {
|
||||
name: "complete multibyte",
|
||||
in: []byte("ab" + snowman),
|
||||
want: []byte("ab" + snowman),
|
||||
}, {
|
||||
name: "two byte rune cut",
|
||||
in: []byte{'a', 0xC3},
|
||||
want: []byte{'a'},
|
||||
}, {
|
||||
name: "three byte rune cut after one",
|
||||
in: []byte{'a', 0xE2},
|
||||
want: []byte{'a'},
|
||||
}, {
|
||||
name: "three byte rune cut after two",
|
||||
in: []byte{'a', 0xE2, 0x98},
|
||||
want: []byte{'a'},
|
||||
}, {
|
||||
name: "four byte rune cut",
|
||||
in: []byte{'a', 0xF0, 0x9F, 0x92}, // U+1F4A9 cut
|
||||
want: []byte{'a'},
|
||||
}, {
|
||||
name: "invalid start byte kept",
|
||||
in: []byte{'a', 0xFF},
|
||||
want: []byte{'a', 0xFF},
|
||||
}, {
|
||||
name: "orphan continuation bytes kept",
|
||||
in: []byte{0x80, 0x81, 0x82, 0x83, 0x84},
|
||||
want: []byte{0x80, 0x81, 0x82, 0x83, 0x84},
|
||||
}, {
|
||||
name: "truncated sequence followed by junk kept",
|
||||
in: []byte{0xE2, 0x98, 0xFF},
|
||||
want: []byte{0xE2, 0x98, 0xFF},
|
||||
}, {
|
||||
name: "empty",
|
||||
in: []byte{},
|
||||
want: []byte{},
|
||||
}}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert.Equal(
|
||||
t, tc.want,
|
||||
handlers.TrimPartialRuneForTest(tc.in),
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user