3 Commits
Author SHA1 Message Date
sneak 13f881007a Rotate a database target's archive monthly, daily or hourly (closes #379)
check / check (push) Successful in 3m42s
A database target's rotation (none, monthly, daily or hourly) puts the
UTC period of each event's receive time in its archive file name, so
each file holds exactly its period's events. It is on the new webhook
page, the add target form and the target edit form, and shown in the
target list.

Renames move every one of a target's files, the sweep prunes every file
and deletes a rotated file it leaves empty, Download exports every file
oldest first with each row's period, and the target list names the
current file and totals the size of all of them.

Model: opus-5-5
2026-10-02 23:51:42 +00:00
clawbot 22fa502638 Remove an http target's max_queue_size (closes #477)
check / check (push) Successful in 3m18s
An http target's max_queue_size was stored and shown in the target list as "Max Queue Size", but nothing in the delivery engine read it, so an operator who set it expecting deliveries to be bounded got nothing. It is removed from the target, the target list and the README's target table; neither form had a field for it. Nothing checks for a leftover value. An existing database keeps its old column, which is no longer read.

Model: opus-5-5
2026-10-03 01:39:13 +02:00
clawbot 6395210474 Show each page's own title in the browser tab (closes #117)
check / check (push) Successful in 3m17s
Every browser tab read "Webhooker": parsePageTemplate parsed each page before htmlheader.html, whose {{block "title"}} fallback then redefined the page's {{define "title"}}. The page file is now parsed last, so its title replaces the fallback (a later definition of a template name replaces an earlier one, and an empty one never does). A test renders every page template and checks its browser tab title.

Model: opus-5-5
2026-10-03 01:30:02 +02:00
11 changed files with 201 additions and 79 deletions

No files matched your search

-1
View File
@@ -1703,7 +1703,6 @@ events should be forwarded.
| `active` | boolean | Whether deliveries are enabled (default: true) |
| `config` | JSON text | Type-specific configuration |
| `max_retries` | integer | Total delivery attempts for `http` and `slack` targets, not retries on top of the first: 0 is a single fire-and-forget attempt with no retries and no circuit breaker, and a value of N makes N attempts in all, with exponential backoff and a per-target circuit breaker. Ignored by `database` and `log` targets |
| `max_queue_size` | integer | Stored and shown on the target's detail view, but not enforced anywhere yet: nothing in the delivery engine consults it. Queue depth is set by the two fixed 10,000-entry channels |
**Relations:** Belongs to Webhook. Has many Deliveries.
+1 -2
View File
@@ -31,8 +31,7 @@ type Target struct {
// For HTTP targets (max_retries=0 means fire-and-forget,
// >0 enables retries with backoff)
MaxRetries int `json:"maxRetries,omitempty"`
MaxQueueSize int `json:"maxQueueSize,omitempty"`
MaxRetries int `json:"maxRetries,omitempty"`
// Relations. No model marshals the record it belongs to:
// Webhook.Targets leads back here, and the JSON could loop.
-7
View File
@@ -173,13 +173,6 @@ func httpConfigFields(t *database.Target) []ConfigField {
fields = append(fields, maxRetriesField(t))
if t.MaxQueueSize > 0 {
fields = append(fields, ConfigField{
Label: "Max Queue Size",
Value: strconv.Itoa(t.MaxQueueSize),
})
}
return fields
}
+18 -22
View File
@@ -197,14 +197,12 @@ func TestNewTargetViews_Slack(t *testing.T) {
}
// TestNewTargetViews_SlackRetries proves a Slack target shows
// its retry count the same way an HTTP target does, and no
// queue size even when one is stored: delivery never reads it.
// its retry count the same way an HTTP target does.
func TestNewTargetViews_SlackRetries(t *testing.T) {
t.Parallel()
target := slackTarget()
target.MaxRetries = 2
target.MaxQueueSize = 100
view := viewFor(t, target)
@@ -226,8 +224,7 @@ func TestNewTargetViews_HTTP(t *testing.T) {
Config: `{"url":"` + viewExampleHook + `",` +
`"timeout":30,` +
`"headers":{"Authorization":"Bearer sekrit"}}`,
MaxRetries: 5,
MaxQueueSize: 100,
MaxRetries: 5,
})
fields := fieldMap(view.Config)
@@ -239,7 +236,6 @@ func TestNewTargetViews_HTTP(t *testing.T) {
"Timeout": "30s",
"Headers": "1 configured",
viewMaxRetries: "5",
"Max Queue Size": "100",
},
fields,
)
@@ -334,7 +330,7 @@ func TestNewTargetViews_Database(t *testing.T) {
t,
map[string]string{
"Archive Expiry": tc.want,
"Archive Rotation": "none",
"Archive Rotation": rotationNone,
},
fieldMap(view.Config),
)
@@ -347,29 +343,29 @@ func TestNewTargetViews_Database(t *testing.T) {
func TestNewTargetViews_DatabaseRotation(t *testing.T) {
t.Parallel()
tests := map[string]struct {
config string
want string
}{
"empty config": {config: "", want: "none"},
"empty rotation": {config: `{"rotation":""}`, want: "none"},
"none": {config: `{"rotation":"none"}`, want: "none"},
"monthly": {config: `{"rotation":"monthly"}`, want: "monthly"},
"daily": {config: `{"rotation":"daily"}`, want: "daily"},
"hourly": {config: `{"rotation":"hourly"}`, want: "hourly"},
// Each stored config, and the rotation the list shows for it.
tests := map[string]string{
"": rotationNone,
`{"rotation":""}`: rotationNone,
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
for _, rotation := range []string{
rotationNone, rotationMonthly, rotationDaily, rotationHourly,
} {
tests[`{"rotation":"`+rotation+`"}`] = rotation
}
for config, want := range tests {
t.Run(config, func(t *testing.T) {
t.Parallel()
view := viewFor(t, database.Target{
Type: database.TargetTypeDatabase,
Config: tc.config,
Config: config,
})
assert.Equal(
t, tc.want, fieldMap(view.Config)["Archive Rotation"],
t, want, fieldMap(view.Config)["Archive Rotation"],
)
})
}
@@ -424,7 +420,7 @@ func TestNewTargetViews_Unpresentable(t *testing.T) {
},
"invalid archive rotation": {
Type: database.TargetTypeDatabase,
Config: `{"rotation":"weekly"}`,
Config: weeklyConfig,
},
}
+1 -1
View File
@@ -211,7 +211,7 @@ func (x *ArchiveExport) WriteGzipJSON(
// Close ends the transactions of the files still open and closes
// their connections.
func (x *ArchiveExport) Close() error {
var errs []error
errs := make([]error, 0, len(x.files))
for _, f := range x.files {
errs = append(errs, f.close())
@@ -314,7 +314,7 @@ func TestArchiveExport_EveryFileOldestFirst(t *testing.T) {
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
// Written in an order that is not the export's.
for _, period := range []string{"2026-03-05", "", "2026-03-04-05", "2026-03"} {
for _, period := range []string{nextDayPeriod, "", hourPeriod, "2026-03"} {
require.NoError(t, w.WritePeriod(
delivery.ExportArchivedEvent{EventID: "in-" + period}, 0, period,
))
@@ -341,11 +341,11 @@ func TestArchiveExport_EveryFileOldestFirst(t *testing.T) {
}
assert.Equal(t,
[]string{"in-", "in-2026-03", "in-2026-03-04-05", "in-2026-03-05"},
[]string{"in-", "in-2026-03", "in-" + hourPeriod, "in-" + nextDayPeriod},
ids,
)
assert.Equal(t,
[]any{nil, "2026-03", "2026-03-04-05", "2026-03-05"}, periods,
[]any{nil, "2026-03", hourPeriod, nextDayPeriod}, periods,
)
assert.NotContains(t, events[0], "period",
"a row from the file without a period has no period")
@@ -13,6 +13,26 @@ import (
"sneak.berlin/go/webhooker/internal/delivery"
)
// The archive rotations, and the configs of a daily target and of one
// whose rotation is not one of the four.
const (
rotationNone = "none"
rotationMonthly = "monthly"
rotationDaily = "daily"
rotationHourly = "hourly"
dailyConfig = `{"rotation":"daily"}`
weeklyConfig = `{"rotation":"weekly"}`
)
// The periods the tests archive into most: two days, and an hour of
// the first.
const (
dayPeriod = "2026-03-04"
nextDayPeriod = "2026-03-05"
hourPeriod = "2026-03-04-05"
)
// periodPath returns the archive file for a period of the target whose
// file without a period is path.
func periodPath(path, period string) string {
@@ -57,30 +77,30 @@ func TestDeliverDatabase_RotatesAtEachPeriodBoundary(t *testing.T) {
periods [2]string
}{
{
"monthly", "monthly",
rotationMonthly, rotationMonthly,
time.Date(2026, 1, 31, 23, 59, 59, 0, time.UTC),
time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC),
[2]string{"2026-01", "2026-02"},
},
{
"daily", "daily",
rotationDaily, rotationDaily,
time.Date(2026, 3, 4, 23, 59, 59, 0, time.UTC),
time.Date(2026, 3, 5, 0, 0, 0, 0, time.UTC),
[2]string{"2026-03-04", "2026-03-05"},
[2]string{dayPeriod, nextDayPeriod},
},
{
// The same instants, received in a zone two hours ahead
// of UTC, where they fall on 5 March: the period is UTC's.
"daily in another zone", "daily",
"daily in another zone", rotationDaily,
time.Date(2026, 3, 5, 1, 59, 59, 0, berlin),
time.Date(2026, 3, 5, 2, 0, 0, 0, berlin),
[2]string{"2026-03-04", "2026-03-05"},
[2]string{dayPeriod, nextDayPeriod},
},
{
"hourly", "hourly",
rotationHourly, rotationHourly,
time.Date(2026, 3, 4, 5, 59, 59, 0, time.UTC),
time.Date(2026, 3, 4, 6, 0, 0, 0, time.UTC),
[2]string{"2026-03-04-05", "2026-03-04-06"},
[2]string{hourPeriod, "2026-03-04-06"},
},
}
@@ -104,11 +124,11 @@ func TestDeliverDatabase_RotatesAtEachPeriodBoundary(t *testing.T) {
})
}
t.Run("none", func(t *testing.T) {
t.Run(rotationNone, func(t *testing.T) {
t.Parallel()
env := setupArchiveTest(t)
tgt := env.seedDatabaseTarget(t, `{"rotation":"none"}`)
tgt := env.seedDatabaseTarget(t, `{"rotation":"`+rotationNone+`"}`)
first := env.deliverReceivedAt(t, tgt, cases[0].before)
second := env.deliverReceivedAt(t, tgt, cases[0].after)
@@ -125,7 +145,7 @@ func TestDeliverDatabase_EventWithoutReceiveTime(t *testing.T) {
t.Parallel()
env := setupArchiveTest(t)
tgt := env.seedDatabaseTarget(t, `{"rotation":"monthly"}`)
tgt := env.seedDatabaseTarget(t, `{"rotation":"`+rotationMonthly+`"}`)
path := env.archivePath(tgt)
before := time.Now().UTC().Format("2006-01")
@@ -156,12 +176,12 @@ func TestDeliverDatabase_RotationChangeKeepsOldFiles(t *testing.T) {
first := env.deliverReceivedAt(t, tgt, at)
tgt.Config = `{"rotation":"daily"}`
tgt.Config = dailyConfig
second := env.deliverReceivedAt(t, tgt, at)
assert.Equal(t, []string{first}, archivedEventIDs(t, path))
assert.Equal(t, []string{second},
archivedEventIDs(t, periodPath(path, "2026-03-04")))
archivedEventIDs(t, periodPath(path, dayPeriod)))
}
// TestArchiveSweep_PrunesEveryFile gives a daily target three files:
@@ -175,11 +195,11 @@ func TestArchiveSweep_PrunesEveryFile(t *testing.T) {
env := setupArchiveTest(t)
tgt := env.seedDatabaseTarget(
t, `{"expiry":"1h","rotation":"daily"}`,
t, `{"expiry":"1h","rotation":"`+rotationDaily+`"}`,
)
path := env.archivePath(tgt)
emptied := periodPath(path, "2026-03-04")
kept := periodPath(path, "2026-03-05")
emptied := periodPath(path, dayPeriod)
kept := periodPath(path, nextDayPeriod)
now := time.Now()
old := now.Add(-48 * time.Hour)
@@ -212,7 +232,7 @@ func TestRename_MovesEveryFile(t *testing.T) {
unrotated := env.deliverReceivedAt(t, tgt, day)
tgt.Config = `{"rotation":"daily"}`
tgt.Config = dailyConfig
first := env.deliverReceivedAt(t, tgt, day)
second := env.deliverReceivedAt(t, tgt, day.Add(24*time.Hour))
@@ -224,17 +244,17 @@ func TestRename_MovesEveryFile(t *testing.T) {
for _, old := range []string{
oldPath,
periodPath(oldPath, "2026-03-04"),
periodPath(oldPath, "2026-03-05"),
periodPath(oldPath, dayPeriod),
periodPath(oldPath, nextDayPeriod),
} {
assert.NoFileExists(t, old)
}
assert.Equal(t, []string{unrotated}, archivedEventIDs(t, newPath))
assert.Equal(t, []string{first},
archivedEventIDs(t, periodPath(newPath, "2026-03-04")))
archivedEventIDs(t, periodPath(newPath, dayPeriod)))
assert.Equal(t, []string{second},
archivedEventIDs(t, periodPath(newPath, "2026-03-05")))
archivedEventIDs(t, periodPath(newPath, nextDayPeriod)))
third := env.deliverReceivedAt(t, tgt, day.Add(48*time.Hour))
assert.Equal(t, []string{third},
@@ -248,7 +268,7 @@ func TestRename_NeverReplacesARotatedFile(t *testing.T) {
t.Parallel()
env := setupArchiveTest(t)
tgt := env.seedDatabaseTarget(t, `{"rotation":"daily"}`)
tgt := env.seedDatabaseTarget(t, dailyConfig)
oldPath := env.archivePath(tgt)
day := time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC)
@@ -258,7 +278,7 @@ func TestRename_NeverReplacesARotatedFile(t *testing.T) {
newPath := filepath.Join(
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
)
planted := periodPath(newPath, "2026-03-05")
planted := periodPath(newPath, nextDayPeriod)
require.NoError(t, os.WriteFile(planted, []byte("planted"), 0o600))
require.ErrorIs(
@@ -266,9 +286,9 @@ func TestRename_NeverReplacesARotatedFile(t *testing.T) {
delivery.ErrArchiveNameTaken,
)
assert.FileExists(t, periodPath(oldPath, "2026-03-04"))
assert.FileExists(t, periodPath(oldPath, "2026-03-05"))
assert.NoFileExists(t, periodPath(newPath, "2026-03-04"))
assert.FileExists(t, periodPath(oldPath, dayPeriod))
assert.FileExists(t, periodPath(oldPath, nextDayPeriod))
assert.NoFileExists(t, periodPath(newPath, dayPeriod))
}
// TestStatArchive_EveryFile proves StatArchive counts and adds up every
@@ -280,7 +300,7 @@ func TestStatArchive_EveryFile(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "archive-wh.db")
files := []string{
path, periodPath(path, "2026-03"), periodPath(path, "2026-03-04-05"),
path, periodPath(path, "2026-03"), periodPath(path, hourPeriod),
}
written := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)
@@ -322,11 +342,11 @@ func TestArchivePathAt(t *testing.T) {
at := time.Date(2026, 10, 1, 19, 30, 0, 0, time.UTC)
cases := map[string]string{
"": "",
`{"rotation":"none"}`: "",
`{"rotation":"monthly"}`: "-2026-10",
`{"rotation":"daily"}`: "-2026-10-01",
`{"rotation":"hourly"}`: "-2026-10-01-19",
"": "",
`{"rotation":"` + rotationNone + `"}`: "",
`{"rotation":"` + rotationMonthly + `"}`: "-2026-10",
dailyConfig: "-2026-10-01",
`{"rotation":"` + rotationHourly + `"}`: "-2026-10-01-19",
}
for config, period := range cases {
@@ -347,7 +367,7 @@ func TestArchivePathAt(t *testing.T) {
}
_, err := delivery.ArchivePathAt(dbMgr, webhook, &database.Target{
Config: `{"rotation":"weekly"}`,
Config: weeklyConfig,
}, at)
require.Error(t, err)
}
@@ -357,7 +377,9 @@ func TestArchivePathAt(t *testing.T) {
func TestValidateArchiveRotation(t *testing.T) {
t.Parallel()
for _, ok := range []string{"", "none", "monthly", "daily", "hourly"} {
for _, ok := range []string{
"", rotationNone, rotationMonthly, rotationDaily, rotationHourly,
} {
require.NoError(t, delivery.ValidateArchiveRotation(ok), ok)
}
+2 -2
View File
@@ -208,7 +208,7 @@ func TestNewTargetConfigForm(t *testing.T) {
})
require.NoError(t, err)
assert.Equal(t, "720h", form.Expiry)
assert.Equal(t, "daily", form.Rotation)
assert.Equal(t, rotationDaily, form.Rotation)
form, err = delivery.NewTargetConfigForm(&database.Target{
Type: database.TargetTypeLog,
@@ -251,7 +251,7 @@ func TestNewTargetConfigForm_UnreadableConfigErrors(t *testing.T) {
},
{
Type: database.TargetTypeDatabase,
Config: `{"rotation":"weekly"}`,
Config: weeklyConfig,
},
{Type: database.TargetType("nope")},
}
+6 -3
View File
@@ -16,6 +16,9 @@ import (
"sneak.berlin/go/webhooker/internal/delivery"
)
// rotationNone is the archive rotation that keeps one file.
const rotationNone = "none"
// rotationShown returns the archive rotations the webhook page's
// target list shows.
func rotationShown(
@@ -38,7 +41,7 @@ func TestArchiveRotationChoices(t *testing.T) {
env := setupSourceTest(t)
for _, rotation := range []string{"none", "monthly", "daily", "hourly"} {
for _, rotation := range []string{rotationNone, "monthly", "daily", "hourly"} {
t.Run(rotation, func(t *testing.T) {
t.Parallel()
@@ -95,8 +98,8 @@ func TestArchiveRotationEditStartsOnNone(t *testing.T) {
)
page := targetEditPage(t, env, webhook.ID, target.ID)
assert.Equal(t, []string{"none"}, selectedIn(page, "rotation"))
assert.Equal(t, []string{"none"}, rotationShown(t, env, webhook.ID))
assert.Equal(t, []string{rotationNone}, selectedIn(page, "rotation"))
assert.Equal(t, []string{rotationNone}, rotationShown(t, env, webhook.ID))
}
// TestArchiveRotationRefused proves a rotation that is not one of the
+8 -5
View File
@@ -110,22 +110,25 @@ type Handlers struct {
// parsePageTemplate parses a page-specific template set from the
// embedded FS. Each page template is combined with the shared
// base, htmlheader, navbar and notice templates, and with any further
// files the page includes. The page file must be listed first so that
// its root action ({{template "base" .}}) becomes the template set's
// entry point.
// files the page includes. The set is named after the page file, so
// the page's root action ({{template "base" .}}) is its entry point.
//
// The page file is parsed last because a later definition of a name
// replaces an earlier one: the page's {{define "title"}} must replace
// the {{block "title"}} fallback in htmlheader.html.
func parsePageTemplate(
pageFile string, included ...string,
) *template.Template {
files := append([]string{
pageFile,
"base.html",
"htmlheader.html",
"navbar.html",
"notice.html",
}, included...)
files = append(files, pageFile)
return template.Must(
template.ParseFS(templates.Templates, files...),
template.New(pageFile).ParseFS(templates.Templates, files...),
)
}
+107
View File
@@ -0,0 +1,107 @@
package handlers_test
import (
"html/template"
"net/http"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/handlers"
"sneak.berlin/go/webhooker/internal/session"
"sneak.berlin/go/webhooker/templates"
)
// TestEveryPageRendersItsOwnTitle renders each page template and checks
// the browser tab title is the one the page declares, not the
// "Webhooker" fallback in htmlheader.html. A page that fails to render
// shows the error page's title instead, and fails here too.
func TestEveryPageRendersItsOwnTitle(t *testing.T) {
t.Parallel()
var h *handlers.Handlers
var sess *session.Session
app := newTestApp(t, &h, &sess)
app.RequireStart()
t.Cleanup(app.RequireStop)
// A pointer, as in the handlers: some pages call
// Webhook.RetentionLabel, a pointer method.
webhook := &database.Webhook{Name: "orders", RetentionDays: 14}
webhook.ID = testWebhookID
pages := []struct {
page string
data map[string]any
title string
}{
{"login.html", map[string]any{}, "Login - Webhooker"},
{"profile.html", map[string]any{}, "Profile - Webhooker"},
{"settings.html", map[string]any{}, "Settings - Webhooker"},
{"sources_list.html", map[string]any{}, "Webhooks - Webhooker"},
{"sources_new.html", map[string]any{}, "New Webhook - Webhooker"},
{
"source_detail.html",
map[string]any{dataKeyWebhook: webhook},
"orders - Webhooker",
},
{
"source_edit.html",
map[string]any{dataKeyWebhook: webhook},
"Edit orders - Webhooker",
},
{
"source_logs.html",
map[string]any{dataKeyWebhook: webhook, "TotalEvents": int64(0)},
"Full Event Log - orders - Webhooker",
},
{
"event_detail.html",
map[string]any{dataKeyWebhook: webhook},
"Event - orders - Webhooker",
},
{
"target_edit.html",
map[string]any{
dataKeyWebhook: webhook,
"Target": map[string]any{"Name": "alerts", "Type": "slack"},
},
"Edit alerts - Webhooker",
},
{
"error.html",
map[string]any{"StatusText": http.StatusText(http.StatusNotFound)},
"Not Found - Webhooker",
},
}
for _, p := range pages {
body := renderPage(t, h, sess, p.page, p.data)
_, afterOpen, _ := strings.Cut(body, "<title>")
title, _, _ := strings.Cut(afterOpen, "</title>")
assert.Equal(t, p.title, title, p.page)
}
}
// TestTitleFallbackIsWebhooker checks the title htmlheader.html gives a
// page that declares none. Every page declares one, so it is checked on
// htmlheader.html alone.
func TestTitleFallbackIsWebhooker(t *testing.T) {
t.Parallel()
header := template.Must(
template.ParseFS(templates.Templates, "htmlheader.html"),
)
var buf strings.Builder
require.NoError(t, header.ExecuteTemplate(&buf, "htmlheader", nil))
assert.Contains(t, buf.String(), "<title>Webhooker</title>")
}