1 Commits
Author SHA1 Message Date
sneak 58d56cc91b Rotate a database target's archive monthly, daily or hourly (closes #379)
check / check (push) Failing after 1m9s
A database target's rotation (none, monthly, daily or hourly) puts the
UTC period of each event's receive time in its archive file name, so
each file holds exactly its period's events. It is on the new webhook
page, the add target form and the target edit form, and shown in the
target list.

Renames move every one of a target's files, the sweep prunes every file
and deletes a rotated file it leaves empty, Download exports every file
oldest first with each row's period, and the target list names the
current file and totals the size of all of them.

Model: opus-5-5
2026-10-02 23:41:42 +00:00
11 changed files with 76 additions and 198 deletions

No files matched your search

+1
View File
@@ -1703,6 +1703,7 @@ events should be forwarded.
| `active` | boolean | Whether deliveries are enabled (default: true) |
| `config` | JSON text | Type-specific configuration |
| `max_retries` | integer | Total delivery attempts for `http` and `slack` targets, not retries on top of the first: 0 is a single fire-and-forget attempt with no retries and no circuit breaker, and a value of N makes N attempts in all, with exponential backoff and a per-target circuit breaker. Ignored by `database` and `log` targets |
| `max_queue_size` | integer | Stored and shown on the target's detail view, but not enforced anywhere yet: nothing in the delivery engine consults it. Queue depth is set by the two fixed 10,000-entry channels |
**Relations:** Belongs to Webhook. Has many Deliveries.
+1
View File
@@ -32,6 +32,7 @@ type Target struct {
// For HTTP targets (max_retries=0 means fire-and-forget,
// >0 enables retries with backoff)
MaxRetries int `json:"maxRetries,omitempty"`
MaxQueueSize int `json:"maxQueueSize,omitempty"`
// Relations. No model marshals the record it belongs to:
// Webhook.Targets leads back here, and the JSON could loop.
+7
View File
@@ -173,6 +173,13 @@ func httpConfigFields(t *database.Target) []ConfigField {
fields = append(fields, maxRetriesField(t))
if t.MaxQueueSize > 0 {
fields = append(fields, ConfigField{
Label: "Max Queue Size",
Value: strconv.Itoa(t.MaxQueueSize),
})
}
return fields
}
+21 -17
View File
@@ -197,12 +197,14 @@ func TestNewTargetViews_Slack(t *testing.T) {
}
// TestNewTargetViews_SlackRetries proves a Slack target shows
// its retry count the same way an HTTP target does.
// its retry count the same way an HTTP target does, and no
// queue size even when one is stored: delivery never reads it.
func TestNewTargetViews_SlackRetries(t *testing.T) {
t.Parallel()
target := slackTarget()
target.MaxRetries = 2
target.MaxQueueSize = 100
view := viewFor(t, target)
@@ -225,6 +227,7 @@ func TestNewTargetViews_HTTP(t *testing.T) {
`"timeout":30,` +
`"headers":{"Authorization":"Bearer sekrit"}}`,
MaxRetries: 5,
MaxQueueSize: 100,
})
fields := fieldMap(view.Config)
@@ -236,6 +239,7 @@ func TestNewTargetViews_HTTP(t *testing.T) {
"Timeout": "30s",
"Headers": "1 configured",
viewMaxRetries: "5",
"Max Queue Size": "100",
},
fields,
)
@@ -330,7 +334,7 @@ func TestNewTargetViews_Database(t *testing.T) {
t,
map[string]string{
"Archive Expiry": tc.want,
"Archive Rotation": rotationNone,
"Archive Rotation": "none",
},
fieldMap(view.Config),
)
@@ -343,29 +347,29 @@ func TestNewTargetViews_Database(t *testing.T) {
func TestNewTargetViews_DatabaseRotation(t *testing.T) {
t.Parallel()
// Each stored config, and the rotation the list shows for it.
tests := map[string]string{
"": rotationNone,
`{"rotation":""}`: rotationNone,
tests := map[string]struct {
config string
want string
}{
"empty config": {config: "", want: "none"},
"empty rotation": {config: `{"rotation":""}`, want: "none"},
"none": {config: `{"rotation":"none"}`, want: "none"},
"monthly": {config: `{"rotation":"monthly"}`, want: "monthly"},
"daily": {config: `{"rotation":"daily"}`, want: "daily"},
"hourly": {config: `{"rotation":"hourly"}`, want: "hourly"},
}
for _, rotation := range []string{
rotationNone, rotationMonthly, rotationDaily, rotationHourly,
} {
tests[`{"rotation":"`+rotation+`"}`] = rotation
}
for config, want := range tests {
t.Run(config, func(t *testing.T) {
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
t.Parallel()
view := viewFor(t, database.Target{
Type: database.TargetTypeDatabase,
Config: config,
Config: tc.config,
})
assert.Equal(
t, want, fieldMap(view.Config)["Archive Rotation"],
t, tc.want, fieldMap(view.Config)["Archive Rotation"],
)
})
}
@@ -420,7 +424,7 @@ func TestNewTargetViews_Unpresentable(t *testing.T) {
},
"invalid archive rotation": {
Type: database.TargetTypeDatabase,
Config: weeklyConfig,
Config: `{"rotation":"weekly"}`,
},
}
+1 -1
View File
@@ -211,7 +211,7 @@ func (x *ArchiveExport) WriteGzipJSON(
// Close ends the transactions of the files still open and closes
// their connections.
func (x *ArchiveExport) Close() error {
errs := make([]error, 0, len(x.files))
var errs []error
for _, f := range x.files {
errs = append(errs, f.close())
@@ -314,7 +314,7 @@ func TestArchiveExport_EveryFileOldestFirst(t *testing.T) {
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
// Written in an order that is not the export's.
for _, period := range []string{nextDayPeriod, "", hourPeriod, "2026-03"} {
for _, period := range []string{"2026-03-05", "", "2026-03-04-05", "2026-03"} {
require.NoError(t, w.WritePeriod(
delivery.ExportArchivedEvent{EventID: "in-" + period}, 0, period,
))
@@ -341,11 +341,11 @@ func TestArchiveExport_EveryFileOldestFirst(t *testing.T) {
}
assert.Equal(t,
[]string{"in-", "in-2026-03", "in-" + hourPeriod, "in-" + nextDayPeriod},
[]string{"in-", "in-2026-03", "in-2026-03-04-05", "in-2026-03-05"},
ids,
)
assert.Equal(t,
[]any{nil, "2026-03", hourPeriod, nextDayPeriod}, periods,
[]any{nil, "2026-03", "2026-03-04-05", "2026-03-05"}, periods,
)
assert.NotContains(t, events[0], "period",
"a row from the file without a period has no period")
@@ -13,26 +13,6 @@ import (
"sneak.berlin/go/webhooker/internal/delivery"
)
// The archive rotations, and the configs of a daily target and of one
// whose rotation is not one of the four.
const (
rotationNone = "none"
rotationMonthly = "monthly"
rotationDaily = "daily"
rotationHourly = "hourly"
dailyConfig = `{"rotation":"daily"}`
weeklyConfig = `{"rotation":"weekly"}`
)
// The periods the tests archive into most: two days, and an hour of
// the first.
const (
dayPeriod = "2026-03-04"
nextDayPeriod = "2026-03-05"
hourPeriod = "2026-03-04-05"
)
// periodPath returns the archive file for a period of the target whose
// file without a period is path.
func periodPath(path, period string) string {
@@ -77,30 +57,30 @@ func TestDeliverDatabase_RotatesAtEachPeriodBoundary(t *testing.T) {
periods [2]string
}{
{
rotationMonthly, rotationMonthly,
"monthly", "monthly",
time.Date(2026, 1, 31, 23, 59, 59, 0, time.UTC),
time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC),
[2]string{"2026-01", "2026-02"},
},
{
rotationDaily, rotationDaily,
"daily", "daily",
time.Date(2026, 3, 4, 23, 59, 59, 0, time.UTC),
time.Date(2026, 3, 5, 0, 0, 0, 0, time.UTC),
[2]string{dayPeriod, nextDayPeriod},
[2]string{"2026-03-04", "2026-03-05"},
},
{
// The same instants, received in a zone two hours ahead
// of UTC, where they fall on 5 March: the period is UTC's.
"daily in another zone", rotationDaily,
"daily in another zone", "daily",
time.Date(2026, 3, 5, 1, 59, 59, 0, berlin),
time.Date(2026, 3, 5, 2, 0, 0, 0, berlin),
[2]string{dayPeriod, nextDayPeriod},
[2]string{"2026-03-04", "2026-03-05"},
},
{
rotationHourly, rotationHourly,
"hourly", "hourly",
time.Date(2026, 3, 4, 5, 59, 59, 0, time.UTC),
time.Date(2026, 3, 4, 6, 0, 0, 0, time.UTC),
[2]string{hourPeriod, "2026-03-04-06"},
[2]string{"2026-03-04-05", "2026-03-04-06"},
},
}
@@ -124,11 +104,11 @@ func TestDeliverDatabase_RotatesAtEachPeriodBoundary(t *testing.T) {
})
}
t.Run(rotationNone, func(t *testing.T) {
t.Run("none", func(t *testing.T) {
t.Parallel()
env := setupArchiveTest(t)
tgt := env.seedDatabaseTarget(t, `{"rotation":"`+rotationNone+`"}`)
tgt := env.seedDatabaseTarget(t, `{"rotation":"none"}`)
first := env.deliverReceivedAt(t, tgt, cases[0].before)
second := env.deliverReceivedAt(t, tgt, cases[0].after)
@@ -145,7 +125,7 @@ func TestDeliverDatabase_EventWithoutReceiveTime(t *testing.T) {
t.Parallel()
env := setupArchiveTest(t)
tgt := env.seedDatabaseTarget(t, `{"rotation":"`+rotationMonthly+`"}`)
tgt := env.seedDatabaseTarget(t, `{"rotation":"monthly"}`)
path := env.archivePath(tgt)
before := time.Now().UTC().Format("2006-01")
@@ -176,12 +156,12 @@ func TestDeliverDatabase_RotationChangeKeepsOldFiles(t *testing.T) {
first := env.deliverReceivedAt(t, tgt, at)
tgt.Config = dailyConfig
tgt.Config = `{"rotation":"daily"}`
second := env.deliverReceivedAt(t, tgt, at)
assert.Equal(t, []string{first}, archivedEventIDs(t, path))
assert.Equal(t, []string{second},
archivedEventIDs(t, periodPath(path, dayPeriod)))
archivedEventIDs(t, periodPath(path, "2026-03-04")))
}
// TestArchiveSweep_PrunesEveryFile gives a daily target three files:
@@ -195,11 +175,11 @@ func TestArchiveSweep_PrunesEveryFile(t *testing.T) {
env := setupArchiveTest(t)
tgt := env.seedDatabaseTarget(
t, `{"expiry":"1h","rotation":"`+rotationDaily+`"}`,
t, `{"expiry":"1h","rotation":"daily"}`,
)
path := env.archivePath(tgt)
emptied := periodPath(path, dayPeriod)
kept := periodPath(path, nextDayPeriod)
emptied := periodPath(path, "2026-03-04")
kept := periodPath(path, "2026-03-05")
now := time.Now()
old := now.Add(-48 * time.Hour)
@@ -232,7 +212,7 @@ func TestRename_MovesEveryFile(t *testing.T) {
unrotated := env.deliverReceivedAt(t, tgt, day)
tgt.Config = dailyConfig
tgt.Config = `{"rotation":"daily"}`
first := env.deliverReceivedAt(t, tgt, day)
second := env.deliverReceivedAt(t, tgt, day.Add(24*time.Hour))
@@ -244,17 +224,17 @@ func TestRename_MovesEveryFile(t *testing.T) {
for _, old := range []string{
oldPath,
periodPath(oldPath, dayPeriod),
periodPath(oldPath, nextDayPeriod),
periodPath(oldPath, "2026-03-04"),
periodPath(oldPath, "2026-03-05"),
} {
assert.NoFileExists(t, old)
}
assert.Equal(t, []string{unrotated}, archivedEventIDs(t, newPath))
assert.Equal(t, []string{first},
archivedEventIDs(t, periodPath(newPath, dayPeriod)))
archivedEventIDs(t, periodPath(newPath, "2026-03-04")))
assert.Equal(t, []string{second},
archivedEventIDs(t, periodPath(newPath, nextDayPeriod)))
archivedEventIDs(t, periodPath(newPath, "2026-03-05")))
third := env.deliverReceivedAt(t, tgt, day.Add(48*time.Hour))
assert.Equal(t, []string{third},
@@ -268,7 +248,7 @@ func TestRename_NeverReplacesARotatedFile(t *testing.T) {
t.Parallel()
env := setupArchiveTest(t)
tgt := env.seedDatabaseTarget(t, dailyConfig)
tgt := env.seedDatabaseTarget(t, `{"rotation":"daily"}`)
oldPath := env.archivePath(tgt)
day := time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC)
@@ -278,7 +258,7 @@ func TestRename_NeverReplacesARotatedFile(t *testing.T) {
newPath := filepath.Join(
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
)
planted := periodPath(newPath, nextDayPeriod)
planted := periodPath(newPath, "2026-03-05")
require.NoError(t, os.WriteFile(planted, []byte("planted"), 0o600))
require.ErrorIs(
@@ -286,9 +266,9 @@ func TestRename_NeverReplacesARotatedFile(t *testing.T) {
delivery.ErrArchiveNameTaken,
)
assert.FileExists(t, periodPath(oldPath, dayPeriod))
assert.FileExists(t, periodPath(oldPath, nextDayPeriod))
assert.NoFileExists(t, periodPath(newPath, dayPeriod))
assert.FileExists(t, periodPath(oldPath, "2026-03-04"))
assert.FileExists(t, periodPath(oldPath, "2026-03-05"))
assert.NoFileExists(t, periodPath(newPath, "2026-03-04"))
}
// TestStatArchive_EveryFile proves StatArchive counts and adds up every
@@ -300,7 +280,7 @@ func TestStatArchive_EveryFile(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "archive-wh.db")
files := []string{
path, periodPath(path, "2026-03"), periodPath(path, hourPeriod),
path, periodPath(path, "2026-03"), periodPath(path, "2026-03-04-05"),
}
written := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)
@@ -343,10 +323,10 @@ func TestArchivePathAt(t *testing.T) {
cases := map[string]string{
"": "",
`{"rotation":"` + rotationNone + `"}`: "",
`{"rotation":"` + rotationMonthly + `"}`: "-2026-10",
dailyConfig: "-2026-10-01",
`{"rotation":"` + rotationHourly + `"}`: "-2026-10-01-19",
`{"rotation":"none"}`: "",
`{"rotation":"monthly"}`: "-2026-10",
`{"rotation":"daily"}`: "-2026-10-01",
`{"rotation":"hourly"}`: "-2026-10-01-19",
}
for config, period := range cases {
@@ -367,7 +347,7 @@ func TestArchivePathAt(t *testing.T) {
}
_, err := delivery.ArchivePathAt(dbMgr, webhook, &database.Target{
Config: weeklyConfig,
Config: `{"rotation":"weekly"}`,
}, at)
require.Error(t, err)
}
@@ -377,9 +357,7 @@ func TestArchivePathAt(t *testing.T) {
func TestValidateArchiveRotation(t *testing.T) {
t.Parallel()
for _, ok := range []string{
"", rotationNone, rotationMonthly, rotationDaily, rotationHourly,
} {
for _, ok := range []string{"", "none", "monthly", "daily", "hourly"} {
require.NoError(t, delivery.ValidateArchiveRotation(ok), ok)
}
+2 -2
View File
@@ -208,7 +208,7 @@ func TestNewTargetConfigForm(t *testing.T) {
})
require.NoError(t, err)
assert.Equal(t, "720h", form.Expiry)
assert.Equal(t, rotationDaily, form.Rotation)
assert.Equal(t, "daily", form.Rotation)
form, err = delivery.NewTargetConfigForm(&database.Target{
Type: database.TargetTypeLog,
@@ -251,7 +251,7 @@ func TestNewTargetConfigForm_UnreadableConfigErrors(t *testing.T) {
},
{
Type: database.TargetTypeDatabase,
Config: weeklyConfig,
Config: `{"rotation":"weekly"}`,
},
{Type: database.TargetType("nope")},
}
+3 -6
View File
@@ -16,9 +16,6 @@ import (
"sneak.berlin/go/webhooker/internal/delivery"
)
// rotationNone is the archive rotation that keeps one file.
const rotationNone = "none"
// rotationShown returns the archive rotations the webhook page's
// target list shows.
func rotationShown(
@@ -41,7 +38,7 @@ func TestArchiveRotationChoices(t *testing.T) {
env := setupSourceTest(t)
for _, rotation := range []string{rotationNone, "monthly", "daily", "hourly"} {
for _, rotation := range []string{"none", "monthly", "daily", "hourly"} {
t.Run(rotation, func(t *testing.T) {
t.Parallel()
@@ -98,8 +95,8 @@ func TestArchiveRotationEditStartsOnNone(t *testing.T) {
)
page := targetEditPage(t, env, webhook.ID, target.ID)
assert.Equal(t, []string{rotationNone}, selectedIn(page, "rotation"))
assert.Equal(t, []string{rotationNone}, rotationShown(t, env, webhook.ID))
assert.Equal(t, []string{"none"}, selectedIn(page, "rotation"))
assert.Equal(t, []string{"none"}, rotationShown(t, env, webhook.ID))
}
// TestArchiveRotationRefused proves a rotation that is not one of the
+5 -8
View File
@@ -110,25 +110,22 @@ type Handlers struct {
// parsePageTemplate parses a page-specific template set from the
// embedded FS. Each page template is combined with the shared
// base, htmlheader, navbar and notice templates, and with any further
// files the page includes. The set is named after the page file, so
// the page's root action ({{template "base" .}}) is its entry point.
//
// The page file is parsed last because a later definition of a name
// replaces an earlier one: the page's {{define "title"}} must replace
// the {{block "title"}} fallback in htmlheader.html.
// files the page includes. The page file must be listed first so that
// its root action ({{template "base" .}}) becomes the template set's
// entry point.
func parsePageTemplate(
pageFile string, included ...string,
) *template.Template {
files := append([]string{
pageFile,
"base.html",
"htmlheader.html",
"navbar.html",
"notice.html",
}, included...)
files = append(files, pageFile)
return template.Must(
template.New(pageFile).ParseFS(templates.Templates, files...),
template.ParseFS(templates.Templates, files...),
)
}
-107
View File
@@ -1,107 +0,0 @@
package handlers_test
import (
"html/template"
"net/http"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/handlers"
"sneak.berlin/go/webhooker/internal/session"
"sneak.berlin/go/webhooker/templates"
)
// TestEveryPageRendersItsOwnTitle renders each page template and checks
// the browser tab title is the one the page declares, not the
// "Webhooker" fallback in htmlheader.html. A page that fails to render
// shows the error page's title instead, and fails here too.
func TestEveryPageRendersItsOwnTitle(t *testing.T) {
t.Parallel()
var h *handlers.Handlers
var sess *session.Session
app := newTestApp(t, &h, &sess)
app.RequireStart()
t.Cleanup(app.RequireStop)
// A pointer, as in the handlers: some pages call
// Webhook.RetentionLabel, a pointer method.
webhook := &database.Webhook{Name: "orders", RetentionDays: 14}
webhook.ID = testWebhookID
pages := []struct {
page string
data map[string]any
title string
}{
{"login.html", map[string]any{}, "Login - Webhooker"},
{"profile.html", map[string]any{}, "Profile - Webhooker"},
{"settings.html", map[string]any{}, "Settings - Webhooker"},
{"sources_list.html", map[string]any{}, "Webhooks - Webhooker"},
{"sources_new.html", map[string]any{}, "New Webhook - Webhooker"},
{
"source_detail.html",
map[string]any{dataKeyWebhook: webhook},
"orders - Webhooker",
},
{
"source_edit.html",
map[string]any{dataKeyWebhook: webhook},
"Edit orders - Webhooker",
},
{
"source_logs.html",
map[string]any{dataKeyWebhook: webhook, "TotalEvents": int64(0)},
"Full Event Log - orders - Webhooker",
},
{
"event_detail.html",
map[string]any{dataKeyWebhook: webhook},
"Event - orders - Webhooker",
},
{
"target_edit.html",
map[string]any{
dataKeyWebhook: webhook,
"Target": map[string]any{"Name": "alerts", "Type": "slack"},
},
"Edit alerts - Webhooker",
},
{
"error.html",
map[string]any{"StatusText": http.StatusText(http.StatusNotFound)},
"Not Found - Webhooker",
},
}
for _, p := range pages {
body := renderPage(t, h, sess, p.page, p.data)
_, afterOpen, _ := strings.Cut(body, "<title>")
title, _, _ := strings.Cut(afterOpen, "</title>")
assert.Equal(t, p.title, title, p.page)
}
}
// TestTitleFallbackIsWebhooker checks the title htmlheader.html gives a
// page that declares none. Every page declares one, so it is checked on
// htmlheader.html alone.
func TestTitleFallbackIsWebhooker(t *testing.T) {
t.Parallel()
header := template.Must(
template.ParseFS(templates.Templates, "htmlheader.html"),
)
var buf strings.Builder
require.NoError(t, header.ExecuteTemplate(&buf, "htmlheader", nil))
assert.Contains(t, buf.String(), "<title>Webhooker</title>")
}