A target's settings move from the per-type JSON string in targets.config to columns: url, headers, timeout_seconds, forward_query, archive_expiry and archive_rotation. A column a type does not use stays empty. headers holds the header map as JSON text through GORM's JSON serializer, and archive_expiry holds a time.Duration, 0 meaning never. The forms check each setting once, with the same refusals and messages, and the edit form is filled from the columns, so the per-type config structs, their parsers and the "could not be read" branches are gone. A delivery task carries the six settings in place of the JSON string; tests check that each copy into a task, and back out of one, keeps all six. Model: opus-5-5
This commit was merged in pull request #554.
This commit is contained in:
@@ -143,26 +143,15 @@ func archiveNamePart(name string) string {
|
||||
}
|
||||
|
||||
// Write writes the full event of a delivery to a database target as
|
||||
// a row into the target's archive database, honouring the optional
|
||||
// per-target expiry and rotation parsed from the target config JSON.
|
||||
// With rotation, the event goes to the file for the period of its
|
||||
// receive time.
|
||||
// a row into the target's archive database, honouring the target's
|
||||
// ArchiveExpiry and ArchiveRotation. With rotation, the event goes to
|
||||
// the file for the period of its receive time.
|
||||
func (ws *Writers) Write(d *database.Delivery) error {
|
||||
webhookID := d.Event.WebhookID
|
||||
if webhookID == "" {
|
||||
return errArchiveMissingWebhookID
|
||||
}
|
||||
|
||||
expiry, err := ParseExpiry(d.Target.Config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
rotation, err := ParseRotation(d.Target.Config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// An event whose stored row was gone before its delivery ran has
|
||||
// no receive time (see delivery's Engine.hydrateEvent), and goes
|
||||
// to the file for now.
|
||||
@@ -187,7 +176,10 @@ func (ws *Writers) Write(d *database.Delivery) error {
|
||||
ContentType: d.Event.ContentType,
|
||||
}
|
||||
|
||||
return w.write(row, expiry, archivePeriod(rotation, receivedAt))
|
||||
return w.write(
|
||||
row, d.Target.ArchiveExpiry,
|
||||
archivePeriod(d.Target.ArchiveRotation, receivedAt),
|
||||
)
|
||||
}
|
||||
|
||||
// Rename implements Archives. It renames every one of a database
|
||||
|
||||
@@ -123,9 +123,10 @@ func databaseTargetDelivery(
|
||||
TargetID: tgt.ID,
|
||||
Event: event,
|
||||
Target: database.Target{
|
||||
Name: tgt.Name,
|
||||
Type: database.TargetTypeDatabase,
|
||||
Config: tgt.Config,
|
||||
Name: tgt.Name,
|
||||
Type: database.TargetTypeDatabase,
|
||||
ArchiveExpiry: tgt.ArchiveExpiry,
|
||||
ArchiveRotation: tgt.ArchiveRotation,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -355,13 +356,16 @@ func TestParseArchiveExpiry(t *testing.T) {
|
||||
want time.Duration
|
||||
wantErr bool
|
||||
}{
|
||||
{"empty config", "", 0, false},
|
||||
{"explicit never", `{"expiry":"never"}`, 0, false},
|
||||
{"empty expiry", `{"expiry":""}`, 0, false},
|
||||
{"duration", `{"expiry":"1h"}`, time.Hour, false},
|
||||
{"unparseable", `{"expiry":"nonsense"}`, 0, true},
|
||||
{"zero duration", `{"expiry":"0s"}`, 0, true},
|
||||
{"negative duration", `{"expiry":"-5h"}`, 0, true},
|
||||
{"empty", "", 0, false},
|
||||
{"explicit never", "never", 0, false},
|
||||
{"hour", "1h", time.Hour, false},
|
||||
{"thirty days", "720h", 720 * time.Hour, false},
|
||||
{"minutes", "30m", 30 * time.Minute, false},
|
||||
{"unparseable", "nonsense", 0, true},
|
||||
{"days are not a Go unit", "7d", 0, true},
|
||||
{"zero duration", "0s", 0, true},
|
||||
{"bare zero", "0", 0, true},
|
||||
{"negative duration", "-5h", 0, true},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
@@ -381,26 +385,6 @@ func TestParseArchiveExpiry(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateArchiveExpiry(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
valid := []string{"", "never", "1h", "720h", "30m"}
|
||||
for _, in := range valid {
|
||||
require.NoError(t,
|
||||
archive.ValidateExpiry(in),
|
||||
"expiry %q should be accepted", in,
|
||||
)
|
||||
}
|
||||
|
||||
invalid := []string{"nonsense", "7d", "-5h", "0s", "0"}
|
||||
for _, in := range invalid {
|
||||
require.Error(t,
|
||||
archive.ValidateExpiry(in),
|
||||
"expiry %q should be rejected", in,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// TestArchiveFileName pins the archive file name and the rules
|
||||
// that make a webhook or target name safe to put in it.
|
||||
func TestArchiveFileName(t *testing.T) {
|
||||
@@ -473,8 +457,8 @@ func TestWrite_EachTargetHasItsOwnArchive(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
first := env.seedDatabaseTarget(t, "")
|
||||
second := env.addDatabaseTarget(t, first.WebhookID, "")
|
||||
first := env.seedDatabaseTarget(t, 0, "")
|
||||
second := env.addDatabaseTarget(t, first.WebhookID, 0, "")
|
||||
|
||||
event := newEvent(`{"n":1}`)
|
||||
|
||||
@@ -503,7 +487,7 @@ func TestRename_MovesTheFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, "")
|
||||
tgt := env.seedDatabaseTarget(t, 0, "")
|
||||
oldPath := env.archivePath(tgt)
|
||||
|
||||
first := newEvent(`{"n":1}`)
|
||||
@@ -545,7 +529,7 @@ func TestRename_NeverReplacesAFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, "")
|
||||
tgt := env.seedDatabaseTarget(t, 0, "")
|
||||
oldPath := env.archivePath(tgt)
|
||||
|
||||
first := newEvent(`{"n":1}`)
|
||||
@@ -593,7 +577,7 @@ func TestRename_BeforeTheNameIsSaved(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, "")
|
||||
tgt := env.seedDatabaseTarget(t, 0, "")
|
||||
|
||||
require.NoError(
|
||||
t, env.writers.Rename(tgt.ID, "Orders", "Archive"),
|
||||
@@ -721,7 +705,7 @@ func TestRenameLock_HeldByOneEditAtATime(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, "")
|
||||
tgt := env.seedDatabaseTarget(t, 0, "")
|
||||
event := newEvent(`{"n":1}`)
|
||||
|
||||
env.writers.RenameLock().Lock()
|
||||
|
||||
@@ -37,9 +37,9 @@ func TestEvictWebhook_ClosesAndRemovesWriter(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
first := env.seedDatabaseTarget(t, "")
|
||||
second := env.addDatabaseTarget(t, first.WebhookID, "")
|
||||
other := env.seedDatabaseTarget(t, "")
|
||||
first := env.seedDatabaseTarget(t, 0, "")
|
||||
second := env.addDatabaseTarget(t, first.WebhookID, 0, "")
|
||||
other := env.seedDatabaseTarget(t, 0, "")
|
||||
|
||||
for _, tgt := range []*database.Target{first, second, other} {
|
||||
deliverTo(t, env, tgt)
|
||||
@@ -80,8 +80,8 @@ func TestEvictTarget_LeavesOtherTargets(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
doomed := env.seedDatabaseTarget(t, "")
|
||||
kept := env.addDatabaseTarget(t, doomed.WebhookID, "")
|
||||
doomed := env.seedDatabaseTarget(t, 0, "")
|
||||
kept := env.addDatabaseTarget(t, doomed.WebhookID, 0, "")
|
||||
|
||||
deliverTo(t, env, doomed)
|
||||
deliverTo(t, env, kept)
|
||||
@@ -310,7 +310,7 @@ func TestEvictWebhook_RacingWriteDoesNotReopenHandle(
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, "")
|
||||
tgt := env.seedDatabaseTarget(t, 0, "")
|
||||
|
||||
// Prime the registry so the test can hold the very writer the
|
||||
// eviction is about to detach.
|
||||
@@ -355,7 +355,7 @@ func TestEvictWebhook_LaterDeliveryRecreatesWriter(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, "")
|
||||
tgt := env.seedDatabaseTarget(t, 0, "")
|
||||
|
||||
deliverTo(t, env, tgt)
|
||||
require.True(t, env.writers.ExportHasArchiveWriter(tgt.ID))
|
||||
@@ -380,7 +380,7 @@ func TestEvictAll_WriteAfterIsRefused(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, "")
|
||||
tgt := env.seedDatabaseTarget(t, 0, "")
|
||||
|
||||
deliverTo(t, env, tgt)
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -54,35 +53,10 @@ func ValidateRotation(rotation string) error {
|
||||
}
|
||||
}
|
||||
|
||||
// ParseRotation reads the rotation from a database target's
|
||||
// config JSON. An empty config or an empty rotation is none.
|
||||
func ParseRotation(configJSON string) (string, error) {
|
||||
if configJSON == "" {
|
||||
return archiveRotationNone, nil
|
||||
}
|
||||
|
||||
var cfg DatabaseTargetConfig
|
||||
|
||||
err := json.Unmarshal([]byte(configJSON), &cfg)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parsing database target config: %w", err)
|
||||
}
|
||||
|
||||
err = ValidateRotation(cfg.Rotation)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if cfg.Rotation == "" {
|
||||
return archiveRotationNone, nil
|
||||
}
|
||||
|
||||
return cfg.Rotation, nil
|
||||
}
|
||||
|
||||
// archivePeriod returns the period, in UTC, that a rotation puts an
|
||||
// event received at receivedAt in: "2026-10" for monthly,
|
||||
// "2026-10-01" for daily, "2026-10-01-19" for hourly, and "" for none.
|
||||
// "2026-10-01" for daily, "2026-10-01-19" for hourly, and "" for none
|
||||
// or an empty rotation.
|
||||
func archivePeriod(rotation string, receivedAt time.Time) string {
|
||||
switch rotation {
|
||||
case archiveRotationMonthly:
|
||||
@@ -115,16 +89,11 @@ func PathAt(
|
||||
webhook *database.Webhook,
|
||||
target *database.Target,
|
||||
receivedAt time.Time,
|
||||
) (string, error) {
|
||||
rotation, err := ParseRotation(target.Config)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
) string {
|
||||
return archivePeriodPath(
|
||||
Path(dbMgr, webhook, target),
|
||||
archivePeriod(rotation, receivedAt),
|
||||
), nil
|
||||
archivePeriod(target.ArchiveRotation, receivedAt),
|
||||
)
|
||||
}
|
||||
|
||||
// archiveFiles lists the database target's archive files that exist,
|
||||
|
||||
@@ -14,16 +14,12 @@ import (
|
||||
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
||||
)
|
||||
|
||||
// The archive rotations, and the configs of a daily target and of one
|
||||
// whose rotation is not one of the four.
|
||||
// The archive rotations.
|
||||
const (
|
||||
rotationNone = "none"
|
||||
rotationMonthly = "monthly"
|
||||
rotationDaily = "daily"
|
||||
rotationHourly = "hourly"
|
||||
|
||||
dailyConfig = `{"rotation":"daily"}`
|
||||
weeklyConfig = `{"rotation":"weekly"}`
|
||||
)
|
||||
|
||||
// The periods the tests archive into most: two days, and an hour of
|
||||
@@ -108,7 +104,7 @@ func TestWrite_RotatesAtEachPeriodBoundary(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, `{"rotation":"`+tc.rotation+`"}`)
|
||||
tgt := env.seedDatabaseTarget(t, 0, tc.rotation)
|
||||
path := env.archivePath(tgt)
|
||||
|
||||
first := env.deliverReceivedAt(t, tgt, tc.before)
|
||||
@@ -127,7 +123,7 @@ func TestWrite_RotatesAtEachPeriodBoundary(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, `{"rotation":"`+rotationNone+`"}`)
|
||||
tgt := env.seedDatabaseTarget(t, 0, rotationNone)
|
||||
|
||||
first := env.deliverReceivedAt(t, tgt, cases[0].before)
|
||||
second := env.deliverReceivedAt(t, tgt, cases[0].after)
|
||||
@@ -144,7 +140,7 @@ func TestWrite_EventWithoutReceiveTime(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, `{"rotation":"`+rotationMonthly+`"}`)
|
||||
tgt := env.seedDatabaseTarget(t, 0, rotationMonthly)
|
||||
path := env.archivePath(tgt)
|
||||
|
||||
before := time.Now().UTC().Format("2006-01")
|
||||
@@ -169,13 +165,13 @@ func TestWrite_RotationChangeKeepsOldFiles(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, "")
|
||||
tgt := env.seedDatabaseTarget(t, 0, rotationNone)
|
||||
path := env.archivePath(tgt)
|
||||
at := time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
first := env.deliverReceivedAt(t, tgt, at)
|
||||
|
||||
tgt.Config = dailyConfig
|
||||
tgt.ArchiveRotation = rotationDaily
|
||||
second := env.deliverReceivedAt(t, tgt, at)
|
||||
|
||||
assert.Equal(t, []string{first}, archivedEventIDs(t, path))
|
||||
@@ -193,9 +189,7 @@ func TestArchiveSweep_PrunesEveryFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(
|
||||
t, `{"expiry":"1h","rotation":"`+rotationDaily+`"}`,
|
||||
)
|
||||
tgt := env.seedDatabaseTarget(t, time.Hour, rotationDaily)
|
||||
path := env.archivePath(tgt)
|
||||
emptied := periodPath(path, dayPeriod)
|
||||
kept := periodPath(path, nextDayPeriod)
|
||||
@@ -225,13 +219,13 @@ func TestRename_MovesEveryFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, "")
|
||||
tgt := env.seedDatabaseTarget(t, 0, rotationNone)
|
||||
oldPath := env.archivePath(tgt)
|
||||
day := time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
unrotated := env.deliverReceivedAt(t, tgt, day)
|
||||
|
||||
tgt.Config = dailyConfig
|
||||
tgt.ArchiveRotation = rotationDaily
|
||||
first := env.deliverReceivedAt(t, tgt, day)
|
||||
second := env.deliverReceivedAt(t, tgt, day.Add(24*time.Hour))
|
||||
|
||||
@@ -267,7 +261,7 @@ func TestRename_NeverReplacesARotatedFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, dailyConfig)
|
||||
tgt := env.seedDatabaseTarget(t, 0, rotationDaily)
|
||||
oldPath := env.archivePath(tgt)
|
||||
day := time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
@@ -341,34 +335,27 @@ func TestArchivePathAt(t *testing.T) {
|
||||
at := time.Date(2026, 10, 1, 19, 30, 0, 0, time.UTC)
|
||||
|
||||
cases := map[string]string{
|
||||
"": "",
|
||||
`{"rotation":"` + rotationNone + `"}`: "",
|
||||
`{"rotation":"` + rotationMonthly + `"}`: "-2026-10",
|
||||
dailyConfig: "-2026-10-01",
|
||||
`{"rotation":"` + rotationHourly + `"}`: "-2026-10-01-19",
|
||||
"": "",
|
||||
rotationNone: "",
|
||||
rotationMonthly: "-2026-10",
|
||||
rotationDaily: "-2026-10-01",
|
||||
rotationHourly: "-2026-10-01-19",
|
||||
}
|
||||
|
||||
for config, period := range cases {
|
||||
for rotation, period := range cases {
|
||||
target := &database.Target{
|
||||
BaseModel: database.BaseModel{ID: "tgt-id"},
|
||||
Name: "Archive",
|
||||
Config: config,
|
||||
BaseModel: database.BaseModel{ID: "tgt-id"},
|
||||
Name: "Archive",
|
||||
ArchiveRotation: rotation,
|
||||
}
|
||||
|
||||
got, err := archive.PathAt(dbMgr, webhook, target, at)
|
||||
require.NoError(t, err, config)
|
||||
assert.Equal(t,
|
||||
filepath.Join(
|
||||
dataDir, "archive-orders-archive-tgt-id"+period+".db",
|
||||
),
|
||||
got, config,
|
||||
archive.PathAt(dbMgr, webhook, target, at), rotation,
|
||||
)
|
||||
}
|
||||
|
||||
_, err := archive.PathAt(dbMgr, webhook, &database.Target{
|
||||
Config: weeklyConfig,
|
||||
}, at)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
// TestValidateArchiveRotation accepts the four rotations, and empty,
|
||||
|
||||
@@ -199,22 +199,11 @@ func (s *Sweeper) sweep(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// sweepTarget prunes the archive of a single database target.
|
||||
// A missing, empty, or "never" expiry parses as a zero duration
|
||||
// and is skipped entirely, so those archives are never pruned.
|
||||
// sweepTarget prunes the archive of a single database target. A
|
||||
// target with an ArchiveExpiry of 0 keeps its archive forever and is
|
||||
// skipped entirely, so that archive is never pruned.
|
||||
func (s *Sweeper) sweepTarget(target *database.Target) {
|
||||
expiry, err := ParseExpiry(target.Config)
|
||||
if err != nil {
|
||||
s.log.Error(
|
||||
"archive sweep: invalid database target config",
|
||||
"webhook_id", target.WebhookID,
|
||||
"target_id", target.ID,
|
||||
"error", err,
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
expiry := target.ArchiveExpiry
|
||||
if expiry <= 0 {
|
||||
return
|
||||
}
|
||||
@@ -223,7 +212,7 @@ func (s *Sweeper) sweepTarget(target *database.Target) {
|
||||
return
|
||||
}
|
||||
|
||||
err = s.writers.sweepArchive(target.ID, expiry)
|
||||
err := s.writers.sweepArchive(target.ID, expiry)
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -106,10 +106,10 @@ func (env *archiveEnv) archivePath(tgt *database.Target) string {
|
||||
)
|
||||
}
|
||||
|
||||
// seedDatabaseTarget creates a webhook with one database target
|
||||
// carrying the given target config JSON, and returns the target.
|
||||
// seedDatabaseTarget creates a webhook with one database target with
|
||||
// the given archive expiry and rotation, and returns the target.
|
||||
func (env *archiveEnv) seedDatabaseTarget(
|
||||
t *testing.T, configJSON string,
|
||||
t *testing.T, expiry time.Duration, rotation string,
|
||||
) *database.Target {
|
||||
t.Helper()
|
||||
|
||||
@@ -124,23 +124,27 @@ func (env *archiveEnv) seedDatabaseTarget(
|
||||
Create(wh).Error,
|
||||
)
|
||||
|
||||
return env.addDatabaseTarget(t, wh.ID, configJSON)
|
||||
return env.addDatabaseTarget(t, wh.ID, expiry, rotation)
|
||||
}
|
||||
|
||||
// addDatabaseTarget creates one more database target on an
|
||||
// existing webhook and returns it.
|
||||
func (env *archiveEnv) addDatabaseTarget(
|
||||
t *testing.T, webhookID, configJSON string,
|
||||
t *testing.T,
|
||||
webhookID string,
|
||||
expiry time.Duration,
|
||||
rotation string,
|
||||
) *database.Target {
|
||||
t.Helper()
|
||||
|
||||
tgt := &database.Target{
|
||||
WebhookID: webhookID,
|
||||
Name: "Archive",
|
||||
Type: database.TargetTypeDatabase,
|
||||
Active: true,
|
||||
Config: configJSON,
|
||||
MaxAttempts: 1,
|
||||
WebhookID: webhookID,
|
||||
Name: "Archive",
|
||||
Type: database.TargetTypeDatabase,
|
||||
Active: true,
|
||||
ArchiveExpiry: expiry,
|
||||
ArchiveRotation: rotation,
|
||||
MaxAttempts: 1,
|
||||
}
|
||||
require.NoError(
|
||||
t,
|
||||
@@ -323,7 +327,7 @@ func TestArchiveSweeper_LoopOutlivesStartHookContext(
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
||||
|
||||
now := time.Now()
|
||||
path := env.seedArchiveRows(
|
||||
@@ -377,7 +381,7 @@ func TestArchiveSweep_DoesNotResurrectEvictedWriter(
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
||||
env.seedArchiveRows(
|
||||
t, tgt, time.Now().Add(-48*time.Hour),
|
||||
)
|
||||
@@ -411,7 +415,7 @@ func TestArchiveSweep_LeavesNoRegistryEntry(t *testing.T) {
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
||||
path := env.seedArchiveRows(
|
||||
t, tgt,
|
||||
time.Now().Add(-48*time.Hour),
|
||||
@@ -444,7 +448,7 @@ func TestArchiveSweep_KeepsWriterAdoptedByDelivery(
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
||||
env.seedArchiveRows(
|
||||
t, tgt, time.Now().Add(-48*time.Hour),
|
||||
)
|
||||
@@ -498,7 +502,7 @@ func TestArchiveSweep_KeepsWriterAdoptedDuringSweep(
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
||||
env.seedArchiveRows(
|
||||
t, tgt, time.Now().Add(-48*time.Hour),
|
||||
)
|
||||
@@ -551,8 +555,8 @@ func TestArchiveSweep_KeepsWriterAdoptedDuringSweep(
|
||||
|
||||
// TestArchiveSweep_ContinuesAfterPerTargetFailure proves a
|
||||
// failure for one target does not abort the sweep for the
|
||||
// others: an unparseable expiry and an unreadable archive both
|
||||
// have to be logged and stepped over.
|
||||
// others: an unreadable archive has to be logged and stepped
|
||||
// over.
|
||||
func TestArchiveSweep_ContinuesAfterPerTargetFailure(
|
||||
t *testing.T,
|
||||
) {
|
||||
@@ -560,21 +564,16 @@ func TestArchiveSweep_ContinuesAfterPerTargetFailure(
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
|
||||
// Seeded first so the sweep reaches them before the healthy
|
||||
// Seeded first so the sweep reaches it before the healthy
|
||||
// target: targets come back in insertion order.
|
||||
badConfig := env.seedDatabaseTarget(t, `{"expiry":"!!!"}`)
|
||||
env.seedArchiveRows(
|
||||
t, badConfig, time.Now().Add(-48*time.Hour),
|
||||
)
|
||||
|
||||
corrupt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
corrupt := env.seedDatabaseTarget(t, time.Hour, "")
|
||||
require.NoError(t, os.WriteFile(
|
||||
env.archivePath(corrupt),
|
||||
[]byte("this is not a sqlite database"),
|
||||
0o600,
|
||||
))
|
||||
|
||||
healthy := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
healthy := env.seedDatabaseTarget(t, time.Hour, "")
|
||||
healthyPath := env.seedArchiveRows(
|
||||
t, healthy,
|
||||
time.Now().Add(-48*time.Hour),
|
||||
@@ -631,7 +630,7 @@ func TestArchiveSweep_PrunesIdleArchive(t *testing.T) {
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
||||
|
||||
now := time.Now()
|
||||
path := env.seedArchiveRows(
|
||||
@@ -670,7 +669,7 @@ func TestArchiveSweep_LeavesArchiveClosed(t *testing.T) {
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
||||
path := env.seedArchiveRows(
|
||||
t, tgt, time.Now().Add(-48*time.Hour),
|
||||
)
|
||||
@@ -710,7 +709,7 @@ func TestArchiveSweep_ClosesHandleOfRegisteredWriter(
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
||||
env.seedArchiveRows(
|
||||
t, tgt, time.Now().Add(-48*time.Hour),
|
||||
)
|
||||
@@ -795,36 +794,29 @@ func TestArchiveSweep_CancelledSweepLogsNoError(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestArchiveSweep_NeverExpiryUntouched proves the sweep is a
|
||||
// no-op for the default retention policy, so archives with no
|
||||
// expiry (or the literal "never") behave exactly as before.
|
||||
// no-op for the default retention policy, so an archive with an
|
||||
// expiry of 0, which keeps it forever, is left alone.
|
||||
func TestArchiveSweep_NeverExpiryUntouched(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, configJSON := range []string{
|
||||
`{"expiry":"never"}`,
|
||||
`{"expiry":""}`,
|
||||
"",
|
||||
} {
|
||||
env := setupArchiveTest(t)
|
||||
env := setupArchiveTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, configJSON)
|
||||
path := env.seedArchiveRows(
|
||||
t, tgt,
|
||||
time.Now().Add(-10000*time.Hour),
|
||||
)
|
||||
tgt := env.seedDatabaseTarget(t, 0, "")
|
||||
path := env.seedArchiveRows(
|
||||
t, tgt,
|
||||
time.Now().Add(-10000*time.Hour),
|
||||
)
|
||||
|
||||
env.sweeper.ExportSweep(context.Background())
|
||||
env.sweeper.ExportSweep(context.Background())
|
||||
|
||||
assert.Equal(
|
||||
t, []string{sweepRowOld}, archivedEventIDs(t, path),
|
||||
"config %q must keep rows forever", configJSON,
|
||||
)
|
||||
assert.False(
|
||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
||||
"config %q must leave no registry entry behind",
|
||||
configJSON,
|
||||
)
|
||||
}
|
||||
assert.Equal(
|
||||
t, []string{sweepRowOld}, archivedEventIDs(t, path),
|
||||
"an expiry of 0 must keep rows forever",
|
||||
)
|
||||
assert.False(
|
||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
||||
"an expiry of 0 must leave no registry entry behind",
|
||||
)
|
||||
}
|
||||
|
||||
// TestArchiveSweep_NeverExpirySkipsBeforeOpening pins the
|
||||
@@ -845,7 +837,7 @@ func TestArchiveSweep_NeverExpirySkipsBeforeOpening(
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"never"}`)
|
||||
tgt := env.seedDatabaseTarget(t, 0, "")
|
||||
path := env.archivePath(tgt)
|
||||
|
||||
seedUnmigratedArchive(t, path)
|
||||
@@ -897,7 +889,7 @@ func TestArchiveSweep_DoesNotCreateArchiveFile(t *testing.T) {
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
||||
path := env.archivePath(tgt)
|
||||
|
||||
require.NoFileExists(t, path)
|
||||
@@ -928,7 +920,7 @@ func TestArchiveSweep_DoesNotCreateAfterWriterExists(
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
||||
|
||||
path, err := env.writers.ExportEnsureArchiveWriter(tgt.ID)
|
||||
require.NoError(t, err)
|
||||
@@ -947,7 +939,7 @@ func TestArchiveSweep_SkipsDeletedWebhookTargets(t *testing.T) {
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
||||
path := env.seedArchiveRows(
|
||||
t, tgt, time.Now().Add(-48*time.Hour),
|
||||
)
|
||||
@@ -975,7 +967,7 @@ func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
||||
|
||||
deliveries := make(
|
||||
[]*database.Delivery, 0, sweepConcurrentWrites,
|
||||
@@ -1020,7 +1012,7 @@ func TestArchiveSweeper_StopsCleanly(t *testing.T) {
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
tgt := env.seedDatabaseTarget(t, time.Hour, "")
|
||||
env.seedArchiveRows(
|
||||
t, tgt, time.Now().Add(-48*time.Hour),
|
||||
)
|
||||
|
||||
+14
-70
@@ -1,7 +1,6 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
@@ -18,8 +17,9 @@ import (
|
||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||
)
|
||||
|
||||
// ExpiryNever is the expiry sentinel (and default) that
|
||||
// disables pruning so archived rows are kept forever.
|
||||
// ExpiryNever is the archive expiry, as the target forms take it,
|
||||
// that disables pruning so archived rows are kept forever. It is
|
||||
// stored as an ArchiveExpiry of 0.
|
||||
const ExpiryNever = "never"
|
||||
|
||||
// archiveReopenDebounce bounds how often an archive file is
|
||||
@@ -79,19 +79,6 @@ var (
|
||||
)
|
||||
)
|
||||
|
||||
// DatabaseTargetConfig is the optional per-target JSON config
|
||||
// for a database (archive) target.
|
||||
type DatabaseTargetConfig struct {
|
||||
// Expiry is a Go duration (e.g. "720h") after which
|
||||
// archived rows are pruned, or "never" (the default) to
|
||||
// keep them forever.
|
||||
Expiry string `json:"expiry"`
|
||||
|
||||
// Rotation is none (the default), monthly, daily or hourly: see
|
||||
// archivePeriod.
|
||||
Rotation string `json:"rotation"`
|
||||
}
|
||||
|
||||
// archivedEvent is one fully captured webhook event stored in a
|
||||
// database target's archive for long-term retention. It is a
|
||||
// self-contained copy — independent of the per-webhook event
|
||||
@@ -112,63 +99,20 @@ type archivedEvent struct {
|
||||
ArchivedAt time.Time `gorm:"index"`
|
||||
}
|
||||
|
||||
// ParseExpiry reads the optional expiry from a database
|
||||
// target's config JSON. An empty config, an empty expiry, or
|
||||
// the literal "never" all mean keep forever, returned as a zero
|
||||
// duration. Any other value must parse as a positive Go
|
||||
// duration; a set-but-invalid value (unparseable, zero, or
|
||||
// negative) is an error rather than a silent default, matching
|
||||
// ValidateExpiry at target creation.
|
||||
func ParseExpiry(
|
||||
configJSON string,
|
||||
) (time.Duration, error) {
|
||||
if configJSON == "" {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var cfg DatabaseTargetConfig
|
||||
|
||||
err := json.Unmarshal([]byte(configJSON), &cfg)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf(
|
||||
"parsing database target config: %w", err,
|
||||
)
|
||||
}
|
||||
|
||||
if cfg.Expiry == "" || cfg.Expiry == ExpiryNever {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
dur, err := time.ParseDuration(cfg.Expiry)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf(
|
||||
"parsing archive expiry %q: %w", cfg.Expiry, err,
|
||||
)
|
||||
}
|
||||
|
||||
if dur <= 0 {
|
||||
return 0, fmt.Errorf(
|
||||
"%w: %q", errArchiveExpiryNotPositive, cfg.Expiry,
|
||||
)
|
||||
}
|
||||
|
||||
return dur, nil
|
||||
}
|
||||
|
||||
// ValidateExpiry checks a user-supplied archive expiry
|
||||
// for a database target at configuration time. Valid values are
|
||||
// empty, "never" (both meaning keep forever), or a positive Go
|
||||
// duration such as "720h". Anything else is an error, so a bad
|
||||
// expiry is rejected when the target is created rather than
|
||||
// failing every subsequent delivery.
|
||||
func ValidateExpiry(expiry string) error {
|
||||
// ParseExpiry reads a user-supplied archive expiry for a database
|
||||
// target, as the target forms submit it, into the ArchiveExpiry the
|
||||
// target stores. Empty and "never" both mean keep forever, which is
|
||||
// 0; anything else must be a positive Go duration such as "720h". Any
|
||||
// other value is an error, so a bad expiry is refused at the form
|
||||
// rather than stored.
|
||||
func ParseExpiry(expiry string) (time.Duration, error) {
|
||||
if expiry == "" || expiry == ExpiryNever {
|
||||
return nil
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
dur, err := time.ParseDuration(expiry)
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
return 0, fmt.Errorf(
|
||||
"expiry must be %q or a Go duration "+
|
||||
"such as \"720h\": %w",
|
||||
ExpiryNever, err,
|
||||
@@ -176,12 +120,12 @@ func ValidateExpiry(expiry string) error {
|
||||
}
|
||||
|
||||
if dur <= 0 {
|
||||
return fmt.Errorf(
|
||||
return 0, fmt.Errorf(
|
||||
"%w: %q", errArchiveExpiryNotPositive, expiry,
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
return dur, nil
|
||||
}
|
||||
|
||||
// archiveWriter owns one database target's archive SQLite files.
|
||||
|
||||
Reference in New Issue
Block a user