Add per-webhook event retention reaper (closes #63) (#78)
All checks were successful
check / check (push) Successful in 2m42s
All checks were successful
check / check (push) Successful in 2m42s
Enforces each webhook's `RetentionDays` so per-webhook SQLite files no longer grow without bound. ## Reaper New `RetentionReaper` in `internal/database/retention.go`. A background ticker runs each sweep: it lists all webhooks from the main DB and, for each webhook with a positive `RetentionDays`, opens its per-webhook DB via `WebhookDBManager.GetDB` and deletes every `Event` (and its dependent `Delivery` and `DeliveryResult` rows) whose `CreatedAt` is older than `RetentionDays` days. - Deletions run in foreign-key-safe order: delivery results, then deliveries, then events. - Deletes are unscoped (hard deletes) so rows are physically removed and disk is reclaimed, rather than GORM soft-deleting them. - `RetentionDays <= 0` means retain forever; those webhooks are skipped. - Webhooks whose per-webhook DB does not yet exist are skipped. ## Config `internal/config/config.go` gains `RetentionSweepInterval` (env `RETENTION_SWEEP_INTERVAL`, parsed as a Go duration, default `1h`) via a new `envDuration` helper, following the existing env-helper conventions. ## Wiring `cmd/webhooker/main.go` registers `database.NewRetentionReaper` as an fx provider and forces its construction in `fx.Invoke`. The reaper starts its sweep loop on an fx `OnStart` hook and stops cleanly on `OnStop` via context cancellation, matching the existing lifecycle components. ## Test `internal/database/retention_test.go` seeds an old event chain (event + delivery + result, 40 days old) and a recent one (1 day old) in a real per-webhook DB and asserts a single sweep removes only the expired chain while keeping the recent one. A second test forces a non-positive `RetentionDays` and asserts an ancient event is retained. Note: the `Webhook.RetentionDays` column carries `gorm:"default:30"`, so a `0` passed to a GORM `Create` is replaced by the default; the test forces the value with an explicit column update to exercise the retain-forever path. No model changes were made. Validated with `docker build .` (fmt-check, lint, test, build) exit 0. Closes #63 Co-authored-by: sneak <sneak@sneak.berlin> Reviewed-on: #78 Co-authored-by: clawbot <clawbot@noreply.example.org> Co-committed-by: clawbot <clawbot@noreply.example.org>
This commit was merged in pull request #78.
This commit is contained in:
@@ -8,6 +8,7 @@ import (
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/webhooker/internal/globals"
|
||||
@@ -26,6 +27,10 @@ const (
|
||||
|
||||
// defaultPort is the default HTTP listen port.
|
||||
defaultPort = 8080
|
||||
|
||||
// defaultRetentionSweepInterval is how often the retention
|
||||
// reaper deletes events older than each webhook's RetentionDays.
|
||||
defaultRetentionSweepInterval = time.Hour
|
||||
)
|
||||
|
||||
// ErrInvalidEnvironment is returned when WEBHOOKER_ENVIRONMENT
|
||||
@@ -51,8 +56,12 @@ type Config struct {
|
||||
MetricsUsername string
|
||||
Port int
|
||||
SentryDSN string
|
||||
params *ConfigParams
|
||||
log *slog.Logger
|
||||
|
||||
// RetentionSweepInterval is how often the retention reaper runs.
|
||||
RetentionSweepInterval time.Duration
|
||||
|
||||
params *ConfigParams
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// IsDev returns true if running in development environment.
|
||||
@@ -95,6 +104,30 @@ func envInt(key string, defaultValue int) int {
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
// envDuration returns the value of the named environment variable
|
||||
// parsed as a Go duration (e.g. "1h", "30m"). Returns defaultValue if
|
||||
// not set. If the variable is set but cannot be parsed, it returns a
|
||||
// wrapped error naming the key and the bad value, so startup fails
|
||||
// loudly rather than silently falling back to the default.
|
||||
func envDuration(
|
||||
key string,
|
||||
defaultValue time.Duration,
|
||||
) (time.Duration, error) {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
return defaultValue, nil
|
||||
}
|
||||
|
||||
d, err := time.ParseDuration(v)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf(
|
||||
"invalid duration for %s: %q: %w", key, v, err,
|
||||
)
|
||||
}
|
||||
|
||||
return d, nil
|
||||
}
|
||||
|
||||
// New creates a Config by reading environment variables.
|
||||
//
|
||||
//nolint:revive // lc parameter is required by fx even if unused.
|
||||
@@ -118,18 +151,30 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) {
|
||||
)
|
||||
}
|
||||
|
||||
// Parse the retention sweep interval; a set-but-unparseable value
|
||||
// is a hard error so fx aborts startup rather than silently using
|
||||
// the default.
|
||||
retentionSweepInterval, err := envDuration(
|
||||
"RETENTION_SWEEP_INTERVAL",
|
||||
defaultRetentionSweepInterval,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Load configuration values from environment variables
|
||||
s := &Config{
|
||||
DataDir: envString("DATA_DIR"),
|
||||
Debug: envBool("DEBUG", false),
|
||||
MaintenanceMode: envBool("MAINTENANCE_MODE", false),
|
||||
Environment: environment,
|
||||
MetricsUsername: envString("METRICS_USERNAME"),
|
||||
MetricsPassword: envString("METRICS_PASSWORD"),
|
||||
Port: envInt("PORT", defaultPort),
|
||||
SentryDSN: envString("SENTRY_DSN"),
|
||||
log: log,
|
||||
params: ¶ms,
|
||||
DataDir: envString("DATA_DIR"),
|
||||
Debug: envBool("DEBUG", false),
|
||||
MaintenanceMode: envBool("MAINTENANCE_MODE", false),
|
||||
Environment: environment,
|
||||
MetricsUsername: envString("METRICS_USERNAME"),
|
||||
MetricsPassword: envString("METRICS_PASSWORD"),
|
||||
Port: envInt("PORT", defaultPort),
|
||||
SentryDSN: envString("SENTRY_DSN"),
|
||||
RetentionSweepInterval: retentionSweepInterval,
|
||||
log: log,
|
||||
params: ¶ms,
|
||||
}
|
||||
|
||||
// Set default DataDir. All SQLite databases (main application
|
||||
@@ -151,6 +196,7 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) {
|
||||
"debug", s.Debug,
|
||||
"maintenanceMode", s.MaintenanceMode,
|
||||
"dataDir", s.DataDir,
|
||||
"retentionSweepInterval", s.RetentionSweepInterval.String(),
|
||||
"hasSentryDSN", s.SentryDSN != "",
|
||||
"hasMetricsAuth",
|
||||
s.MetricsUsername != "" && s.MetricsPassword != "",
|
||||
|
||||
@@ -3,6 +3,7 @@ package config_test
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -120,6 +121,100 @@ func testEnvironmentConfigSuccess(
|
||||
assert.Equal(t, isProd, cfg.IsProd())
|
||||
}
|
||||
|
||||
func TestRetentionSweepInterval(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
set bool
|
||||
value string
|
||||
expectError bool
|
||||
expected time.Duration
|
||||
}{
|
||||
{
|
||||
name: "unset uses default",
|
||||
set: false,
|
||||
expected: time.Hour,
|
||||
},
|
||||
{
|
||||
name: "valid value is parsed",
|
||||
set: true,
|
||||
value: "15m",
|
||||
expected: 15 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "unparseable value fails startup",
|
||||
set: true,
|
||||
value: "not-a-duration",
|
||||
expectError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Cannot use t.Parallel() here because t.Setenv
|
||||
// is incompatible with parallel subtests.
|
||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||
|
||||
if tt.set {
|
||||
t.Setenv("RETENTION_SWEEP_INTERVAL", tt.value)
|
||||
} else {
|
||||
require.NoError(t, os.Unsetenv(
|
||||
"RETENTION_SWEEP_INTERVAL",
|
||||
))
|
||||
}
|
||||
|
||||
if tt.expectError {
|
||||
testRetentionSweepIntervalError(t)
|
||||
} else {
|
||||
testRetentionSweepIntervalSuccess(t, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func testRetentionSweepIntervalError(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
var cfg *config.Config
|
||||
|
||||
app := fx.New(
|
||||
fx.NopLogger,
|
||||
fx.Provide(
|
||||
globals.New,
|
||||
logger.New,
|
||||
config.New,
|
||||
),
|
||||
fx.Populate(&cfg),
|
||||
)
|
||||
|
||||
assert.Error(t, app.Err())
|
||||
}
|
||||
|
||||
func testRetentionSweepIntervalSuccess(
|
||||
t *testing.T,
|
||||
expected time.Duration,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
var cfg *config.Config
|
||||
|
||||
app := fxtest.New(
|
||||
t,
|
||||
fx.Provide(
|
||||
globals.New,
|
||||
logger.New,
|
||||
config.New,
|
||||
),
|
||||
fx.Populate(&cfg),
|
||||
)
|
||||
require.NoError(t, app.Err())
|
||||
|
||||
app.RequireStart()
|
||||
|
||||
defer app.RequireStop()
|
||||
|
||||
assert.Equal(t, expected, cfg.RetentionSweepInterval)
|
||||
}
|
||||
|
||||
func TestDefaultDataDir(t *testing.T) {
|
||||
for _, env := range []string{"", "dev", "prod"} {
|
||||
name := env
|
||||
|
||||
Reference in New Issue
Block a user