All checks were successful
check / check (push) Successful in 3m10s
The per-webhook archiveWriter registry in the database delivery target was never evicted, so a deleted webhook's writer -- and any archive file handle open within its debounce window -- lingered for the process lifetime. Separately, expiry pruning ran only when an archive was (re)opened, and reopens only happen on writes, so an archive belonging to a webhook that stopped receiving events kept its expired rows forever. Eviction: a new one-method delivery.WebhookEvictor interface (kept separate from Notifier: archiving lifecycle is not notification) is implemented by the Engine and injected into the handlers. Deleting a webhook, or deleting its last database target, drops the writer from the registry and closes its handle under the writer's own mutex, so eviction can never race an in-flight write. An evicted writer refuses further writes rather than reopening a file nothing holds. The archive file is deliberately left on disk: it is long-term storage an operator may want to keep or move away, and destroying it as a side effect of deleting a webhook would be unrecoverable. Idle sweep: a new ArchiveSweeper, modelled on the event RetentionReaper (fx lifecycle hooks, cancellable context, WaitGroup, ticker loop), prunes archives whose database target declares a positive expiry. It reuses the existing RETENTION_SWEEP_INTERVAL rather than adding a config key. It never creates an archive -- a missing file is skipped, and the reopen uses SQLite mode=rw so the file cannot be conjured even if it disappears mid-sweep -- routes the prune through the per-webhook writer so its mutex orders the sweep against concurrent writes, and leaves the archive closed so the move-the-file-away workflow keeps working. A failure for one webhook is logged and the sweep continues. Archives with no expiry or the expiry "never" are untouched.
421 lines
9.7 KiB
Go
421 lines
9.7 KiB
Go
package delivery
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"net"
|
|
"net/http"
|
|
"time"
|
|
|
|
"gorm.io/gorm"
|
|
"sneak.berlin/go/webhooker/internal/database"
|
|
)
|
|
|
|
// Exported constants for test access.
|
|
const (
|
|
ExportDeliveryChannelSize = deliveryChannelSize
|
|
ExportRetryChannelSize = retryChannelSize
|
|
ExportDefaultFailureThreshold = defaultFailureThreshold
|
|
ExportDefaultCooldown = defaultCooldown
|
|
)
|
|
|
|
// ExportIsBlockedIP exposes isBlockedIP for testing.
|
|
func ExportIsBlockedIP(ip net.IP) bool {
|
|
return isBlockedIP(ip)
|
|
}
|
|
|
|
// ExportBlockedNetworks exposes blockedNetworks.
|
|
func ExportBlockedNetworks() []*net.IPNet {
|
|
return blockedNetworks
|
|
}
|
|
|
|
// ExportIsForwardableHeader exposes isForwardableHeader.
|
|
func ExportIsForwardableHeader(name string) bool {
|
|
return isForwardableHeader(name)
|
|
}
|
|
|
|
// ExportTruncate exposes truncate for testing.
|
|
func ExportTruncate(s string, maxLen int) string {
|
|
return truncate(s, maxLen)
|
|
}
|
|
|
|
// ExportDeliverHTTP delivers via the http target for testing.
|
|
func (e *Engine) ExportDeliverHTTP(
|
|
ctx context.Context,
|
|
webhookDB *gorm.DB,
|
|
d *database.Delivery,
|
|
task *Task,
|
|
) {
|
|
e.httpTarget.Deliver(ctx, webhookDB, d, task, e)
|
|
}
|
|
|
|
// ExportDeliverDatabase delivers via the database target.
|
|
func (e *Engine) ExportDeliverDatabase(
|
|
webhookDB *gorm.DB, d *database.Delivery,
|
|
) {
|
|
e.targets[database.TargetTypeDatabase].Deliver(
|
|
context.Background(), webhookDB, d, &Task{}, e,
|
|
)
|
|
}
|
|
|
|
// ExportDeliverLog delivers via the log target for testing.
|
|
func (e *Engine) ExportDeliverLog(
|
|
webhookDB *gorm.DB, d *database.Delivery,
|
|
) {
|
|
e.targets[database.TargetTypeLog].Deliver(
|
|
context.Background(), webhookDB, d, &Task{}, e,
|
|
)
|
|
}
|
|
|
|
// ExportDeliverSlack delivers via the slack target for
|
|
// testing.
|
|
func (e *Engine) ExportDeliverSlack(
|
|
ctx context.Context,
|
|
webhookDB *gorm.DB,
|
|
d *database.Delivery,
|
|
) {
|
|
task := &Task{
|
|
DeliveryID: d.ID,
|
|
TargetID: d.TargetID,
|
|
AttemptNum: 1,
|
|
}
|
|
|
|
e.targets[database.TargetTypeSlack].Deliver(
|
|
ctx, webhookDB, d, task, e,
|
|
)
|
|
}
|
|
|
|
// ExportProcessNewTask exposes processNewTask.
|
|
func (e *Engine) ExportProcessNewTask(
|
|
ctx context.Context, task *Task,
|
|
) {
|
|
e.processNewTask(ctx, task)
|
|
}
|
|
|
|
// ExportProcessRetryTask exposes processRetryTask.
|
|
func (e *Engine) ExportProcessRetryTask(
|
|
ctx context.Context, task *Task,
|
|
) {
|
|
e.processRetryTask(ctx, task)
|
|
}
|
|
|
|
// ExportProcessDelivery exposes processDelivery.
|
|
func (e *Engine) ExportProcessDelivery(
|
|
ctx context.Context,
|
|
webhookDB *gorm.DB,
|
|
d *database.Delivery,
|
|
task *Task,
|
|
) {
|
|
e.processDelivery(ctx, webhookDB, d, task)
|
|
}
|
|
|
|
// ExportGetCircuitBreaker exposes the http target's
|
|
// getCircuitBreaker.
|
|
func (e *Engine) ExportGetCircuitBreaker(
|
|
targetID string,
|
|
) *CircuitBreaker {
|
|
return e.httpTarget.getCircuitBreaker(targetID)
|
|
}
|
|
|
|
// ExportParseHTTPConfig exposes parseHTTPConfig.
|
|
func (e *Engine) ExportParseHTTPConfig(
|
|
configJSON string,
|
|
) (*HTTPTargetConfig, error) {
|
|
return parseHTTPConfig(configJSON)
|
|
}
|
|
|
|
// ExportParseSlackConfig exposes parseSlackConfig.
|
|
func (e *Engine) ExportParseSlackConfig(
|
|
configJSON string,
|
|
) (*SlackTargetConfig, error) {
|
|
return parseSlackConfig(configJSON)
|
|
}
|
|
|
|
// ExportDoHTTPRequest exposes the http target's
|
|
// doHTTPRequest.
|
|
func (e *Engine) ExportDoHTTPRequest(
|
|
ctx context.Context,
|
|
cfg *HTTPTargetConfig,
|
|
event *database.Event,
|
|
) (int, string, int64, error) {
|
|
return e.httpTarget.doHTTPRequest(ctx, cfg, event)
|
|
}
|
|
|
|
// ExportClientForConfig exposes the http target's
|
|
// clientForConfig.
|
|
func (e *Engine) ExportClientForConfig(
|
|
cfg *HTTPTargetConfig,
|
|
) *http.Client {
|
|
return e.httpTarget.clientForConfig(cfg)
|
|
}
|
|
|
|
// ExportClient returns the http target's shared HTTP client.
|
|
func (e *Engine) ExportClient() *http.Client {
|
|
return e.httpTarget.client
|
|
}
|
|
|
|
// ExportScheduleRetry exposes ScheduleRetry.
|
|
func (e *Engine) ExportScheduleRetry(
|
|
task Task, delay time.Duration,
|
|
) {
|
|
e.ScheduleRetry(task, delay)
|
|
}
|
|
|
|
// ExportRecoverPendingDeliveries exposes
|
|
// recoverPendingDeliveries.
|
|
func (e *Engine) ExportRecoverPendingDeliveries(
|
|
ctx context.Context,
|
|
webhookDB *gorm.DB,
|
|
webhookID string,
|
|
) {
|
|
e.recoverPendingDeliveries(
|
|
ctx, webhookDB, webhookID,
|
|
)
|
|
}
|
|
|
|
// ExportRecoverWebhookDeliveries exposes
|
|
// recoverWebhookDeliveries.
|
|
func (e *Engine) ExportRecoverWebhookDeliveries(
|
|
ctx context.Context, webhookID string,
|
|
) {
|
|
e.recoverWebhookDeliveries(ctx, webhookID)
|
|
}
|
|
|
|
// ExportRecoverInFlight exposes recoverInFlight.
|
|
func (e *Engine) ExportRecoverInFlight(
|
|
ctx context.Context,
|
|
) {
|
|
e.recoverInFlight(ctx)
|
|
}
|
|
|
|
// ExportStart exposes start for testing.
|
|
func (e *Engine) ExportStart(ctx context.Context) {
|
|
e.start(ctx)
|
|
}
|
|
|
|
// ExportStop exposes stop for testing.
|
|
func (e *Engine) ExportStop() {
|
|
e.stop()
|
|
}
|
|
|
|
// ExportDeliveryCh returns the delivery channel.
|
|
func (e *Engine) ExportDeliveryCh() chan Task {
|
|
return e.deliveryCh
|
|
}
|
|
|
|
// ExportRetryCh returns the retry channel.
|
|
func (e *Engine) ExportRetryCh() chan Task {
|
|
return e.retryCh
|
|
}
|
|
|
|
// NewTestEngine creates an Engine for unit tests without
|
|
// database dependencies.
|
|
func NewTestEngine(
|
|
log *slog.Logger,
|
|
client *http.Client,
|
|
workers int,
|
|
) *Engine {
|
|
e := &Engine{
|
|
log: log,
|
|
deliveryCh: make(chan Task, deliveryChannelSize),
|
|
retryCh: make(chan Task, retryChannelSize),
|
|
workers: workers,
|
|
}
|
|
e.initTargets(client)
|
|
|
|
return e
|
|
}
|
|
|
|
// NewTestEngineSmallRetry creates an Engine with a tiny
|
|
// retry channel buffer for overflow testing.
|
|
func NewTestEngineSmallRetry(
|
|
log *slog.Logger,
|
|
) *Engine {
|
|
e := &Engine{
|
|
log: log,
|
|
retryCh: make(chan Task, 1),
|
|
}
|
|
e.initTargets(nil)
|
|
|
|
return e
|
|
}
|
|
|
|
// NewTestEngineWithDB creates an Engine with a real
|
|
// database and dbManager for integration tests.
|
|
func NewTestEngineWithDB(
|
|
db *database.Database,
|
|
dbMgr *database.WebhookDBManager,
|
|
log *slog.Logger,
|
|
client *http.Client,
|
|
workers int,
|
|
) *Engine {
|
|
e := &Engine{
|
|
database: db,
|
|
dbManager: dbMgr,
|
|
log: log,
|
|
deliveryCh: make(chan Task, deliveryChannelSize),
|
|
retryCh: make(chan Task, retryChannelSize),
|
|
workers: workers,
|
|
}
|
|
e.initTargets(client)
|
|
|
|
return e
|
|
}
|
|
|
|
// NewTestCircuitBreaker creates a CircuitBreaker with
|
|
// custom settings for testing.
|
|
func NewTestCircuitBreaker(
|
|
threshold int, cooldown time.Duration,
|
|
) *CircuitBreaker {
|
|
return &CircuitBreaker{
|
|
state: CircuitClosed,
|
|
threshold: threshold,
|
|
cooldown: cooldown,
|
|
}
|
|
}
|
|
|
|
// ExportArchivedEvent aliases the archive row type so black-box
|
|
// tests can construct and read archive rows.
|
|
type ExportArchivedEvent = archivedEvent
|
|
|
|
// ExportArchiveWriter wraps an archiveWriter so black-box tests
|
|
// can exercise the per-webhook archive file mechanics.
|
|
type ExportArchiveWriter struct {
|
|
w *archiveWriter
|
|
}
|
|
|
|
// NewExportArchiveWriter builds an archive writer for tests,
|
|
// optionally overriding the reopen debounce (a non-positive
|
|
// debounce keeps the production default).
|
|
func NewExportArchiveWriter(
|
|
path string, log *slog.Logger, debounce time.Duration,
|
|
) *ExportArchiveWriter {
|
|
w := newArchiveWriter(path, log)
|
|
if debounce > 0 {
|
|
w.debounce = debounce
|
|
}
|
|
|
|
return &ExportArchiveWriter{w: w}
|
|
}
|
|
|
|
// Write archives a row through the writer.
|
|
func (e *ExportArchiveWriter) Write(
|
|
row ExportArchivedEvent, expiry time.Duration,
|
|
) error {
|
|
return e.w.write(row, expiry)
|
|
}
|
|
|
|
// Open opens the archive file, pruning when expiry is positive.
|
|
func (e *ExportArchiveWriter) Open(expiry time.Duration) error {
|
|
return e.w.open(expiry)
|
|
}
|
|
|
|
// Reopen closes and reopens the archive file.
|
|
func (e *ExportArchiveWriter) Reopen(
|
|
expiry time.Duration,
|
|
) error {
|
|
return e.w.reopen(expiry)
|
|
}
|
|
|
|
// Reopens reports how many times the file has been opened.
|
|
func (e *ExportArchiveWriter) Reopens() int {
|
|
return e.w.reopens
|
|
}
|
|
|
|
// DB returns the writer's current open handle for row
|
|
// inspection in tests.
|
|
func (e *ExportArchiveWriter) DB() *gorm.DB {
|
|
return e.w.db
|
|
}
|
|
|
|
// ExportHasArchiveWriter reports whether the database target
|
|
// currently caches an archive writer for a webhook.
|
|
func (e *Engine) ExportHasArchiveWriter(
|
|
webhookID string,
|
|
) bool {
|
|
e.dbTarget.mu.Lock()
|
|
defer e.dbTarget.mu.Unlock()
|
|
|
|
_, ok := e.dbTarget.writers[webhookID]
|
|
|
|
return ok
|
|
}
|
|
|
|
// ExportArchiveHandleOpen reports whether the cached archive
|
|
// writer for a webhook holds an open database handle. It
|
|
// returns false when no writer is cached.
|
|
func (e *Engine) ExportArchiveHandleOpen(
|
|
webhookID string,
|
|
) bool {
|
|
e.dbTarget.mu.Lock()
|
|
w, ok := e.dbTarget.writers[webhookID]
|
|
e.dbTarget.mu.Unlock()
|
|
|
|
if !ok {
|
|
return false
|
|
}
|
|
|
|
w.mu.Lock()
|
|
defer w.mu.Unlock()
|
|
|
|
return w.db != nil
|
|
}
|
|
|
|
// ExportEnsureArchiveWriter creates (if needed) and returns the
|
|
// archive file path of the cached writer for a webhook, so a
|
|
// test can prime the registry the way a delivery would.
|
|
func (e *Engine) ExportEnsureArchiveWriter(
|
|
webhookID string,
|
|
) (string, error) {
|
|
w, err := e.dbTarget.writerFor(webhookID)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
return w.path, nil
|
|
}
|
|
|
|
// NewTestArchiveSweeper builds an ArchiveSweeper backed by the
|
|
// given main database and engine, without the fx lifecycle.
|
|
// Intended for tests.
|
|
func NewTestArchiveSweeper(
|
|
db *database.Database,
|
|
eng *Engine,
|
|
log *slog.Logger,
|
|
) *ArchiveSweeper {
|
|
return &ArchiveSweeper{
|
|
db: db,
|
|
eng: eng,
|
|
log: log,
|
|
interval: time.Hour,
|
|
}
|
|
}
|
|
|
|
// ExportSweep runs a single archive sweep synchronously for
|
|
// tests.
|
|
func (s *ArchiveSweeper) ExportSweep(ctx context.Context) {
|
|
s.sweep(ctx)
|
|
}
|
|
|
|
// ExportStart starts the sweeper's background loop for tests.
|
|
func (s *ArchiveSweeper) ExportStart(ctx context.Context) {
|
|
s.start(ctx)
|
|
}
|
|
|
|
// ExportStop stops the sweeper's background loop for tests.
|
|
func (s *ArchiveSweeper) ExportStop() {
|
|
s.stop()
|
|
}
|
|
|
|
// ExportSetInterval overrides the sweep interval for tests.
|
|
func (s *ArchiveSweeper) ExportSetInterval(d time.Duration) {
|
|
s.interval = d
|
|
}
|
|
|
|
// ExportParseArchiveExpiry exposes parseArchiveExpiry.
|
|
func ExportParseArchiveExpiry(
|
|
configJSON string,
|
|
) (time.Duration, error) {
|
|
return parseArchiveExpiry(configJSON)
|
|
}
|