Deploy main eacf1e7 to prod

Replaces #559, which conflicted.

Model: opus-5-5

internal/delivery/export_test.go

@@ -9,18 +9,14 @@ import (
 	"net/url"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"go.uber.org/fx"
 	"gorm.io/gorm"
+	"sneak.berlin/go/webhooker/internal/archive"
 	"sneak.berlin/go/webhooker/internal/database"
 	"sneak.berlin/go/webhooker/internal/metrics"
 )
 
-// ErrExportArchiveWriterEvicted exposes the sentinel returned by
-// an evicted archive writer. It carries the Err prefix rather
-// than this file's usual Export one because it is a sentinel
-// error.
-var ErrExportArchiveWriterEvicted = errArchiveWriterEvicted
-
 // Exported constants for test access.
 const (
 	ExportDeliveryChannelSize     = deliveryChannelSize
@@ -40,11 +36,6 @@ const (
 	ExportPendingSweepMinAge = pendingSweepMinAge
 )
 
-// ExportIsBlockedIP exposes isBlockedIP for testing.
-func ExportIsBlockedIP(ip net.IP) bool {
-	return isBlockedIP(ip)
-}
-
 // NewTestGuard builds an SSRF Guard from an explicit egress
 // allowlist, without going through config. Passing no prefixes
 // yields the default guard, which blocks every private/reserved
@@ -70,6 +61,11 @@ func ExportBlockedNetworks() []*net.IPNet {
 	return blockedNetworks
 }
 
+// ExportBlockedPublicNetworks exposes blockedPublicNetworks.
+func ExportBlockedPublicNetworks() []*net.IPNet {
+	return blockedPublicNetworks
+}
+
 // ExportIsForwardableHeader exposes isForwardableHeader.
 func ExportIsForwardableHeader(name string) bool {
 	return isForwardableHeader(name)
@@ -81,9 +77,10 @@ func ExportIsForwardableHeader(name string) bool {
 func ExportApplyRequestHeaders(
 	req *http.Request,
 	event *database.Event,
-	cfg *HTTPTargetConfig,
+	configured map[string]string,
+	userAgent string,
 ) []string {
-	return applyRequestHeaders(req, event, cfg)
+	return applyRequestHeaders(req, event, configured, userAgent)
 }
 
 // ExportTruncate exposes truncate for testing.
@@ -140,9 +137,10 @@ func (e *Engine) ExportDeliverSlack(
 	d *database.Delivery,
 ) {
 	task := &Task{
-		DeliveryID: d.ID,
-		TargetID:   d.TargetID,
-		AttemptNum: 1,
+		DeliveryID:  d.ID,
+		TargetID:    d.TargetID,
+		MaxAttempts: d.Target.MaxAttempts,
+		AttemptNum:  1,
 	}
 
 	e.targets[database.TargetTypeSlack].Deliver(
@@ -150,6 +148,21 @@ func (e *Engine) ExportDeliverSlack(
 	)
 }
 
+// ExportUpdateDeliveryStatus exposes updateDeliveryStatus. It passes no
+// target type, so no metric moves.
+func (e *Engine) ExportUpdateDeliveryStatus(
+	webhookDB *gorm.DB,
+	d *database.Delivery,
+	status database.DeliveryStatus,
+) error {
+	return e.updateDeliveryStatus(webhookDB, d, "", status)
+}
+
+// ExportNotify exposes notify.
+func (e *Engine) ExportNotify(tasks []Task) {
+	e.notify(tasks)
+}
+
 // ExportProcessNewTask exposes processNewTask.
 func (e *Engine) ExportProcessNewTask(
 	ctx context.Context, task *Task,
@@ -200,37 +213,31 @@ func (e *Engine) ExportSetCircuitBreaker(
 	e.httpTarget.circuitBreakers.Store(targetID, cb)
 }
 
-// ExportParseHTTPConfig exposes parseHTTPConfig.
-func (e *Engine) ExportParseHTTPConfig(
-	configJSON string,
-) (*HTTPTargetConfig, error) {
-	return parseHTTPConfig(configJSON)
-}
-
-// ExportParseSlackConfig exposes parseSlackConfig.
-func (e *Engine) ExportParseSlackConfig(
-	configJSON string,
-) (*SlackTargetConfig, error) {
-	return parseSlackConfig(configJSON)
+// ExportSetSlackCircuitBreaker is ExportSetCircuitBreaker for the
+// slack target.
+func (e *Engine) ExportSetSlackCircuitBreaker(
+	targetID string, cb *CircuitBreaker,
+) {
+	e.slackTarget.circuitBreakers.Store(targetID, cb)
 }
 
 // ExportDoHTTPRequest exposes the http target's
 // doHTTPRequest.
 func (e *Engine) ExportDoHTTPRequest(
 	ctx context.Context,
-	cfg *HTTPTargetConfig,
+	target *database.Target,
 	event *database.Event,
 ) (int, string, int64, error) {
-	return e.httpTarget.doHTTPRequest(ctx, cfg, event)
+	return e.httpTarget.doHTTPRequest(ctx, target, event)
 }
 
 // ExportClientForRequest exposes the http target's
 // clientForRequest.
 func (e *Engine) ExportClientForRequest(
-	cfg *HTTPTargetConfig,
+	timeoutSeconds int,
 	originScoped []string,
 ) *http.Client {
-	return e.httpTarget.clientForRequest(cfg, originScoped)
+	return e.httpTarget.clientForRequest(timeoutSeconds, originScoped)
 }
 
 // ErrExportTooManyRedirects exposes the sentinel the redirect
@@ -265,32 +272,27 @@ func (e *Engine) ExportRecoverPendingDeliveries(
 	ctx context.Context,
 	webhookDB *gorm.DB,
 	webhookID string,
+	atStart bool,
 ) {
 	e.recoverPendingDeliveries(
-		ctx, webhookDB, webhookID,
+		ctx, webhookDB, webhookID, atStart,
 	)
 }
 
 // ExportRecoverWebhookDeliveries exposes
 // recoverWebhookDeliveries.
 func (e *Engine) ExportRecoverWebhookDeliveries(
-	ctx context.Context, webhookID string,
+	ctx context.Context, webhookID string, atStart bool,
 ) {
-	e.recoverWebhookDeliveries(ctx, webhookID)
+	e.recoverWebhookDeliveries(ctx, webhookID, atStart)
 }
 
-// ExportRecoverInFlight exposes recoverInFlight.
-func (e *Engine) ExportRecoverInFlight(
-	ctx context.Context,
+// ExportRecoverDeliveries exposes recoverDeliveries, the recovery
+// routine.
+func (e *Engine) ExportRecoverDeliveries(
+	ctx context.Context, atStart bool,
 ) {
-	e.recoverInFlight(ctx)
-}
-
-// ExportSweepWebhookRetries exposes sweepWebhookRetries.
-func (e *Engine) ExportSweepWebhookRetries(
-	ctx context.Context, webhookID string,
-) {
-	e.sweepWebhookRetries(ctx, webhookID)
+	e.recoverDeliveries(ctx, atStart)
 }
 
 // ExportStart exposes start for testing.
@@ -337,9 +339,9 @@ func (e *Engine) ExportRetainDelivery(deliveryID string) bool {
 
 // ExportRecoverRetryingDeliveries exposes recoverRetryingDeliveries.
 func (e *Engine) ExportRecoverRetryingDeliveries(
-	webhookDB *gorm.DB, webhookID string,
+	ctx context.Context, webhookDB *gorm.DB, webhookID string,
 ) {
-	e.recoverRetryingDeliveries(webhookDB, webhookID)
+	e.recoverRetryingDeliveries(ctx, webhookDB, webhookID)
 }
 
 // ExportFailMissingTarget exposes failMissingTarget, so a test can hand
@@ -378,7 +380,8 @@ func (e *Engine) ExportRetryCh() chan Task {
 }
 
 // NewTestEngine creates an Engine for unit tests without
-// database dependencies.
+// database dependencies. Its archive writers have no data
+// directory, so a delivery to a database target fails.
 func NewTestEngine(
 	log *slog.Logger,
 	client *http.Client,
@@ -389,7 +392,8 @@ func NewTestEngine(
 		deliveryCh: make(chan Task, deliveryChannelSize),
 		retryCh:    make(chan Task, retryChannelSize),
 		workers:    workers,
-		mtr:        metrics.Default(),
+		mtr:        metrics.New(prometheus.NewRegistry()),
+		archives:   archive.NewWriters(archive.WritersParams{Logger: log}),
 	}
 	e.initTargets(client)
 
@@ -402,9 +406,10 @@ func NewTestEngineSmallRetry(
 	log *slog.Logger,
 ) *Engine {
 	e := &Engine{
-		log:     log,
-		retryCh: make(chan Task, 1),
-		mtr:     metrics.Default(),
+		log:      log,
+		retryCh:  make(chan Task, 1),
+		mtr:      metrics.New(prometheus.NewRegistry()),
+		archives: archive.NewWriters(archive.WritersParams{Logger: log}),
 	}
 	e.initTargets(nil)
 
@@ -412,7 +417,8 @@ func NewTestEngineSmallRetry(
 }
 
 // NewTestEngineWithDB creates an Engine with a real
-// database and dbManager for integration tests.
+// database and dbManager for integration tests, and archive
+// writers of its own over the same two.
 func NewTestEngineWithDB(
 	db *database.Database,
 	dbMgr *database.WebhookDBManager,
@@ -427,16 +433,25 @@ func NewTestEngineWithDB(
 		deliveryCh: make(chan Task, deliveryChannelSize),
 		retryCh:    make(chan Task, retryChannelSize),
 		workers:    workers,
-		mtr:        metrics.Default(),
+		mtr:        metrics.New(prometheus.NewRegistry()),
+		archives: archive.NewWriters(archive.WritersParams{
+			Database:  db,
+			DBManager: dbMgr,
+			Logger:    log,
+		}),
 	}
 	e.initTargets(client)
 
 	return e
 }
 
+// ExportArchives returns the engine's archive writers.
+func (e *Engine) ExportArchives() *archive.Writers {
+	return e.archives
+}
+
 // ExportSetMetrics substitutes the engine's metric set, so a test can
-// assert on collectors registered on a private registry instead of
-// the process-wide ones every other test is also moving.
+// assert on collectors registered on a registry it holds.
 func (e *Engine) ExportSetMetrics(mtr *metrics.Set) {
 	e.mtr = mtr
 }
@@ -457,252 +472,3 @@ func NewTestCircuitBreaker(
 		cooldown:  cooldown,
 	}
 }
-
-// ExportArchivedEvent aliases the archive row type so black-box
-// tests can construct and read archive rows.
-type ExportArchivedEvent = archivedEvent
-
-// ExportArchiveWriter wraps an archiveWriter so black-box tests
-// can exercise the per-webhook archive file mechanics.
-type ExportArchiveWriter struct {
-	w *archiveWriter
-}
-
-// NewExportArchiveWriter builds an archive writer for tests,
-// optionally overriding the reopen debounce (a non-positive
-// debounce keeps the production default).
-func NewExportArchiveWriter(
-	path string, log *slog.Logger, debounce time.Duration,
-) *ExportArchiveWriter {
-	w := newArchiveWriter(path, log)
-	if debounce > 0 {
-		w.debounce = debounce
-	}
-
-	return &ExportArchiveWriter{w: w}
-}
-
-// Write archives a row through the writer.
-func (e *ExportArchiveWriter) Write(
-	row ExportArchivedEvent, expiry time.Duration,
-) error {
-	return e.w.write(row, expiry)
-}
-
-// Open opens the archive file, pruning when expiry is positive.
-func (e *ExportArchiveWriter) Open(expiry time.Duration) error {
-	return e.w.open(expiry)
-}
-
-// Reopen closes and reopens the archive file.
-func (e *ExportArchiveWriter) Reopen(
-	expiry time.Duration,
-) error {
-	return e.w.reopen(expiry)
-}
-
-// Reopens reports how many times the file has been opened.
-func (e *ExportArchiveWriter) Reopens() int {
-	return e.w.reopens
-}
-
-// DB returns the writer's current open handle for row
-// inspection in tests.
-func (e *ExportArchiveWriter) DB() *gorm.DB {
-	return e.w.db
-}
-
-// Path returns the archive file the writer owns.
-func (e *ExportArchiveWriter) Path() string {
-	return e.w.path
-}
-
-// OpenExisting opens the archive without permitting creation,
-// the way the idle sweep does.
-func (e *ExportArchiveWriter) OpenExisting(
-	expiry time.Duration,
-) error {
-	return e.w.openMode(archiveModeExisting, expiry)
-}
-
-// SweepExpired runs an idle sweep of the archive.
-func (e *ExportArchiveWriter) SweepExpired(
-	expiry time.Duration,
-) error {
-	return e.w.sweepExpired(expiry)
-}
-
-// Evict marks the writer evicted and closes its handle, exactly
-// as leaving the registry does.
-func (e *ExportArchiveWriter) Evict() {
-	e.w.evict()
-}
-
-// HandleOpen reports whether the writer currently holds an open
-// archive handle.
-func (e *ExportArchiveWriter) HandleOpen() bool {
-	e.w.mu.Lock()
-	defer e.w.mu.Unlock()
-
-	return e.w.db != nil
-}
-
-// Same reports whether both wrappers refer to the very same
-// underlying archive writer, so a test can prove a registry entry
-// is the writer it was handed rather than a replacement.
-func (e *ExportArchiveWriter) Same(
-	other *ExportArchiveWriter,
-) bool {
-	return other != nil && e.w == other.w
-}
-
-// ExportArchiveWriterFor returns the archive writer the registry
-// currently caches for a webhook, or nil when none is cached. It
-// never creates one, so a test can hold a reference to the very
-// writer an eviction is about to detach.
-func (e *Engine) ExportArchiveWriterFor(
-	webhookID string,
-) *ExportArchiveWriter {
-	e.dbTarget.mu.Lock()
-	defer e.dbTarget.mu.Unlock()
-
-	w, ok := e.dbTarget.writers[webhookID]
-	if !ok {
-		return nil
-	}
-
-	return &ExportArchiveWriter{w: w}
-}
-
-// ExportHasArchiveWriter reports whether the database target
-// currently caches an archive writer for a webhook.
-func (e *Engine) ExportHasArchiveWriter(
-	webhookID string,
-) bool {
-	e.dbTarget.mu.Lock()
-	defer e.dbTarget.mu.Unlock()
-
-	_, ok := e.dbTarget.writers[webhookID]
-
-	return ok
-}
-
-// ExportArchiveHandleOpen reports whether the cached archive
-// writer for a webhook holds an open database handle. It
-// returns false when no writer is cached.
-func (e *Engine) ExportArchiveHandleOpen(
-	webhookID string,
-) bool {
-	e.dbTarget.mu.Lock()
-	w, ok := e.dbTarget.writers[webhookID]
-	e.dbTarget.mu.Unlock()
-
-	if !ok {
-		return false
-	}
-
-	w.mu.Lock()
-	defer w.mu.Unlock()
-
-	return w.db != nil
-}
-
-// ExportEnsureArchiveWriter creates (if needed) and returns the
-// archive file path of the cached writer for a webhook, so a
-// test can prime the registry the way a delivery would.
-func (e *Engine) ExportEnsureArchiveWriter(
-	webhookID string,
-) (string, error) {
-	w, err := e.dbTarget.writerFor(webhookID)
-	if err != nil {
-		return "", err
-	}
-
-	return w.path, nil
-}
-
-// ExportSweepWriterFor takes a webhook's registry writer exactly
-// as the idle sweep does, reporting whether the sweep had to
-// create the entry. It lets a test drive the registry through the
-// sweep's own entry point instead of choreographing goroutines.
-func (e *Engine) ExportSweepWriterFor(
-	webhookID string,
-) (*ExportArchiveWriter, bool, error) {
-	w, created, err := e.dbTarget.sweepWriterFor(webhookID)
-	if err != nil {
-		return nil, false, err
-	}
-
-	return &ExportArchiveWriter{w: w}, created, nil
-}
-
-// ExportReleaseSweepWriter releases a sweep-created registry entry
-// exactly as a finished sweep does.
-func (e *Engine) ExportReleaseSweepWriter(
-	webhookID string, w *ExportArchiveWriter,
-) {
-	e.dbTarget.releaseSweepWriter(webhookID, w.w)
-}
-
-// NewTestArchiveSweeper builds an ArchiveSweeper backed by the
-// given main database and engine, without the fx lifecycle.
-// Intended for tests.
-func NewTestArchiveSweeper(
-	db *database.Database,
-	eng *Engine,
-	log *slog.Logger,
-) *ArchiveSweeper {
-	return &ArchiveSweeper{
-		db:       db,
-		eng:      eng,
-		log:      log,
-		interval: time.Hour,
-	}
-}
-
-// ExportSweep runs a single archive sweep synchronously for
-// tests.
-func (s *ArchiveSweeper) ExportSweep(ctx context.Context) {
-	s.sweep(ctx)
-}
-
-// ExportStart starts the sweeper's background loop for tests.
-func (s *ArchiveSweeper) ExportStart() {
-	s.start()
-}
-
-// ExportRegisterHooks registers the sweeper's real fx lifecycle
-// hooks on a lifecycle supplied by a test, so a test can drive
-// the exact OnStart/OnStop functions the application runs and
-// hand OnStart the kind of context fx actually supplies.
-func (s *ArchiveSweeper) ExportRegisterHooks(lc fx.Lifecycle) {
-	s.registerHooks(lc)
-}
-
-// ExportStop stops the sweeper's background loop for tests.
-func (s *ArchiveSweeper) ExportStop(ctx context.Context) error {
-	return s.stop(ctx)
-}
-
-// ExportWedgeLoop adds a goroutine to the sweeper's WaitGroup
-// that never observes cancellation and returns only when release
-// is closed. It stands in for a prune stuck on a locked archive.
-func (s *ArchiveSweeper) ExportWedgeLoop(
-	release <-chan struct{},
-) {
-	s.wg.Go(func() {
-		<-release
-	})
-}
-
-// ExportSetInterval overrides the sweep interval for tests.
-func (s *ArchiveSweeper) ExportSetInterval(d time.Duration) {
-	s.interval = d
-}
-
-// ExportParseArchiveExpiry exposes parseArchiveExpiry.
-func ExportParseArchiveExpiry(
-	configJSON string,
-) (time.Duration, error) {
-	return parseArchiveExpiry(configJSON)
-}