Open SQLite with WAL and bound delivery re-dispatch by ownership (closes #256)

internal/delivery/target_http.go

@@ -77,14 +77,19 @@ func (c *httpCore) fireAndForget(
 ) {
 	c.eng.observeAttempt(d.Target.Type, res.elapsed())
 
-	c.eng.recordResult(
+	err := c.eng.recordResult(
 		webhookDB, d, 1, res.success,
 		res.statusCode, res.respBody, res.errMsg,
 		res.duration,
 	)
+	if err != nil {
+		c.eng.bookkeepingFailed(d, err)
+
+		return
+	}
 
 	if res.success {
-		c.eng.updateDeliveryStatus(
+		c.eng.settleStatus(
 			webhookDB, d, d.Target.Type,
 			database.DeliveryStatusDelivered,
 		)
@@ -92,7 +97,7 @@ func (c *httpCore) fireAndForget(
 		return
 	}
 
-	c.eng.updateDeliveryStatus(
+	c.eng.settleStatus(
 		webhookDB, d, d.Target.Type,
 		database.DeliveryStatusFailed,
 	)
@@ -122,16 +127,25 @@ func (c *httpCore) withRetry(
 
 	c.eng.observeAttempt(d.Target.Type, res.elapsed())
 
-	c.eng.recordResult(
+	err := c.eng.recordResult(
 		webhookDB, d, attemptNum, res.success,
 		res.statusCode, res.respBody, res.errMsg,
 		res.duration,
 	)
+	if err != nil {
+		// The breaker still learns the outcome: it describes the
+		// target's health, which is unaffected by this database's.
+		c.recordCircuitOutcome(cb, res.success)
+
+		c.eng.bookkeepingFailed(d, err)
+
+		return
+	}
 
 	if res.success {
 		cb.RecordSuccess()
 
-		c.eng.updateDeliveryStatus(
+		c.eng.settleStatus(
 			webhookDB, d, d.Target.Type,
 			database.DeliveryStatusDelivered,
 		)
@@ -146,6 +160,20 @@ func (c *httpCore) withRetry(
 	)
 }
 
+// recordCircuitOutcome feeds one attempt's outcome to the target's
+// circuit breaker.
+func (c *httpCore) recordCircuitOutcome(
+	cb *CircuitBreaker, success bool,
+) {
+	if success {
+		cb.RecordSuccess()
+
+		return
+	}
+
+	cb.RecordFailure()
+}
+
 func (c *httpCore) circuitBreakerBlock(
 	webhookDB *gorm.DB,
 	d *database.Delivery,
@@ -169,7 +197,7 @@ func (c *httpCore) circuitBreakerBlock(
 		"cooldown_remaining", remaining,
 	)
 
-	c.eng.updateDeliveryStatus(
+	c.eng.settleStatus(
 		webhookDB, d, d.Target.Type,
 		database.DeliveryStatusRetrying,
 	)
@@ -189,7 +217,7 @@ func (c *httpCore) handleRetry(
 	attemptNum int,
 ) {
 	if attemptNum >= maxRetries {
-		c.eng.updateDeliveryStatus(
+		c.eng.settleStatus(
 			webhookDB, d, d.Target.Type,
 			database.DeliveryStatusFailed,
 		)
@@ -197,7 +225,7 @@ func (c *httpCore) handleRetry(
 		return
 	}
 
-	c.eng.updateDeliveryStatus(
+	c.eng.settleStatus(
 		webhookDB, d, d.Target.Type,
 		database.DeliveryStatusRetrying,
 	)
@@ -332,12 +360,17 @@ func (t *httpTarget) Deliver(
 			"error", err,
 		)
 
-		t.eng.recordResult(
+		recErr := t.eng.recordResult(
 			webhookDB, d, task.AttemptNum,
 			false, 0, "", err.Error(), 0,
 		)
+		if recErr != nil {
+			t.eng.bookkeepingFailed(d, recErr)
 
-		t.eng.updateDeliveryStatus(
+			return
+		}
+
+		t.eng.settleStatus(
 			webhookDB, d, d.Target.Type,
 			database.DeliveryStatusFailed,
 		)