fix: cancel in-progress deploy on new webhook trigger
When a webhook-triggered deploy starts for an app that already has a deploy in progress, the new deploy now cancels the existing one via context cancellation, waits for the lock to be released, and then starts the new deploy. Changes: - Add per-app context cancellation (appCancels sync.Map) to deploy.Service - Deploy() creates a cancellable context and registers it for the app - Add CancelAppDeploy() method to cancel an in-progress deploy - Add ErrDeployCancelled sentinel error for cancelled deploys - Handle context cancellation in build and deploy phases, marking deployments as failed with a clear cancellation message - Webhook triggerDeployment() now cancels in-progress deploys and retries until the lock is released (up to 30 attempts with 2s delay) fixes #38
This commit is contained in:
@@ -5,8 +5,10 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"go.uber.org/fx"
|
||||
|
||||
@@ -129,6 +131,12 @@ func (svc *Service) HandleWebhook(
|
||||
return nil
|
||||
}
|
||||
|
||||
// cancelRetryDelay is the time to wait after cancelling a deploy before retrying.
|
||||
const cancelRetryDelay = 2 * time.Second
|
||||
|
||||
// cancelRetryAttempts is the maximum number of times to retry after cancelling.
|
||||
const cancelRetryAttempts = 30
|
||||
|
||||
func (svc *Service) triggerDeployment(
|
||||
ctx context.Context,
|
||||
app *models.App,
|
||||
@@ -144,6 +152,25 @@ func (svc *Service) triggerDeployment(
|
||||
deployCtx := context.WithoutCancel(ctx)
|
||||
|
||||
deployErr := svc.deploy.Deploy(deployCtx, app, &eventID)
|
||||
if deployErr != nil && errors.Is(deployErr, deploy.ErrDeploymentInProgress) {
|
||||
// Cancel the in-progress deployment and retry
|
||||
svc.log.Info("cancelling in-progress deployment for new webhook trigger", "app", appName)
|
||||
svc.deploy.CancelAppDeploy(app.ID)
|
||||
|
||||
// Retry until the lock is released by the cancelled deploy
|
||||
for attempt := range cancelRetryAttempts {
|
||||
time.Sleep(cancelRetryDelay)
|
||||
|
||||
svc.log.Info("retrying deployment after cancel",
|
||||
"app", appName, "attempt", attempt+1)
|
||||
|
||||
deployErr = svc.deploy.Deploy(deployCtx, app, &eventID)
|
||||
if deployErr == nil || !errors.Is(deployErr, deploy.ErrDeploymentInProgress) {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if deployErr != nil {
|
||||
svc.log.Error("deployment failed", "error", deployErr, "app", appName)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user