fix: cancel in-progress deploy on new webhook trigger

When a webhook-triggered deploy starts for an app that already has a deploy
in progress, the new deploy now cancels the existing one via context
cancellation, waits for the lock to be released, and then starts the new
deploy.

Changes:
- Add per-app context cancellation (appCancels sync.Map) to deploy.Service
- Deploy() creates a cancellable context and registers it for the app
- Add CancelAppDeploy() method to cancel an in-progress deploy
- Add ErrDeployCancelled sentinel error for cancelled deploys
- Handle context cancellation in build and deploy phases, marking
  deployments as failed with a clear cancellation message
- Webhook triggerDeployment() now cancels in-progress deploys and retries
  until the lock is released (up to 30 attempts with 2s delay)

fixes #38
This commit is contained in:
user
2026-02-15 22:13:20 -08:00
parent 07ac71974c
commit 16640ef88e
4 changed files with 244 additions and 3 deletions

View File

@@ -5,8 +5,10 @@ import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"log/slog"
"time"
"go.uber.org/fx"
@@ -129,6 +131,12 @@ func (svc *Service) HandleWebhook(
return nil
}
// cancelRetryDelay is the time to wait after cancelling a deploy before retrying.
const cancelRetryDelay = 2 * time.Second
// cancelRetryAttempts is the maximum number of times to retry after cancelling.
const cancelRetryAttempts = 30
func (svc *Service) triggerDeployment(
ctx context.Context,
app *models.App,
@@ -144,6 +152,25 @@ func (svc *Service) triggerDeployment(
deployCtx := context.WithoutCancel(ctx)
deployErr := svc.deploy.Deploy(deployCtx, app, &eventID)
if deployErr != nil && errors.Is(deployErr, deploy.ErrDeploymentInProgress) {
// Cancel the in-progress deployment and retry
svc.log.Info("cancelling in-progress deployment for new webhook trigger", "app", appName)
svc.deploy.CancelAppDeploy(app.ID)
// Retry until the lock is released by the cancelled deploy
for attempt := range cancelRetryAttempts {
time.Sleep(cancelRetryDelay)
svc.log.Info("retrying deployment after cancel",
"app", appName, "attempt", attempt+1)
deployErr = svc.deploy.Deploy(deployCtx, app, &eventID)
if deployErr == nil || !errors.Is(deployErr, deploy.ErrDeploymentInProgress) {
break
}
}
}
if deployErr != nil {
svc.log.Error("deployment failed", "error", deployErr, "app", appName)
}