Use DataDir/builds instead of /tmp for clone directories so that bind mounts work correctly when upaas itself runs in a Docker container. The /tmp directory inside the upaas container isn't accessible to the Docker daemon on the host, causing bind mount failures. Also fix test setups to pass Config to deploy service and add delay to webhook test to avoid temp directory cleanup race with async deployment goroutine.
463 lines
11 KiB
Go
463 lines
11 KiB
Go
// Package deploy provides deployment services.
|
|
package deploy
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"go.uber.org/fx"
|
|
|
|
"git.eeqj.de/sneak/upaas/internal/config"
|
|
"git.eeqj.de/sneak/upaas/internal/database"
|
|
"git.eeqj.de/sneak/upaas/internal/docker"
|
|
"git.eeqj.de/sneak/upaas/internal/logger"
|
|
"git.eeqj.de/sneak/upaas/internal/models"
|
|
"git.eeqj.de/sneak/upaas/internal/service/notify"
|
|
)
|
|
|
|
// Time constants.
|
|
const (
|
|
healthCheckDelaySeconds = 60
|
|
// upaasLabelCount is the number of upaas-specific labels added to containers.
|
|
upaasLabelCount = 1
|
|
// buildsDirPermissions is the permission mode for the builds directory.
|
|
buildsDirPermissions = 0o750
|
|
)
|
|
|
|
// Sentinel errors for deployment failures.
|
|
var (
|
|
// ErrContainerUnhealthy indicates the container failed health check.
|
|
ErrContainerUnhealthy = errors.New("container unhealthy after 60 seconds")
|
|
)
|
|
|
|
// ServiceParams contains dependencies for Service.
|
|
type ServiceParams struct {
|
|
fx.In
|
|
|
|
Logger *logger.Logger
|
|
Config *config.Config
|
|
Database *database.Database
|
|
Docker *docker.Client
|
|
Notify *notify.Service
|
|
}
|
|
|
|
// Service provides deployment functionality.
|
|
type Service struct {
|
|
log *slog.Logger
|
|
db *database.Database
|
|
docker *docker.Client
|
|
notify *notify.Service
|
|
config *config.Config
|
|
params *ServiceParams
|
|
}
|
|
|
|
// New creates a new deploy Service.
|
|
func New(_ fx.Lifecycle, params ServiceParams) (*Service, error) {
|
|
return &Service{
|
|
log: params.Logger.Get(),
|
|
db: params.Database,
|
|
docker: params.Docker,
|
|
notify: params.Notify,
|
|
config: params.Config,
|
|
params: ¶ms,
|
|
}, nil
|
|
}
|
|
|
|
// GetBuildDir returns the build directory path for an app.
|
|
func (svc *Service) GetBuildDir(appID string) string {
|
|
return filepath.Join(svc.config.DataDir, "builds", appID)
|
|
}
|
|
|
|
// Deploy deploys an app.
|
|
func (svc *Service) Deploy(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
webhookEventID *int64,
|
|
) error {
|
|
deployment, err := svc.createDeploymentRecord(ctx, app, webhookEventID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = svc.updateAppStatusBuilding(ctx, app)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
svc.notify.NotifyBuildStart(ctx, app, deployment)
|
|
|
|
imageID, err := svc.buildImage(ctx, app, deployment)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
svc.notify.NotifyBuildSuccess(ctx, app, deployment)
|
|
|
|
err = svc.updateDeploymentDeploying(ctx, deployment)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
svc.removeOldContainer(ctx, app, deployment)
|
|
|
|
_, err = svc.createAndStartContainer(ctx, app, deployment, imageID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = svc.updateAppRunning(ctx, app, imageID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Use context.WithoutCancel to ensure health check completes even if
|
|
// the parent context is cancelled (e.g., HTTP request ends).
|
|
go svc.checkHealthAfterDelay(context.WithoutCancel(ctx), app, deployment)
|
|
|
|
return nil
|
|
}
|
|
|
|
func (svc *Service) createDeploymentRecord(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
webhookEventID *int64,
|
|
) (*models.Deployment, error) {
|
|
deployment := models.NewDeployment(svc.db)
|
|
deployment.AppID = app.ID
|
|
|
|
if webhookEventID != nil {
|
|
deployment.WebhookEventID = sql.NullInt64{
|
|
Int64: *webhookEventID,
|
|
Valid: true,
|
|
}
|
|
}
|
|
|
|
deployment.Status = models.DeploymentStatusBuilding
|
|
|
|
saveErr := deployment.Save(ctx)
|
|
if saveErr != nil {
|
|
return nil, fmt.Errorf("failed to create deployment: %w", saveErr)
|
|
}
|
|
|
|
return deployment, nil
|
|
}
|
|
|
|
func (svc *Service) updateAppStatusBuilding(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
) error {
|
|
app.Status = models.AppStatusBuilding
|
|
|
|
saveErr := app.Save(ctx)
|
|
if saveErr != nil {
|
|
return fmt.Errorf("failed to update app status: %w", saveErr)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (svc *Service) buildImage(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
deployment *models.Deployment,
|
|
) (string, error) {
|
|
tempDir, cleanup, err := svc.cloneRepository(ctx, app, deployment)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
defer cleanup()
|
|
|
|
imageTag := "upaas/" + app.Name + ":latest"
|
|
|
|
imageID, err := svc.docker.BuildImage(ctx, docker.BuildImageOptions{
|
|
ContextDir: tempDir,
|
|
DockerfilePath: app.DockerfilePath,
|
|
Tags: []string{imageTag},
|
|
})
|
|
if err != nil {
|
|
svc.notify.NotifyBuildFailed(ctx, app, deployment, err)
|
|
svc.failDeployment(
|
|
ctx,
|
|
app,
|
|
deployment,
|
|
fmt.Errorf("failed to build image: %w", err),
|
|
)
|
|
|
|
return "", fmt.Errorf("failed to build image: %w", err)
|
|
}
|
|
|
|
deployment.ImageID = sql.NullString{String: imageID, Valid: true}
|
|
_ = deployment.AppendLog(ctx, "Image built: "+imageID)
|
|
|
|
return imageID, nil
|
|
}
|
|
|
|
func (svc *Service) cloneRepository(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
deployment *models.Deployment,
|
|
) (string, func(), error) {
|
|
// Use a subdirectory of DataDir for builds since it's mounted from the host
|
|
// and accessible to Docker for bind mounts (unlike /tmp inside the container)
|
|
buildsDir := filepath.Join(svc.config.DataDir, "builds")
|
|
|
|
err := os.MkdirAll(buildsDir, buildsDirPermissions)
|
|
if err != nil {
|
|
svc.failDeployment(ctx, app, deployment, fmt.Errorf("failed to create builds dir: %w", err))
|
|
|
|
return "", nil, fmt.Errorf("failed to create builds dir: %w", err)
|
|
}
|
|
|
|
tempDir, err := os.MkdirTemp(buildsDir, app.ID+"-*")
|
|
if err != nil {
|
|
svc.failDeployment(ctx, app, deployment, fmt.Errorf("failed to create temp dir: %w", err))
|
|
|
|
return "", nil, fmt.Errorf("failed to create temp dir: %w", err)
|
|
}
|
|
|
|
cleanup := func() { _ = os.RemoveAll(tempDir) }
|
|
|
|
cloneErr := svc.docker.CloneRepo(ctx, app.RepoURL, app.Branch, app.SSHPrivateKey, tempDir)
|
|
if cloneErr != nil {
|
|
cleanup()
|
|
svc.failDeployment(ctx, app, deployment, fmt.Errorf("failed to clone repo: %w", cloneErr))
|
|
|
|
return "", nil, fmt.Errorf("failed to clone repo: %w", cloneErr)
|
|
}
|
|
|
|
_ = deployment.AppendLog(ctx, "Repository cloned successfully")
|
|
|
|
return tempDir, cleanup, nil
|
|
}
|
|
|
|
func (svc *Service) updateDeploymentDeploying(
|
|
ctx context.Context,
|
|
deployment *models.Deployment,
|
|
) error {
|
|
deployment.Status = models.DeploymentStatusDeploying
|
|
|
|
saveErr := deployment.Save(ctx)
|
|
if saveErr != nil {
|
|
return fmt.Errorf("failed to update deployment status: %w", saveErr)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (svc *Service) removeOldContainer(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
deployment *models.Deployment,
|
|
) {
|
|
containerInfo, err := svc.docker.FindContainerByAppID(ctx, app.ID)
|
|
if err != nil || containerInfo == nil {
|
|
return
|
|
}
|
|
|
|
svc.log.Info("removing old container", "id", containerInfo.ID)
|
|
|
|
removeErr := svc.docker.RemoveContainer(ctx, containerInfo.ID, true)
|
|
if removeErr != nil {
|
|
svc.log.Warn("failed to remove old container", "error", removeErr)
|
|
}
|
|
|
|
_ = deployment.AppendLog(ctx, "Old container removed")
|
|
}
|
|
|
|
func (svc *Service) createAndStartContainer(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
deployment *models.Deployment,
|
|
imageID string,
|
|
) (string, error) {
|
|
containerOpts, err := svc.buildContainerOptions(ctx, app, imageID)
|
|
if err != nil {
|
|
svc.failDeployment(ctx, app, deployment, err)
|
|
|
|
return "", err
|
|
}
|
|
|
|
containerID, err := svc.docker.CreateContainer(ctx, containerOpts)
|
|
if err != nil {
|
|
svc.notify.NotifyDeployFailed(ctx, app, deployment, err)
|
|
svc.failDeployment(
|
|
ctx,
|
|
app,
|
|
deployment,
|
|
fmt.Errorf("failed to create container: %w", err),
|
|
)
|
|
|
|
return "", fmt.Errorf("failed to create container: %w", err)
|
|
}
|
|
|
|
deployment.ContainerID = sql.NullString{String: containerID, Valid: true}
|
|
_ = deployment.AppendLog(ctx, "Container created: "+containerID)
|
|
|
|
startErr := svc.docker.StartContainer(ctx, containerID)
|
|
if startErr != nil {
|
|
svc.notify.NotifyDeployFailed(ctx, app, deployment, startErr)
|
|
svc.failDeployment(
|
|
ctx,
|
|
app,
|
|
deployment,
|
|
fmt.Errorf("failed to start container: %w", startErr),
|
|
)
|
|
|
|
return "", fmt.Errorf("failed to start container: %w", startErr)
|
|
}
|
|
|
|
_ = deployment.AppendLog(ctx, "Container started")
|
|
|
|
return containerID, nil
|
|
}
|
|
|
|
func (svc *Service) buildContainerOptions(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
_ string,
|
|
) (docker.CreateContainerOptions, error) {
|
|
envVars, err := app.GetEnvVars(ctx)
|
|
if err != nil {
|
|
return docker.CreateContainerOptions{}, fmt.Errorf("failed to get env vars: %w", err)
|
|
}
|
|
|
|
labels, err := app.GetLabels(ctx)
|
|
if err != nil {
|
|
return docker.CreateContainerOptions{}, fmt.Errorf("failed to get labels: %w", err)
|
|
}
|
|
|
|
volumes, err := app.GetVolumes(ctx)
|
|
if err != nil {
|
|
return docker.CreateContainerOptions{}, fmt.Errorf("failed to get volumes: %w", err)
|
|
}
|
|
|
|
envMap := make(map[string]string, len(envVars))
|
|
for _, envVar := range envVars {
|
|
envMap[envVar.Key] = envVar.Value
|
|
}
|
|
|
|
network := ""
|
|
if app.DockerNetwork.Valid {
|
|
network = app.DockerNetwork.String
|
|
}
|
|
|
|
return docker.CreateContainerOptions{
|
|
Name: "upaas-" + app.Name,
|
|
Image: "upaas/" + app.Name + ":latest",
|
|
Env: envMap,
|
|
Labels: buildLabelMap(app, labels),
|
|
Volumes: buildVolumeMounts(volumes),
|
|
Network: network,
|
|
}, nil
|
|
}
|
|
|
|
func buildLabelMap(app *models.App, labels []*models.Label) map[string]string {
|
|
labelMap := make(map[string]string, len(labels)+upaasLabelCount)
|
|
for _, label := range labels {
|
|
labelMap[label.Key] = label.Value
|
|
}
|
|
|
|
// Add the upaas.id label to identify this container
|
|
labelMap[docker.LabelUpaasID] = app.ID
|
|
|
|
return labelMap
|
|
}
|
|
|
|
func buildVolumeMounts(volumes []*models.Volume) []docker.VolumeMount {
|
|
mounts := make([]docker.VolumeMount, 0, len(volumes))
|
|
for _, vol := range volumes {
|
|
mounts = append(mounts, docker.VolumeMount{
|
|
HostPath: vol.HostPath,
|
|
ContainerPath: vol.ContainerPath,
|
|
ReadOnly: vol.ReadOnly,
|
|
})
|
|
}
|
|
|
|
return mounts
|
|
}
|
|
|
|
func (svc *Service) updateAppRunning(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
imageID string,
|
|
) error {
|
|
app.ImageID = sql.NullString{String: imageID, Valid: true}
|
|
app.Status = models.AppStatusRunning
|
|
|
|
saveErr := app.Save(ctx)
|
|
if saveErr != nil {
|
|
return fmt.Errorf("failed to update app: %w", saveErr)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (svc *Service) checkHealthAfterDelay(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
deployment *models.Deployment,
|
|
) {
|
|
svc.log.Info(
|
|
"waiting 60 seconds to check container health",
|
|
"app", app.Name,
|
|
)
|
|
time.Sleep(healthCheckDelaySeconds * time.Second)
|
|
|
|
// Reload app to get current state
|
|
reloadedApp, err := models.FindApp(ctx, svc.db, app.ID)
|
|
if err != nil || reloadedApp == nil {
|
|
svc.log.Error("failed to reload app for health check", "error", err)
|
|
|
|
return
|
|
}
|
|
|
|
containerInfo, containerErr := svc.docker.FindContainerByAppID(ctx, app.ID)
|
|
if containerErr != nil || containerInfo == nil {
|
|
return
|
|
}
|
|
|
|
healthy, err := svc.docker.IsContainerHealthy(ctx, containerInfo.ID)
|
|
if err != nil {
|
|
svc.log.Error("failed to check container health", "error", err)
|
|
svc.notify.NotifyDeployFailed(ctx, reloadedApp, deployment, err)
|
|
_ = deployment.MarkFinished(ctx, models.DeploymentStatusFailed)
|
|
|
|
return
|
|
}
|
|
|
|
if healthy {
|
|
svc.log.Info("container healthy after 60 seconds", "app", reloadedApp.Name)
|
|
svc.notify.NotifyDeploySuccess(ctx, reloadedApp, deployment)
|
|
_ = deployment.MarkFinished(ctx, models.DeploymentStatusSuccess)
|
|
} else {
|
|
svc.log.Warn(
|
|
"container unhealthy after 60 seconds",
|
|
"app", reloadedApp.Name,
|
|
)
|
|
svc.notify.NotifyDeployFailed(ctx, reloadedApp, deployment, ErrContainerUnhealthy)
|
|
_ = deployment.MarkFinished(ctx, models.DeploymentStatusFailed)
|
|
reloadedApp.Status = models.AppStatusError
|
|
_ = reloadedApp.Save(ctx)
|
|
}
|
|
}
|
|
|
|
func (svc *Service) failDeployment(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
deployment *models.Deployment,
|
|
deployErr error,
|
|
) {
|
|
svc.log.Error("deployment failed", "app", app.Name, "error", deployErr)
|
|
_ = deployment.AppendLog(ctx, "ERROR: "+deployErr.Error())
|
|
_ = deployment.MarkFinished(ctx, models.DeploymentStatusFailed)
|
|
app.Status = models.AppStatusError
|
|
_ = app.Save(ctx)
|
|
}
|