- Replace UUID with ULID for app ID generation (lexicographically sortable) - Remove container_id column from apps table (migration 002) - Add upaas.id Docker label to identify containers by app ID - Implement FindContainerByAppID in Docker client to query by label - Update handlers and deploy service to use label-based container lookup - Show system-managed upaas.id label in UI with editing disabled Container association is now determined dynamically via Docker label rather than stored in the database, making the system more resilient to container recreation or external changes.
450 lines
11 KiB
Go
450 lines
11 KiB
Go
// Package deploy provides deployment services.
|
|
package deploy
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"go.uber.org/fx"
|
|
|
|
"git.eeqj.de/sneak/upaas/internal/config"
|
|
"git.eeqj.de/sneak/upaas/internal/database"
|
|
"git.eeqj.de/sneak/upaas/internal/docker"
|
|
"git.eeqj.de/sneak/upaas/internal/logger"
|
|
"git.eeqj.de/sneak/upaas/internal/models"
|
|
"git.eeqj.de/sneak/upaas/internal/service/notify"
|
|
)
|
|
|
|
// Time constants.
|
|
const (
|
|
healthCheckDelaySeconds = 60
|
|
// upaasLabelCount is the number of upaas-specific labels added to containers.
|
|
upaasLabelCount = 1
|
|
)
|
|
|
|
// Sentinel errors for deployment failures.
|
|
var (
|
|
// ErrContainerUnhealthy indicates the container failed health check.
|
|
ErrContainerUnhealthy = errors.New("container unhealthy after 60 seconds")
|
|
)
|
|
|
|
// ServiceParams contains dependencies for Service.
|
|
type ServiceParams struct {
|
|
fx.In
|
|
|
|
Logger *logger.Logger
|
|
Config *config.Config
|
|
Database *database.Database
|
|
Docker *docker.Client
|
|
Notify *notify.Service
|
|
}
|
|
|
|
// Service provides deployment functionality.
|
|
type Service struct {
|
|
log *slog.Logger
|
|
db *database.Database
|
|
docker *docker.Client
|
|
notify *notify.Service
|
|
config *config.Config
|
|
params *ServiceParams
|
|
}
|
|
|
|
// New creates a new deploy Service.
|
|
func New(_ fx.Lifecycle, params ServiceParams) (*Service, error) {
|
|
return &Service{
|
|
log: params.Logger.Get(),
|
|
db: params.Database,
|
|
docker: params.Docker,
|
|
notify: params.Notify,
|
|
config: params.Config,
|
|
params: ¶ms,
|
|
}, nil
|
|
}
|
|
|
|
// GetBuildDir returns the build directory path for an app.
|
|
func (svc *Service) GetBuildDir(appID string) string {
|
|
return filepath.Join(svc.config.DataDir, "builds", appID)
|
|
}
|
|
|
|
// Deploy deploys an app.
|
|
func (svc *Service) Deploy(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
webhookEventID *int64,
|
|
) error {
|
|
deployment, err := svc.createDeploymentRecord(ctx, app, webhookEventID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = svc.updateAppStatusBuilding(ctx, app)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
svc.notify.NotifyBuildStart(ctx, app, deployment)
|
|
|
|
imageID, err := svc.buildImage(ctx, app, deployment)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
svc.notify.NotifyBuildSuccess(ctx, app, deployment)
|
|
|
|
err = svc.updateDeploymentDeploying(ctx, deployment)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
svc.removeOldContainer(ctx, app, deployment)
|
|
|
|
_, err = svc.createAndStartContainer(ctx, app, deployment, imageID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = svc.updateAppRunning(ctx, app, imageID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Use context.WithoutCancel to ensure health check completes even if
|
|
// the parent context is cancelled (e.g., HTTP request ends).
|
|
go svc.checkHealthAfterDelay(context.WithoutCancel(ctx), app, deployment)
|
|
|
|
return nil
|
|
}
|
|
|
|
func (svc *Service) createDeploymentRecord(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
webhookEventID *int64,
|
|
) (*models.Deployment, error) {
|
|
deployment := models.NewDeployment(svc.db)
|
|
deployment.AppID = app.ID
|
|
|
|
if webhookEventID != nil {
|
|
deployment.WebhookEventID = sql.NullInt64{
|
|
Int64: *webhookEventID,
|
|
Valid: true,
|
|
}
|
|
}
|
|
|
|
deployment.Status = models.DeploymentStatusBuilding
|
|
|
|
saveErr := deployment.Save(ctx)
|
|
if saveErr != nil {
|
|
return nil, fmt.Errorf("failed to create deployment: %w", saveErr)
|
|
}
|
|
|
|
return deployment, nil
|
|
}
|
|
|
|
func (svc *Service) updateAppStatusBuilding(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
) error {
|
|
app.Status = models.AppStatusBuilding
|
|
|
|
saveErr := app.Save(ctx)
|
|
if saveErr != nil {
|
|
return fmt.Errorf("failed to update app status: %w", saveErr)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (svc *Service) buildImage(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
deployment *models.Deployment,
|
|
) (string, error) {
|
|
tempDir, cleanup, err := svc.cloneRepository(ctx, app, deployment)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
defer cleanup()
|
|
|
|
imageTag := "upaas/" + app.Name + ":latest"
|
|
|
|
imageID, err := svc.docker.BuildImage(ctx, docker.BuildImageOptions{
|
|
ContextDir: tempDir,
|
|
DockerfilePath: app.DockerfilePath,
|
|
Tags: []string{imageTag},
|
|
})
|
|
if err != nil {
|
|
svc.notify.NotifyBuildFailed(ctx, app, deployment, err)
|
|
svc.failDeployment(
|
|
ctx,
|
|
app,
|
|
deployment,
|
|
fmt.Errorf("failed to build image: %w", err),
|
|
)
|
|
|
|
return "", fmt.Errorf("failed to build image: %w", err)
|
|
}
|
|
|
|
deployment.ImageID = sql.NullString{String: imageID, Valid: true}
|
|
_ = deployment.AppendLog(ctx, "Image built: "+imageID)
|
|
|
|
return imageID, nil
|
|
}
|
|
|
|
func (svc *Service) cloneRepository(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
deployment *models.Deployment,
|
|
) (string, func(), error) {
|
|
tempDir, err := os.MkdirTemp("", "upaas-"+app.ID+"-*")
|
|
if err != nil {
|
|
svc.failDeployment(ctx, app, deployment, fmt.Errorf("failed to create temp dir: %w", err))
|
|
|
|
return "", nil, fmt.Errorf("failed to create temp dir: %w", err)
|
|
}
|
|
|
|
cleanup := func() { _ = os.RemoveAll(tempDir) }
|
|
|
|
cloneErr := svc.docker.CloneRepo(ctx, app.RepoURL, app.Branch, app.SSHPrivateKey, tempDir)
|
|
if cloneErr != nil {
|
|
cleanup()
|
|
svc.failDeployment(ctx, app, deployment, fmt.Errorf("failed to clone repo: %w", cloneErr))
|
|
|
|
return "", nil, fmt.Errorf("failed to clone repo: %w", cloneErr)
|
|
}
|
|
|
|
_ = deployment.AppendLog(ctx, "Repository cloned successfully")
|
|
|
|
return tempDir, cleanup, nil
|
|
}
|
|
|
|
func (svc *Service) updateDeploymentDeploying(
|
|
ctx context.Context,
|
|
deployment *models.Deployment,
|
|
) error {
|
|
deployment.Status = models.DeploymentStatusDeploying
|
|
|
|
saveErr := deployment.Save(ctx)
|
|
if saveErr != nil {
|
|
return fmt.Errorf("failed to update deployment status: %w", saveErr)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (svc *Service) removeOldContainer(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
deployment *models.Deployment,
|
|
) {
|
|
containerInfo, err := svc.docker.FindContainerByAppID(ctx, app.ID)
|
|
if err != nil || containerInfo == nil {
|
|
return
|
|
}
|
|
|
|
svc.log.Info("removing old container", "id", containerInfo.ID)
|
|
|
|
removeErr := svc.docker.RemoveContainer(ctx, containerInfo.ID, true)
|
|
if removeErr != nil {
|
|
svc.log.Warn("failed to remove old container", "error", removeErr)
|
|
}
|
|
|
|
_ = deployment.AppendLog(ctx, "Old container removed")
|
|
}
|
|
|
|
func (svc *Service) createAndStartContainer(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
deployment *models.Deployment,
|
|
imageID string,
|
|
) (string, error) {
|
|
containerOpts, err := svc.buildContainerOptions(ctx, app, imageID)
|
|
if err != nil {
|
|
svc.failDeployment(ctx, app, deployment, err)
|
|
|
|
return "", err
|
|
}
|
|
|
|
containerID, err := svc.docker.CreateContainer(ctx, containerOpts)
|
|
if err != nil {
|
|
svc.notify.NotifyDeployFailed(ctx, app, deployment, err)
|
|
svc.failDeployment(
|
|
ctx,
|
|
app,
|
|
deployment,
|
|
fmt.Errorf("failed to create container: %w", err),
|
|
)
|
|
|
|
return "", fmt.Errorf("failed to create container: %w", err)
|
|
}
|
|
|
|
deployment.ContainerID = sql.NullString{String: containerID, Valid: true}
|
|
_ = deployment.AppendLog(ctx, "Container created: "+containerID)
|
|
|
|
startErr := svc.docker.StartContainer(ctx, containerID)
|
|
if startErr != nil {
|
|
svc.notify.NotifyDeployFailed(ctx, app, deployment, startErr)
|
|
svc.failDeployment(
|
|
ctx,
|
|
app,
|
|
deployment,
|
|
fmt.Errorf("failed to start container: %w", startErr),
|
|
)
|
|
|
|
return "", fmt.Errorf("failed to start container: %w", startErr)
|
|
}
|
|
|
|
_ = deployment.AppendLog(ctx, "Container started")
|
|
|
|
return containerID, nil
|
|
}
|
|
|
|
func (svc *Service) buildContainerOptions(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
_ string,
|
|
) (docker.CreateContainerOptions, error) {
|
|
envVars, err := app.GetEnvVars(ctx)
|
|
if err != nil {
|
|
return docker.CreateContainerOptions{}, fmt.Errorf("failed to get env vars: %w", err)
|
|
}
|
|
|
|
labels, err := app.GetLabels(ctx)
|
|
if err != nil {
|
|
return docker.CreateContainerOptions{}, fmt.Errorf("failed to get labels: %w", err)
|
|
}
|
|
|
|
volumes, err := app.GetVolumes(ctx)
|
|
if err != nil {
|
|
return docker.CreateContainerOptions{}, fmt.Errorf("failed to get volumes: %w", err)
|
|
}
|
|
|
|
envMap := make(map[string]string, len(envVars))
|
|
for _, envVar := range envVars {
|
|
envMap[envVar.Key] = envVar.Value
|
|
}
|
|
|
|
network := ""
|
|
if app.DockerNetwork.Valid {
|
|
network = app.DockerNetwork.String
|
|
}
|
|
|
|
return docker.CreateContainerOptions{
|
|
Name: "upaas-" + app.Name,
|
|
Image: "upaas/" + app.Name + ":latest",
|
|
Env: envMap,
|
|
Labels: buildLabelMap(app, labels),
|
|
Volumes: buildVolumeMounts(volumes),
|
|
Network: network,
|
|
}, nil
|
|
}
|
|
|
|
func buildLabelMap(app *models.App, labels []*models.Label) map[string]string {
|
|
labelMap := make(map[string]string, len(labels)+upaasLabelCount)
|
|
for _, label := range labels {
|
|
labelMap[label.Key] = label.Value
|
|
}
|
|
|
|
// Add the upaas.id label to identify this container
|
|
labelMap[docker.LabelUpaasID] = app.ID
|
|
|
|
return labelMap
|
|
}
|
|
|
|
func buildVolumeMounts(volumes []*models.Volume) []docker.VolumeMount {
|
|
mounts := make([]docker.VolumeMount, 0, len(volumes))
|
|
for _, vol := range volumes {
|
|
mounts = append(mounts, docker.VolumeMount{
|
|
HostPath: vol.HostPath,
|
|
ContainerPath: vol.ContainerPath,
|
|
ReadOnly: vol.ReadOnly,
|
|
})
|
|
}
|
|
|
|
return mounts
|
|
}
|
|
|
|
func (svc *Service) updateAppRunning(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
imageID string,
|
|
) error {
|
|
app.ImageID = sql.NullString{String: imageID, Valid: true}
|
|
app.Status = models.AppStatusRunning
|
|
|
|
saveErr := app.Save(ctx)
|
|
if saveErr != nil {
|
|
return fmt.Errorf("failed to update app: %w", saveErr)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (svc *Service) checkHealthAfterDelay(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
deployment *models.Deployment,
|
|
) {
|
|
svc.log.Info(
|
|
"waiting 60 seconds to check container health",
|
|
"app", app.Name,
|
|
)
|
|
time.Sleep(healthCheckDelaySeconds * time.Second)
|
|
|
|
// Reload app to get current state
|
|
reloadedApp, err := models.FindApp(ctx, svc.db, app.ID)
|
|
if err != nil || reloadedApp == nil {
|
|
svc.log.Error("failed to reload app for health check", "error", err)
|
|
|
|
return
|
|
}
|
|
|
|
containerInfo, containerErr := svc.docker.FindContainerByAppID(ctx, app.ID)
|
|
if containerErr != nil || containerInfo == nil {
|
|
return
|
|
}
|
|
|
|
healthy, err := svc.docker.IsContainerHealthy(ctx, containerInfo.ID)
|
|
if err != nil {
|
|
svc.log.Error("failed to check container health", "error", err)
|
|
svc.notify.NotifyDeployFailed(ctx, reloadedApp, deployment, err)
|
|
_ = deployment.MarkFinished(ctx, models.DeploymentStatusFailed)
|
|
|
|
return
|
|
}
|
|
|
|
if healthy {
|
|
svc.log.Info("container healthy after 60 seconds", "app", reloadedApp.Name)
|
|
svc.notify.NotifyDeploySuccess(ctx, reloadedApp, deployment)
|
|
_ = deployment.MarkFinished(ctx, models.DeploymentStatusSuccess)
|
|
} else {
|
|
svc.log.Warn(
|
|
"container unhealthy after 60 seconds",
|
|
"app", reloadedApp.Name,
|
|
)
|
|
svc.notify.NotifyDeployFailed(ctx, reloadedApp, deployment, ErrContainerUnhealthy)
|
|
_ = deployment.MarkFinished(ctx, models.DeploymentStatusFailed)
|
|
reloadedApp.Status = models.AppStatusError
|
|
_ = reloadedApp.Save(ctx)
|
|
}
|
|
}
|
|
|
|
func (svc *Service) failDeployment(
|
|
ctx context.Context,
|
|
app *models.App,
|
|
deployment *models.Deployment,
|
|
deployErr error,
|
|
) {
|
|
svc.log.Error("deployment failed", "app", app.Name, "error", deployErr)
|
|
_ = deployment.AppendLog(ctx, "ERROR: "+deployErr.Error())
|
|
_ = deployment.MarkFinished(ctx, models.DeploymentStatusFailed)
|
|
app.Status = models.AppStatusError
|
|
_ = app.Save(ctx)
|
|
}
|