From ddcd179841f1ba1135b230cde2929c294fc850fc Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 23 Sep 2026 11:44:30 +0200 Subject: [PATCH 1/4] Keep deployment logs findable after the container is recreated (closes #214) Deployment logs were stored under a directory named after the container's hostname, and the path was worked out again from the current hostname on download. Docker gives a recreated container a new hostname, so every older log download returned 404. New logs now go to `logs//` with no hostname in the path. Logs written by older versions under an old hostname directory are still found by looking one directory deeper, inside the same confined log root. Tests cover both the hostname-free path and downloading an old-layout log. Model: opus-5-5 --- TODO.md | 4 +++ internal/handlers/app.go | 22 +++++++++++- internal/handlers/log_download_test.go | 50 ++++++++++++++++++++++++++ internal/service/deploy/deploy.go | 13 ++++--- 4 files changed, 81 insertions(+), 8 deletions(-) diff --git a/TODO.md b/TODO.md index 5f47fa4..72043db 100644 --- a/TODO.md +++ b/TODO.md @@ -20,6 +20,10 @@ regress. # Completed Steps +- 2026-09-23: Deployment log files are now stored under `logs//` + instead of `logs///`, so downloads keep working after the + upaas container is recreated; logs written under an old hostname directory are + still found (#214). - 2026-09-23: Fixed the flaky `t.TempDir` cleanup race in `internal/handlers` (the one fixed in `internal/service/webhook` by #198): `TestHandleWebhookProcessesValidWebhook` now waits with the webhook service's diff --git a/internal/handlers/app.go b/internal/handlers/app.go index a985c11..22d527c 100644 --- a/internal/handlers/app.go +++ b/internal/handlers/app.go @@ -6,9 +6,11 @@ import ( "encoding/json" "errors" "fmt" + "io/fs" "net/http" "net/url" "os" + "path" "path/filepath" "strconv" "strings" @@ -640,7 +642,7 @@ func (h *Handlers) HandleDeploymentLogDownload() http.HandlerFunc { } defer func() { _ = root.Close() }() - file, openErr := root.Open(relPath) + file, openErr := openDeploymentLog(root, relPath) if openErr != nil { http.NotFound(writer, request) @@ -665,6 +667,24 @@ func (h *Handlers) HandleDeploymentLogDownload() http.HandlerFunc { } } +// openDeploymentLog opens a deployment log file inside the log root. +// Logs written by older versions sit one directory deeper, under the +// hostname of the container that wrote them, so when the file is not at +// relPath it is looked for under any directory directly below the root. +func openDeploymentLog(root *os.Root, relPath string) (*os.File, error) { + file, err := root.Open(relPath) + if err == nil { + return file, nil + } + + matches, globErr := fs.Glob(root.FS(), path.Join("*", filepath.ToSlash(relPath))) + if globErr != nil || len(matches) == 0 { + return nil, err + } + + return root.Open(matches[0]) +} + // containerLogsAPITail is the default number of log lines for the container logs API. const containerLogsAPITail = "100" diff --git a/internal/handlers/log_download_test.go b/internal/handlers/log_download_test.go index cbbfd08..595fbee 100644 --- a/internal/handlers/log_download_test.go +++ b/internal/handlers/log_download_test.go @@ -69,6 +69,56 @@ func TestHandleDeploymentLogDownloadServesLegitimateFile(t *testing.T) { assert.Contains(t, recorder.Body.String(), "deploy log contents") } +// TestGetLogFilePathHasNoHostname verifies a deployment log is stored +// directly under logs//, with no hostname directory in between, +// so it is still found after the container is recreated with a new +// hostname. +func TestGetLogFilePathHasNoHostname(t *testing.T) { + t.Parallel() + + testCtx := setupTestHandlers(t) + createdApp := createTestApp(t, testCtx, "log-path-app") + + deployment := models.NewDeployment(testCtx.database) + deployment.AppID = createdApp.ID + + logPath := testCtx.deploySvc.GetLogFilePath(createdApp, deployment) + + assert.Equal(t, + filepath.Join(testCtx.deploySvc.GetLogDir(), createdApp.Name), + filepath.Dir(logPath), + ) +} + +// TestHandleDeploymentLogDownloadServesLogFromOldHostnameDir verifies a +// log written by an older version, under the hostname of a container that +// has since been recreated, can still be downloaded. +func TestHandleDeploymentLogDownloadServesLogFromOldHostnameDir(t *testing.T) { + t.Parallel() + + testCtx := setupTestHandlers(t) + createdApp := createTestApp(t, testCtx, "log-old-hostname-app") + + deployment := models.NewDeployment(testCtx.database) + deployment.AppID = createdApp.ID + deployment.Status = models.DeploymentStatusSuccess + require.NoError(t, deployment.Save(context.Background())) + + logDir := testCtx.deploySvc.GetLogDir() + newPath := testCtx.deploySvc.GetLogFilePath(createdApp, deployment) + relPath, relErr := filepath.Rel(logDir, newPath) + require.NoError(t, relErr) + + oldPath := filepath.Join(logDir, "old-container-hostname", relPath) + require.NoError(t, os.MkdirAll(filepath.Dir(oldPath), 0o750)) + require.NoError(t, os.WriteFile(oldPath, []byte("old deploy log contents"), 0o600)) + + recorder := doLogDownload(t, testCtx, createdApp.ID, deployment.ID) + + assert.Equal(t, http.StatusOK, recorder.Code) + assert.Contains(t, recorder.Body.String(), "old deploy log contents") +} + // TestHandleDeploymentLogDownloadRejectsPathTraversal verifies the // os.Root containment guard. A traversal-shaped app name drives the // resolved log path out of the deploy log directory onto a sentinel diff --git a/internal/service/deploy/deploy.go b/internal/service/deploy/deploy.go index ab1f2c8..79343e2 100644 --- a/internal/service/deploy/deploy.go +++ b/internal/service/deploy/deploy.go @@ -261,15 +261,14 @@ func (svc *Service) GetBuildDir(appName string) string { // GetLogFilePath returns the path to the log file for a deployment. // Returns empty string if the path cannot be determined. +// +// The path must not depend on the container's hostname: Docker assigns a +// new one whenever the container is recreated, and older logs would then +// no longer be found. func (svc *Service) GetLogFilePath( app *models.App, deployment *models.Deployment, ) string { - hostname, err := os.Hostname() - if err != nil { - hostname = "unknown" - } - // Get commit SHA sha := "" if deployment.CommitSHA.Valid && deployment.CommitSHA.String != "" { @@ -291,7 +290,7 @@ func (svc *Service) GetLogFilePath( filename = fmt.Sprintf("%s_%s.log.txt", app.Name, timestamp) } - return filepath.Join(svc.config.DataDir, "logs", hostname, app.Name, filename) + return filepath.Join(svc.config.DataDir, "logs", app.Name, filename) } // GetLogDir returns the root directory under which all deployment log @@ -1294,7 +1293,7 @@ func (svc *Service) failDeployment( } // writeLogsToFile writes the deployment logs to a file on disk. -// Structure: DataDir/logs///__.log.txt +// Structure: DataDir/logs//__.log.txt func (svc *Service) writeLogsToFile(app *models.App, deployment *models.Deployment) { if !deployment.Logs.Valid || deployment.Logs.String == "" { return -- 2.54.0 From 56345bc6f602082a55b113f752a4c44d178f3056 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 23 Sep 2026 12:04:32 +0200 Subject: [PATCH 2/4] Remove the git clone container's anonymous volume with it (closes #215) The pinned `alpine/git` image declares a volume at `/git`, so every clone container got an anonymous volume, and the container was removed without its volumes, leaving one volume behind per deploy. The clone container is now removed together with its volumes, whether the clone succeeds, fails or is cancelled; the removal uses a context that outlives cancellation. Tests cover success, failure and cancellation against a fake Docker API, and a manual check against real Docker is recorded on the PR. The old app container's own anonymous volumes are still kept on redeploy, since deleting them could discard app data. Model: opus-5-5 --- TODO.md | 3 + internal/docker/client.go | 7 ++- internal/docker/validation_test.go | 90 ++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 2 deletions(-) diff --git a/TODO.md b/TODO.md index 72043db..27bd714 100644 --- a/TODO.md +++ b/TODO.md @@ -20,6 +20,9 @@ regress. # Completed Steps +- 2026-09-23: The git clone container is now removed together with its anonymous + volume (the `alpine/git` image declares one at `/git`), so a deploy no longer + leaves a Docker volume behind (#215). - 2026-09-23: Deployment log files are now stored under `logs//` instead of `logs///`, so downloads keep working after the upaas container is recreated; logs written under an old hostname directory are diff --git a/internal/docker/client.go b/internal/docker/client.go index 168f3fa..9e6abc5 100644 --- a/internal/docker/client.go +++ b/internal/docker/client.go @@ -656,11 +656,14 @@ func (c *Client) performClone( return nil, err } + // The git image declares a volume, so Docker gives each clone container + // an anonymous volume; remove it with the container. The removal must + // still run when the deploy is cancelled. defer func() { _ = c.docker.ContainerRemove( - ctx, + context.WithoutCancel(ctx), gitContainerID.String(), - container.RemoveOptions{Force: true}, + container.RemoveOptions{Force: true, RemoveVolumes: true}, ) }() diff --git a/internal/docker/validation_test.go b/internal/docker/validation_test.go index 2d033c8..3ea3378 100644 --- a/internal/docker/validation_test.go +++ b/internal/docker/validation_test.go @@ -1,9 +1,18 @@ package docker //nolint:testpackage // tests unexported regexps and Client struct import ( + "context" "errors" + "fmt" "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "strings" "testing" + + "github.com/docker/docker/client" ) // mainBranch is the branch name used across validation tests. @@ -149,3 +158,84 @@ func TestCloneRepoRejectsInjection(t *testing.T) { }) } } + +// TestPerformCloneRemovesContainerVolumes runs a clone against a fake Docker +// API and checks that the clone container is removed together with its +// anonymous volumes, whether the clone succeeds, fails, or is cancelled. +func TestPerformCloneRemovesContainerVolumes(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + exitCode int + cancel bool + }{ + {name: "succeeds", exitCode: 0}, + {name: "fails", exitCode: 1}, + {name: "cancelled", cancel: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + + removeQuery := make(chan url.Values, 1) + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.Method == http.MethodDelete: + removeQuery <- r.URL.Query() + case strings.HasSuffix(r.URL.Path, "/containers/create"): + _, _ = w.Write([]byte(`{"Id":"gitcontainer"}`)) + case strings.HasSuffix(r.URL.Path, "/wait") && tt.cancel: + // Cancel the deploy while the clone is running. + cancel() + <-r.Context().Done() + case strings.HasSuffix(r.URL.Path, "/wait"): + _, _ = fmt.Fprintf(w, `{"StatusCode":%d}`, tt.exitCode) + default: + _, _ = w.Write([]byte(`{}`)) + } + }, + )) + t.Cleanup(srv.Close) + + dockerAPI, err := client.NewClientWithOpts( + client.WithHost("tcp://" + srv.Listener.Addr().String()), + ) + if err != nil { + t.Fatal(err) + } + + c := &Client{docker: dockerAPI, log: slog.Default()} + + dir := t.TempDir() + cfg := &cloneConfig{ + repoURL: "git@example.com:repo.git", + branch: mainBranch, + sshPrivateKey: "fake-key", + containerDir: filepath.Join(dir, "repo"), + hostDir: filepath.Join(dir, "repo"), + keyFile: filepath.Join(dir, "deploy_key"), + hostKeyFile: filepath.Join(dir, "deploy_key"), + } + + _, _ = c.performClone(ctx, cfg) + + select { + case query := <-removeQuery: + if query.Get("v") != "1" { + t.Errorf("clone container removed without its volumes: %v", query) + } + default: + t.Error("clone container was not removed") + } + }) + } +} -- 2.54.0 From a60ea144fb0be0df2c67730d7e6a17e329dfc63f Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 23 Sep 2026 12:24:36 +0200 Subject: [PATCH 3/4] Remove images from earlier deploys after a successful deploy (closes #216) Every deploy built and tagged a new image and nothing removed the old ones, so disk use grew with each push. After a successful deploy, upaas now removes the app's old `upaas-:` tags by name, without force, keeping the current image and the previous one that rollback starts. Docker deletes an image only when no other tag or container still uses it, so an image shared with another app stays. A failed removal is a warning in the deployment log, not a failed deploy. The post-deploy step is one function, tested against a fake Docker API. Judgement call: untagged images from other stages of a multi-stage build stay; they are the build cache. On the first deploy after upgrading, all older images of that app are removed at once. Model: opus-5-5 --- TODO.md | 3 + internal/docker/client.go | 49 ++++++++ internal/service/deploy/deploy.go | 71 +++++++++++- internal/service/deploy/deploy_images_test.go | 106 ++++++++++++++++++ internal/service/deploy/export_test.go | 10 ++ 5 files changed, 233 insertions(+), 6 deletions(-) create mode 100644 internal/service/deploy/deploy_images_test.go diff --git a/TODO.md b/TODO.md index 27bd714..9a7114a 100644 --- a/TODO.md +++ b/TODO.md @@ -20,6 +20,9 @@ regress. # Completed Steps +- 2026-09-23: After a successful deploy, upaas removes the app's images other + than the running one and the one rollback would use, together with the + untagged images they were built on (#216). - 2026-09-23: The git clone container is now removed together with its anonymous volume (the `alpine/git` image declares one at `/git`), so a deploy no longer leaves a Docker volume behind (#215). diff --git a/internal/docker/client.go b/internal/docker/client.go index 9e6abc5..f348d33 100644 --- a/internal/docker/client.go +++ b/internal/docker/client.go @@ -537,6 +537,55 @@ func (c *Client) RemoveImage(ctx context.Context, imageID ImageID) error { return nil } +// ListImageTags returns the tags in the given repository, such as +// "upaas-myapp:12" in "upaas-myapp", each with the ID of its image. +// Tags the same image has in other repositories are left out. +func (c *Client) ListImageTags( + ctx context.Context, + repository string, +) (map[string]ImageID, error) { + if c.docker == nil { + return nil, ErrNotConnected + } + + images, err := c.docker.ImageList(ctx, image.ListOptions{ + Filters: filters.NewArgs(filters.Arg("reference", repository)), + }) + if err != nil { + return nil, fmt.Errorf("failed to list images: %w", err) + } + + tags := make(map[string]ImageID) + + for _, img := range images { + for _, tag := range img.RepoTags { + if strings.HasPrefix(tag, repository+":") { + tags[tag] = ImageID(img.ID) + } + } + } + + return tags, nil +} + +// RemoveImageTag removes a tag such as "upaas-myapp:12", without force. +// Docker then deletes the image, and the untagged images it was built on, +// only if no other tag and no container still uses it. +func (c *Client) RemoveImageTag(ctx context.Context, tag string) error { + if c.docker == nil { + return ErrNotConnected + } + + _, err := c.docker.ImageRemove(ctx, tag, image.RemoveOptions{ + PruneChildren: true, + }) + if err != nil && !client.IsErrNotFound(err) { + return fmt.Errorf("failed to remove image tag %s: %w", tag, err) + } + + return nil +} + func (c *Client) performBuild( ctx context.Context, opts BuildImageOptions, diff --git a/internal/service/deploy/deploy.go b/internal/service/deploy/deploy.go index 79343e2..e2e1d30 100644 --- a/internal/service/deploy/deploy.go +++ b/internal/service/deploy/deploy.go @@ -9,8 +9,10 @@ import ( "errors" "fmt" "log/slog" + "maps" "os" "path/filepath" + "slices" "strings" "sync" "time" @@ -524,12 +526,7 @@ func (svc *Service) runBuildAndDeploy( return err } - // Save current image as previous before updating to new one - if app.ImageID.Valid && app.ImageID.String != "" { - app.PreviousImageID = app.ImageID - } - - err = svc.updateAppRunning(bgCtx, app, imageID) + err = svc.recordDeployedImage(bgCtx, app, deployment, imageID) if err != nil { return err } @@ -714,6 +711,68 @@ func (svc *Service) checkCancelled( return ErrDeployCancelled } +// recordDeployedImage runs once the new container has started: it makes +// imageID the app's current image, keeps the replaced one as the previous +// image for Rollback, and then removes the app's other images. +func (svc *Service) recordDeployedImage( + ctx context.Context, + app *models.App, + deployment *models.Deployment, + imageID docker.ImageID, +) error { + // Save current image as previous before updating to new one + if app.ImageID.Valid && app.ImageID.String != "" { + app.PreviousImageID = app.ImageID + } + + err := svc.updateAppRunning(ctx, app, imageID) + if err != nil { + return err + } + + svc.removeUnusedImages(ctx, app, deployment) + + return nil +} + +// removeUnusedImages removes the app's tags (upaas-:, set by +// buildImage) except those of the image the running container uses and the +// one Rollback would start. Docker deletes an image only once no other tag, +// such as another app's, and no container still uses it. +func (svc *Service) removeUnusedImages( + ctx context.Context, + app *models.App, + deployment *models.Deployment, +) { + tags, err := svc.docker.ListImageTags(ctx, "upaas-"+app.Name) + if err != nil { + svc.log.Error("failed to list app images", "error", err, "app", app.Name) + + return + } + + for _, tag := range slices.Sorted(maps.Keys(tags)) { + imageID := tags[tag].String() + if imageID == app.ImageID.String || imageID == app.PreviousImageID.String { + continue + } + + removeErr := svc.docker.RemoveImageTag(ctx, tag) + if removeErr != nil { + svc.log.Error("failed to remove old image", + "error", removeErr, "app", app.Name, "tag", tag) + _ = deployment.AppendLog( + ctx, + "WARNING: failed to remove old image "+tag+": "+removeErr.Error(), + ) + + continue + } + + _ = deployment.AppendLog(ctx, "Removed old image: "+tag) + } +} + // cleanupCancelledDeploy removes orphan resources left by a cancelled deployment. func (svc *Service) cleanupCancelledDeploy( ctx context.Context, diff --git a/internal/service/deploy/deploy_images_test.go b/internal/service/deploy/deploy_images_test.go new file mode 100644 index 0000000..086ccb3 --- /dev/null +++ b/internal/service/deploy/deploy_images_test.go @@ -0,0 +1,106 @@ +package deploy_test + +import ( + "context" + "database/sql" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/fx/fxtest" + + "sneak.berlin/go/upaas/internal/config" + "sneak.berlin/go/upaas/internal/database" + "sneak.berlin/go/upaas/internal/docker" + "sneak.berlin/go/upaas/internal/logger" + "sneak.berlin/go/upaas/internal/models" + "sneak.berlin/go/upaas/internal/service/deploy" +) + +// TestRecordDeployedImageRemovesOldImages runs the step after a deploy +// against a fake Docker API. Image one is also tagged for another app, +// image two was the previous image, three the current one, four is new. +func TestRecordDeployedImageRemovesOldImages(t *testing.T) { + t.Parallel() + + var ( + mu sync.Mutex + removed []string + forced bool + ) + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + switch { + case r.Method == http.MethodDelete: + _, name, _ := strings.Cut(r.URL.Path, "/images/") + + mu.Lock() + + removed = append(removed, name) + forced = forced || r.URL.Query().Get("force") != "" + mu.Unlock() + + _, _ = w.Write([]byte(`[]`)) + case strings.HasSuffix(r.URL.Path, "/images/json"): + _, _ = w.Write([]byte(`[ + {"Id":"sha256:one","RepoTags":["upaas-myapp:1","upaas-otherapp:7"]}, + {"Id":"sha256:two","RepoTags":["upaas-myapp:2"]}, + {"Id":"sha256:three","RepoTags":["upaas-myapp:3"]}, + {"Id":"sha256:four","RepoTags":["upaas-myapp:4"]} + ]`)) + default: + _, _ = w.Write([]byte(`{}`)) + } + }, + )) + t.Cleanup(srv.Close) + + log := slog.New(slog.NewTextHandler(os.Stderr, nil)) + lifecycle := fxtest.NewLifecycle(t) + + dockerClient, err := docker.New(lifecycle, docker.Params{ + Logger: logger.NewForTest(log), + Config: &config.Config{DockerHost: "tcp://" + srv.Listener.Addr().String()}, + }) + require.NoError(t, err) + + lifecycle.RequireStart() + t.Cleanup(lifecycle.RequireStop) + + db := database.NewTestDatabase(t) + ctx := context.Background() + + app := models.NewApp(db) + app.ID = "myapp-id" + app.Name = "myapp" + app.ImageID = sql.NullString{String: "sha256:three", Valid: true} + app.PreviousImageID = sql.NullString{String: "sha256:two", Valid: true} + require.NoError(t, app.Save(ctx)) + + deployment := models.NewDeployment(db) + deployment.AppID = app.ID + require.NoError(t, deployment.Save(ctx)) + + svc := deploy.NewTestServiceWithConfig(log, &config.Config{}, dockerClient) + + err = svc.RecordDeployedImage(ctx, app, deployment, "sha256:four") + require.NoError(t, err) + + assert.Equal(t, "sha256:four", app.ImageID.String) + assert.Equal(t, "sha256:three", app.PreviousImageID.String) + + mu.Lock() + defer mu.Unlock() + + assert.Equal(t, []string{"upaas-myapp:1", "upaas-myapp:2"}, removed) + assert.False(t, forced, "old image tags must be removed without force") +} diff --git a/internal/service/deploy/export_test.go b/internal/service/deploy/export_test.go index 1d86a25..5b68733 100644 --- a/internal/service/deploy/export_test.go +++ b/internal/service/deploy/export_test.go @@ -90,6 +90,16 @@ func (svc *Service) GetBuildDirExported(appName string) string { return svc.GetBuildDir(appName) } +// RecordDeployedImage exposes recordDeployedImage for testing. +func (svc *Service) RecordDeployedImage( + ctx context.Context, + app *models.App, + deployment *models.Deployment, + imageID docker.ImageID, +) error { + return svc.recordDeployedImage(ctx, app, deployment, imageID) +} + // BuildContainerOptionsExported exposes buildContainerOptions for testing. func (svc *Service) BuildContainerOptionsExported( ctx context.Context, -- 2.54.0 From a57efeed12e2148ecc55803ccf4d2eae94b2b794 Mon Sep 17 00:00:00 2001 From: clawbot <35+clawbot@noreply.example.org> Date: Wed, 23 Sep 2026 12:45:42 +0200 Subject: [PATCH 4/4] Build apps with BuildKit so build stages stop piling up (closes #220) Multi-stage app builds left an untagged image for every build stage after each deploy, and nothing could safely remove them. upaas now asks Docker for a BuildKit build, which keeps stages in Docker's build cache instead of as images; Docker limits and cleans that cache itself. The final image is still tagged `upaas-:`, so old-image cleanup is unchanged. BuildKit's progress messages are decoded and written to the deployment log as plain text. Deviation: adds `github.com/moby/buildkit` v0.16.0 (matching the pinned Docker client), added with `go get` since no make target adds dependencies. Judgement call: the cache limit is on by default only from Docker Engine 28.2; older engines need the `daemon.json` setting the README now names. Model: opus-5-5 --- README.md | 5 ++ TODO.md | 5 ++ go.mod | 38 ++++++++++ go.sum | 111 +++++++++++++++++++++++++++++ internal/docker/client.go | 76 ++++++++++++++++---- internal/docker/validation_test.go | 75 +++++++++++++++++++ 6 files changed, 295 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index a7488e3..8f7f109 100644 --- a/README.md +++ b/README.md @@ -253,6 +253,11 @@ container. Example: `HOST_DATA_DIR=/srv/upaas/data docker compose up -d` +Apps are built with BuildKit, so the stages of a multi-stage build are kept in +Docker's build cache rather than as untagged images. Docker Engine 28.2 and +later keeps that cache under a size limit by default; on older engines, set +`"builder": {"gc": {"enabled": true}}` in the host's `daemon.json`. + Session secrets are automatically generated on first startup and persisted to `$UPAAS_DATA_DIR/session.key`. diff --git a/TODO.md b/TODO.md index 9a7114a..6b6e056 100644 --- a/TODO.md +++ b/TODO.md @@ -20,6 +20,11 @@ regress. # Completed Steps +- 2026-09-23: Apps are now built with BuildKit, so the stages of a multi-stage + build stay in Docker's size-limited build cache instead of piling up as + untagged images; build progress is still written to the deployment log as + plain text (#220). + - 2026-09-23: After a successful deploy, upaas removes the app's images other than the running one and the one rollback would use, together with the untagged images they were built on (#216). diff --git a/go.mod b/go.mod index 302a86a..c85cc4d 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/gorilla/sessions v1.4.0 github.com/joho/godotenv v1.5.1 github.com/mattn/go-sqlite3 v1.14.32 + github.com/moby/buildkit v0.16.0 github.com/oklog/ulid/v2 v2.1.1 github.com/prometheus/client_golang v1.23.2 github.com/spf13/viper v1.21.0 @@ -24,10 +25,19 @@ require ( require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/containerd/console v1.0.4 // indirect + github.com/containerd/containerd v1.7.21 // indirect + github.com/containerd/containerd/api v1.7.19 // indirect + github.com/containerd/continuity v0.4.3 // indirect + github.com/containerd/errdefs v0.1.0 // indirect github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/containerd/ttrpc v1.2.5 // indirect + github.com/containerd/typeurl/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect @@ -36,12 +46,23 @@ require ( github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/gofrs/flock v0.12.1 // indirect + github.com/gogo/googleapis v1.4.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/gorilla/securecookie v1.1.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/in-toto/in-toto-golang v0.5.0 // indirect github.com/klauspost/compress v1.18.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/locker v1.0.1 // indirect github.com/moby/patternmatcher v0.6.0 // indirect github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/signal v0.7.1 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect @@ -56,25 +77,42 @@ require ( github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/secure-systems-lab/go-securesystemslib v0.4.0 // indirect + github.com/shibumi/go-pathspec v1.3.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c // indirect + github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 // indirect + github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea // indirect + github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 // indirect go.opentelemetry.io/otel v1.39.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 // indirect go.opentelemetry.io/otel/metric v1.39.0 // indirect + go.opentelemetry.io/otel/sdk v1.39.0 // indirect go.opentelemetry.io/otel/trace v1.39.0 // indirect + go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/dig v1.19.0 // indirect go.uber.org/multierr v1.10.0 // indirect go.uber.org/zap v1.26.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.39.0 // indirect golang.org/x/text v0.32.0 // indirect + google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/grpc v1.77.0 // indirect google.golang.org/protobuf v1.36.10 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.5.2 // indirect diff --git a/go.sum b/go.sum index 12df127..207b195 100644 --- a/go.sum +++ b/go.sum @@ -1,19 +1,60 @@ +cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= +cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8 h1:nMpu1t4amK3vJWBibQ5X/Nv0aXL+b69TQf2uK5PH7Go= github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8/go.mod h1:3cARGAK9CfW3HoxCy1a0G4TKrdiKke8ftOMEOHyySYs= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 h1:59MxjQVfjXsBpLy+dbd2/ELV5ofnUkUZBvWSC85sheA= +github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0/go.mod h1:OahwfttHWG6eJ0clwcfBAHoDI6X/LV/15hx/wlMZSrU= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Microsoft/hcsshim v0.11.7 h1:vl/nj3Bar/CvJSYo7gIQPyRWc9f3c6IeSNavBTSZNZQ= +github.com/Microsoft/hcsshim v0.11.7/go.mod h1:MV8xMfmECjl5HdO7U/3/hFVnkmSBjAjmA09d4bExKcU= +github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 h1:aM1rlcoLz8y5B2r4tTLMiVTrMtpfY0O8EScKJxaSaEc= +github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f h1:Y8xYupdHxryycyPlc9Y+bSQAYZnetRJ70VMVKm5CKI0= +github.com/cncf/xds/go v0.0.0-20251022180443-0feb69152e9f/go.mod h1:HlzOvOjVBOfTGSRXRyY0OiCS/3J1akRGQQpRO/7zyF4= +github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE= +github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4= +github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= +github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= +github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro= +github.com/containerd/console v1.0.4/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= +github.com/containerd/containerd v1.7.21 h1:USGXRK1eOC/SX0L195YgxTHb0a00anxajOzgfN0qrCA= +github.com/containerd/containerd v1.7.21/go.mod h1:e3Jz1rYRUZ2Lt51YrH9Rz0zPyJBOlSvB3ghr2jbVD8g= +github.com/containerd/containerd/api v1.7.19 h1:VWbJL+8Ap4Ju2mx9c9qS1uFSB1OVYr5JJrW2yT5vFoA= +github.com/containerd/containerd/api v1.7.19/go.mod h1:fwGavl3LNwAV5ilJ0sbrABL44AQxmNjDRcwheXDb6Ig= +github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8= +github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/containerd/errdefs v0.1.0 h1:m0wCRBiu1WJT/Fr+iOoQHMQS/eP5myQ8lCv4Dz5ZURM= +github.com/containerd/errdefs v0.1.0/go.mod h1:YgWiiHtLmSeBrvpw+UfPijzbLaB77mEG1WwJTDETIV0= +github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= +github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/nydus-snapshotter v0.14.0 h1:6/eAi6d7MjaeLLuMO8Udfe5GVsDudmrDNO4SGETMBco= +github.com/containerd/nydus-snapshotter v0.14.0/go.mod h1:TT4jv2SnIDxEBu4H2YOvWQHPOap031ydTaHTuvc5VQk= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/containerd/stargz-snapshotter v0.15.1 h1:fpsP4kf/Z4n2EYnU0WT8ZCE3eiKDwikDhL6VwxIlgeA= +github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G/ZW/0kEe2oEKCdS/ZxIyoCU= +github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9Nv33Lt6UC7xEx58C+LHRdoqbEKjz1Kk= +github.com/containerd/ttrpc v1.2.5 h1:IFckT1EFQoFBMG4c3sMdT8EP3/aKfumK1msY+Ze4oLU= +github.com/containerd/ttrpc v1.2.5/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= +github.com/containerd/typeurl/v2 v2.2.0 h1:6NBDbQzr7I5LHgp34xAXYF5DOTQDn05X58lsPEmzLso= +github.com/containerd/typeurl/v2 v2.2.0/go.mod h1:8XOOxnyatxSWuG8OfsZXVnAF4iZfedjS/8UHSPJnX4g= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -23,8 +64,12 @@ github.com/docker/docker v27.3.1+incompatible h1:KttF0XoteNTicmUtBO0L2tP+J7FGRFT github.com/docker/docker v27.3.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -42,12 +87,22 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= +github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= +github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0= +github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/csrf v1.7.3 h1:BHWt6FTLZAb2HtWT5KDBf6qgpZzvtbp9QWDRKZMXJC0= @@ -58,6 +113,13 @@ github.com/gorilla/sessions v1.4.0 h1:kpIYOp/oi6MG/p5PgxApU8srsSw9tuFbt46Lt7auzq github.com/gorilla/sessions v1.4.0/go.mod h1:FLWm50oby91+hl7p/wRxDth9bWSuk0qVL2emc7lT5ik= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/in-toto/in-toto-golang v0.5.0 h1:hb8bgwr0M2hGdDsLjkJ3ZqJ8JFLL/tgYdAxF/XEFBbY= +github.com/in-toto/in-toto-golang v0.5.0/go.mod h1:/Rq0IZHLV7Ku5gielPT4wPHJfH1GdHMCq8+WPxw8/BE= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -72,12 +134,20 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/moby/buildkit v0.16.0 h1:wOVBj1o5YNVad/txPQNXUXdelm7Hs/i0PUFjzbK0VKE= +github.com/moby/buildkit v0.16.0/go.mod h1:Xqx/5GlrqE1yIRORk0NSCVDFpQAU1WjlT6KHYZdisIQ= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= +github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/signal v0.7.1 h1:PrQxdvxcGijdo6UXXo/lU/TvHUWyPhj7UOpSo8tuvk0= +github.com/moby/sys/signal v0.7.1/go.mod h1:Se1VGehYokAkrSQwL4tDzHvETwUZlnY7S5XtQ50mQp8= github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= @@ -94,7 +164,13 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= +github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= +github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -114,10 +190,16 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= +github.com/secure-systems-lab/go-securesystemslib v0.4.0 h1:b23VGrQhTA8cN2CbBw7/FulN9fTtqYUdS5+Oxzt+DUE= +github.com/secure-systems-lab/go-securesystemslib v0.4.0/go.mod h1:FGBZgq2tXWICsxWQW1msNf49F0Pf2Op5Htayx335Qbs= +github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI= +github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= +github.com/spdx/tools-golang v0.5.3 h1:ialnHeEYUC4+hkm5vJm4qz2x+oEJbS0mAMFrNXdQraY= +github.com/spdx/tools-golang v0.5.3/go.mod h1:/ETOahiAo96Ob0/RAIBmFZw6XN0yTnyr/uFZm2NTMhI= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= @@ -127,15 +209,32 @@ github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3A github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c h1:+6wg/4ORAbnSoGDzg2Q1i3CeMcT/jjhye/ZfnBHy7/M= +github.com/tonistiigi/fsutil v0.0.0-20240424095704-91a3fc46842c/go.mod h1:vbbYqJlnswsbJqWUcJN8fKtBhnEgldDrcagTgnBVKKM= +github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4 h1:7I5c2Ig/5FgqkYOh/N87NzoyI9U15qUPXhDD8uCupv8= +github.com/tonistiigi/go-csvvalue v0.0.0-20240710180619-ddb21b71c0b4/go.mod h1:278M4p8WsNh3n4a1eqiFcV2FGk7wE5fwUpUom9mK9lE= +github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea h1:SXhTLE6pb6eld/v/cCndK0AMpt1wiVFb/YYmqB3/QG0= +github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea/go.mod h1:WPnis/6cRcDZSUvVmezrxJPkiO87ThFYsoUiMwWNDJk= +github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab h1:H6aJ0yKQ0gF49Qb2z5hI1UHxSQt4JMyxebFR15KnApw= +github.com/tonistiigi/vt100 v0.0.0-20240514184818-90bafcd6abab/go.mod h1:ulncasL3N9uLrVann0m+CDlJKWsIAP34MPcOJF6VRvc= +github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts= +github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1 h1:gbhw/u49SS3gkPWiYweQNJGm/uJN5GkI/FrosxSHT7A= +go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1/go.mod h1:GnOaBaFQ2we3b9AGWJpsBa7v1S5RlQzlC3O7dRMxZhM= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 h1:ssfIgGNANqpVFCndZvcuyKbl0g+UAVcbBcqGkG28H0Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0/go.mod h1:GQ/474YrbE4Jx8gZ4q5I4hrhUzM6UPzyrqJYV2AqPoQ= go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= @@ -175,19 +274,27 @@ golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= +golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= @@ -206,6 +313,10 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls= google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= diff --git a/internal/docker/client.go b/internal/docker/client.go index f348d33..46d2d53 100644 --- a/internal/docker/client.go +++ b/internal/docker/client.go @@ -4,6 +4,7 @@ package docker import ( "bufio" "context" + "encoding/json" "errors" "fmt" "io" @@ -22,7 +23,11 @@ import ( "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/jsonmessage" "github.com/docker/go-connections/nat" + controlapi "github.com/moby/buildkit/api/services/control" + buildkitclient "github.com/moby/buildkit/client" + "github.com/moby/buildkit/util/progress/progressui" "go.uber.org/fx" "sneak.berlin/go/upaas/internal/config" @@ -603,8 +608,11 @@ func (c *Client) performBuild( } }() - // Build image + // Build with BuildKit: the stages of a multi-stage build are kept in + // its build cache, which Docker limits on its own, instead of being + // left behind as untagged images. resp, err := c.docker.ImageBuild(ctx, tarArchive, dockertypes.ImageBuildOptions{ + Version: dockertypes.BuilderBuildKit, Dockerfile: opts.DockerfilePath, Tags: opts.Tags, Remove: true, @@ -622,7 +630,7 @@ func (c *Client) performBuild( }() // Stream build output line by line for real-time log updates - err = c.streamBuildOutput(resp.Body, opts.LogWriter) + err = c.streamBuildOutput(ctx, resp.Body, opts.LogWriter) if err != nil { return "", err } @@ -647,28 +655,66 @@ const scannerInitialBufferSize = 64 * 1024 // 64KB // (base64 layers can be large). const scannerMaxBufferSize = 1024 * 1024 // 1MB -// streamBuildOutput reads Docker build output line by line and writes to -// stdout and optional log writer. Docker sends newline-delimited JSON, so -// reading line by line ensures each log entry is written immediately. -func (c *Client) streamBuildOutput(body io.Reader, logWriter io.Writer) error { +// streamBuildOutput reads Docker build output line by line and writes it to +// stdout and the optional log writer as it arrives. Docker sends +// newline-delimited JSON. BuildKit's progress arrives encoded in +// "moby.buildkit.trace" messages; these are decoded and written as plain +// text, as "docker build --progress=plain" shows it. Other lines, such as +// build errors, are written unchanged. +func (c *Client) streamBuildOutput( + ctx context.Context, + body io.Reader, + logWriter io.Writer, +) error { + out := io.Writer(os.Stdout) + if logWriter != nil { + out = io.MultiWriter(os.Stdout, logWriter) + } + + display, err := progressui.NewDisplay(out, progressui.PlainMode) + if err != nil { + return fmt.Errorf("failed to create build progress display: %w", err) + } + + statuses := make(chan *buildkitclient.SolveStatus) + displayDone := make(chan struct{}) + + go func() { + defer close(displayDone) + // The display must keep reading until statuses is closed, even + // after ctx is cancelled, or the loop below would block. + _, _ = display.UpdateFrom(context.WithoutCancel(ctx), statuses) + }() + scanner := bufio.NewScanner(body) buf := make([]byte, 0, scannerInitialBufferSize) scanner.Buffer(buf, scannerMaxBufferSize) - newline := []byte{'\n'} - for scanner.Scan() { line := scanner.Bytes() - // Write to stdout - _, _ = os.Stdout.Write(line) - _, _ = os.Stdout.Write(newline) - // Write to log writer if provided - if logWriter != nil { - _, _ = logWriter.Write(line) - _, _ = logWriter.Write(newline) + + var msg jsonmessage.JSONMessage + + err = json.Unmarshal(line, &msg) + if err == nil && msg.ID == "moby.buildkit.trace" && msg.Aux != nil { + var data []byte + + var status controlapi.StatusResponse + + if json.Unmarshal(*msg.Aux, &data) == nil && status.Unmarshal(data) == nil { + statuses <- buildkitclient.NewSolveStatus(&status) + } + + continue } + + // One write per line, so it is not split by the display's output. + _, _ = fmt.Fprintf(out, "%s\n", line) } + close(statuses) + <-displayDone + scanErr := scanner.Err() if scanErr != nil { return fmt.Errorf("failed to read build output: %w", scanErr) diff --git a/internal/docker/validation_test.go b/internal/docker/validation_test.go index 3ea3378..7257344 100644 --- a/internal/docker/validation_test.go +++ b/internal/docker/validation_test.go @@ -1,7 +1,9 @@ package docker //nolint:testpackage // tests unexported regexps and Client struct import ( + "bytes" "context" + "encoding/json" "errors" "fmt" "log/slog" @@ -11,8 +13,10 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/docker/docker/client" + controlapi "github.com/moby/buildkit/api/services/control" ) // mainBranch is the branch name used across validation tests. @@ -239,3 +243,74 @@ func TestPerformCloneRemovesContainerVolumes(t *testing.T) { }) } } + +// TestPerformBuildUsesBuildKit runs a build against a fake Docker API and +// checks that it asks for BuildKit and that BuildKit's progress reaches the +// build log as plain text. +func TestPerformBuildUsesBuildKit(t *testing.T) { + t.Parallel() + + now := time.Now() + status := controlapi.StatusResponse{Vertexes: []*controlapi.Vertex{{ + Digest: "sha256:1111", + Name: "[build 2/2] RUN make", + Started: &now, + Completed: &now, + }}} + + data, err := status.Marshal() + if err != nil { + t.Fatal(err) + } + + trace, err := json.Marshal(data) + if err != nil { + t.Fatal(err) + } + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/build"): + if r.URL.Query().Get("version") != "2" { + http.Error(w, "not a BuildKit build", http.StatusBadRequest) + + return + } + + _, _ = fmt.Fprintf(w, "{\"id\":\"moby.buildkit.trace\",\"aux\":%s}\n", trace) + default: + _, _ = w.Write([]byte(`{"Id":"sha256:built"}`)) + } + }, + )) + t.Cleanup(srv.Close) + + dockerAPI, err := client.NewClientWithOpts( + client.WithHost("tcp://" + srv.Listener.Addr().String()), + ) + if err != nil { + t.Fatal(err) + } + + c := &Client{docker: dockerAPI, log: slog.Default()} + + var buildLog bytes.Buffer + + imageID, err := c.performBuild(t.Context(), BuildImageOptions{ + ContextDir: t.TempDir(), + Tags: []string{"upaas-test:1"}, + LogWriter: &buildLog, + }) + if err != nil { + t.Fatal(err) + } + + if imageID != "sha256:built" { + t.Errorf("unexpected image ID %q", imageID) + } + + if !strings.Contains(buildLog.String(), "[build 2/2] RUN make") { + t.Errorf("build log is missing the build step:\n%s", buildLog.String()) + } +} -- 2.54.0