Build apps with BuildKit so build stages stop piling up (closes #220)

Multi-stage app builds left an untagged image for every build stage after each deploy, and nothing could safely remove them. upaas now asks Docker for a BuildKit build, which keeps stages in Docker's build cache instead of as images; Docker limits and cleans that cache itself. The final image is still tagged `upaas-<app>:<N>`, so old-image cleanup is unchanged. BuildKit's progress messages are decoded and written to the deployment log as plain text. Deviation: adds `github.com/moby/buildkit` v0.16.0 (matching the pinned Docker client), added with `go get` since no make target adds dependencies. Judgement call: the cache limit is on by default only from Docker Engine 28.2; older engines need the `daemon.json` setting the README now names. Model: opus-5-5
2026-09-23 12:45:42 +02:00
parent a60ea144fb
commit a57efeed12
6 changed files with 295 additions and 15 deletions
@@ -4,6 +4,7 @@ package docker
 import (
 	"bufio"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -22,7 +23,11 @@ import (
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/jsonmessage"
 	"github.com/docker/go-connections/nat"
+	controlapi "github.com/moby/buildkit/api/services/control"
+	buildkitclient "github.com/moby/buildkit/client"
+	"github.com/moby/buildkit/util/progress/progressui"
 	"go.uber.org/fx"

 	"sneak.berlin/go/upaas/internal/config"
@@ -603,8 +608,11 @@ func (c *Client) performBuild(
 		}
 	}()

-	// Build image
+	// Build with BuildKit: the stages of a multi-stage build are kept in
+	// its build cache, which Docker limits on its own, instead of being
+	// left behind as untagged images.
 	resp, err := c.docker.ImageBuild(ctx, tarArchive, dockertypes.ImageBuildOptions{
+		Version:    dockertypes.BuilderBuildKit,
 		Dockerfile: opts.DockerfilePath,
 		Tags:       opts.Tags,
 		Remove:     true,
@@ -622,7 +630,7 @@ func (c *Client) performBuild(
 	}()

 	// Stream build output line by line for real-time log updates
-	err = c.streamBuildOutput(resp.Body, opts.LogWriter)
+	err = c.streamBuildOutput(ctx, resp.Body, opts.LogWriter)
 	if err != nil {
 		return "", err
 	}
@@ -647,28 +655,66 @@ const scannerInitialBufferSize = 64 * 1024 // 64KB
 // (base64 layers can be large).
 const scannerMaxBufferSize = 1024 * 1024 // 1MB

-// streamBuildOutput reads Docker build output line by line and writes to
-// stdout and optional log writer. Docker sends newline-delimited JSON, so
-// reading line by line ensures each log entry is written immediately.
-func (c *Client) streamBuildOutput(body io.Reader, logWriter io.Writer) error {
+// streamBuildOutput reads Docker build output line by line and writes it to
+// stdout and the optional log writer as it arrives. Docker sends
+// newline-delimited JSON. BuildKit's progress arrives encoded in
+// "moby.buildkit.trace" messages; these are decoded and written as plain
+// text, as "docker build --progress=plain" shows it. Other lines, such as
+// build errors, are written unchanged.
+func (c *Client) streamBuildOutput(
+	ctx context.Context,
+	body io.Reader,
+	logWriter io.Writer,
+) error {
+	out := io.Writer(os.Stdout)
+	if logWriter != nil {
+		out = io.MultiWriter(os.Stdout, logWriter)
+	}
+
+	display, err := progressui.NewDisplay(out, progressui.PlainMode)
+	if err != nil {
+		return fmt.Errorf("failed to create build progress display: %w", err)
+	}
+
+	statuses := make(chan *buildkitclient.SolveStatus)
+	displayDone := make(chan struct{})
+
+	go func() {
+		defer close(displayDone)
+		// The display must keep reading until statuses is closed, even
+		// after ctx is cancelled, or the loop below would block.
+		_, _ = display.UpdateFrom(context.WithoutCancel(ctx), statuses)
+	}()
+
 	scanner := bufio.NewScanner(body)
 	buf := make([]byte, 0, scannerInitialBufferSize)
 	scanner.Buffer(buf, scannerMaxBufferSize)

-	newline := []byte{'\n'}
-
 	for scanner.Scan() {
 		line := scanner.Bytes()
-		// Write to stdout
-		_, _ = os.Stdout.Write(line)
-		_, _ = os.Stdout.Write(newline)
-		// Write to log writer if provided
-		if logWriter != nil {
-			_, _ = logWriter.Write(line)
-			_, _ = logWriter.Write(newline)
+
+		var msg jsonmessage.JSONMessage
+
+		err = json.Unmarshal(line, &msg)
+		if err == nil && msg.ID == "moby.buildkit.trace" && msg.Aux != nil {
+			var data []byte
+
+			var status controlapi.StatusResponse
+
+			if json.Unmarshal(*msg.Aux, &data) == nil && status.Unmarshal(data) == nil {
+				statuses <- buildkitclient.NewSolveStatus(&status)
+			}
+
+			continue
 		}
+
+		// One write per line, so it is not split by the display's output.
+		_, _ = fmt.Fprintf(out, "%s\n", line)
 	}

+	close(statuses)
+	<-displayDone
+
 	scanErr := scanner.Err()
 	if scanErr != nil {
 		return fmt.Errorf("failed to read build output: %w", scanErr)
@@ -1,7 +1,9 @@
 package docker //nolint:testpackage // tests unexported regexps and Client struct

 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"log/slog"
@@ -11,8 +13,10 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"

 	"github.com/docker/docker/client"
+	controlapi "github.com/moby/buildkit/api/services/control"
 )

 // mainBranch is the branch name used across validation tests.
@@ -239,3 +243,74 @@ func TestPerformCloneRemovesContainerVolumes(t *testing.T) {
 		})
 	}
 }
+
+// TestPerformBuildUsesBuildKit runs a build against a fake Docker API and
+// checks that it asks for BuildKit and that BuildKit's progress reaches the
+// build log as plain text.
+func TestPerformBuildUsesBuildKit(t *testing.T) {
+	t.Parallel()
+
+	now := time.Now()
+	status := controlapi.StatusResponse{Vertexes: []*controlapi.Vertex{{
+		Digest:    "sha256:1111",
+		Name:      "[build 2/2] RUN make",
+		Started:   &now,
+		Completed: &now,
+	}}}
+
+	data, err := status.Marshal()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	trace, err := json.Marshal(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	srv := httptest.NewServer(http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+			switch {
+			case strings.HasSuffix(r.URL.Path, "/build"):
+				if r.URL.Query().Get("version") != "2" {
+					http.Error(w, "not a BuildKit build", http.StatusBadRequest)
+
+					return
+				}
+
+				_, _ = fmt.Fprintf(w, "{\"id\":\"moby.buildkit.trace\",\"aux\":%s}\n", trace)
+			default:
+				_, _ = w.Write([]byte(`{"Id":"sha256:built"}`))
+			}
+		},
+	))
+	t.Cleanup(srv.Close)
+
+	dockerAPI, err := client.NewClientWithOpts(
+		client.WithHost("tcp://" + srv.Listener.Addr().String()),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	c := &Client{docker: dockerAPI, log: slog.Default()}
+
+	var buildLog bytes.Buffer
+
+	imageID, err := c.performBuild(t.Context(), BuildImageOptions{
+		ContextDir: t.TempDir(),
+		Tags:       []string{"upaas-test:1"},
+		LogWriter:  &buildLog,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if imageID != "sha256:built" {
+		t.Errorf("unexpected image ID %q", imageID)
+	}
+
+	if !strings.Contains(buildLog.String(), "[build 2/2] RUN make") {
+		t.Errorf("build log is missing the build step:\n%s", buildLog.String())
+	}
+}