Remove old images by tag, and test the step after a deploy (closes #216)

Old images are now removed by their upaas-<app>:<N> tag, without force,
so Docker deletes an image only when no other tag (such as another app's
build of the same source) and no container still uses it. Removing by ID
with force could delete another app's rollback image.

The step after a deploy (record the new image, keep the replaced one for
rollback, remove the rest) is its own function, tested against a fake
Docker API, so the test fails if the removal is dropped or runs before
the images are updated.

Model: opus-5-5

TODO.md

@@ -20,6 +20,16 @@ regress.
 
 # Completed Steps
 
+- 2026-09-23: After a successful deploy, upaas removes the app's images other
+  than the running one and the one rollback would use, together with the
+  untagged images they were built on (#216).
+- 2026-09-23: The git clone container is now removed together with its anonymous
+  volume (the `alpine/git` image declares one at `/git`), so a deploy no longer
+  leaves a Docker volume behind (#215).
+- 2026-09-23: Deployment log files are now stored under `logs/<appname>/`
+  instead of `logs/<hostname>/<appname>/`, so downloads keep working after the
+  upaas container is recreated; logs written under an old hostname directory are
+  still found (#214).
 - 2026-09-23: Fixed the flaky `t.TempDir` cleanup race in `internal/handlers`
   (the one fixed in `internal/service/webhook` by #198):
   `TestHandleWebhookProcessesValidWebhook` now waits with the webhook service's

internal/docker/client.go

@@ -537,6 +537,55 @@ func (c *Client) RemoveImage(ctx context.Context, imageID ImageID) error {
 	return nil
 }
 
+// ListImageTags returns the tags in the given repository, such as
+// "upaas-myapp:12" in "upaas-myapp", each with the ID of its image.
+// Tags the same image has in other repositories are left out.
+func (c *Client) ListImageTags(
+	ctx context.Context,
+	repository string,
+) (map[string]ImageID, error) {
+	if c.docker == nil {
+		return nil, ErrNotConnected
+	}
+
+	images, err := c.docker.ImageList(ctx, image.ListOptions{
+		Filters: filters.NewArgs(filters.Arg("reference", repository)),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to list images: %w", err)
+	}
+
+	tags := make(map[string]ImageID)
+
+	for _, img := range images {
+		for _, tag := range img.RepoTags {
+			if strings.HasPrefix(tag, repository+":") {
+				tags[tag] = ImageID(img.ID)
+			}
+		}
+	}
+
+	return tags, nil
+}
+
+// RemoveImageTag removes a tag such as "upaas-myapp:12", without force.
+// Docker then deletes the image, and the untagged images it was built on,
+// only if no other tag and no container still uses it.
+func (c *Client) RemoveImageTag(ctx context.Context, tag string) error {
+	if c.docker == nil {
+		return ErrNotConnected
+	}
+
+	_, err := c.docker.ImageRemove(ctx, tag, image.RemoveOptions{
+		PruneChildren: true,
+	})
+	if err != nil && !client.IsErrNotFound(err) {
+		return fmt.Errorf("failed to remove image tag %s: %w", tag, err)
+	}
+
+	return nil
+}
+
 func (c *Client) performBuild(
 	ctx context.Context,
 	opts BuildImageOptions,
@@ -656,11 +705,14 @@ func (c *Client) performClone(
 		return nil, err
 	}
 
+	// The git image declares a volume, so Docker gives each clone container
+	// an anonymous volume; remove it with the container. The removal must
+	// still run when the deploy is cancelled.
 	defer func() {
 		_ = c.docker.ContainerRemove(
-			ctx,
+			context.WithoutCancel(ctx),
 			gitContainerID.String(),
-			container.RemoveOptions{Force: true},
+			container.RemoveOptions{Force: true, RemoveVolumes: true},
 		)
 	}()
 

internal/docker/validation_test.go

@@ -1,9 +1,18 @@
 package docker //nolint:testpackage // tests unexported regexps and Client struct
 
 import (
+	"context"
 	"errors"
+	"fmt"
 	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"path/filepath"
+	"strings"
 	"testing"
+
+	"github.com/docker/docker/client"
 )
 
 // mainBranch is the branch name used across validation tests.
@@ -149,3 +158,84 @@ func TestCloneRepoRejectsInjection(t *testing.T) {
 		})
 	}
 }
+
+// TestPerformCloneRemovesContainerVolumes runs a clone against a fake Docker
+// API and checks that the clone container is removed together with its
+// anonymous volumes, whether the clone succeeds, fails, or is cancelled.
+func TestPerformCloneRemovesContainerVolumes(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name     string
+		exitCode int
+		cancel   bool
+	}{
+		{name: "succeeds", exitCode: 0},
+		{name: "fails", exitCode: 1},
+		{name: "cancelled", cancel: true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			ctx, cancel := context.WithCancel(t.Context())
+			t.Cleanup(cancel)
+
+			removeQuery := make(chan url.Values, 1)
+
+			srv := httptest.NewServer(http.HandlerFunc(
+				func(w http.ResponseWriter, r *http.Request) {
+					w.Header().Set("Content-Type", "application/json")
+
+					switch {
+					case r.Method == http.MethodDelete:
+						removeQuery <- r.URL.Query()
+					case strings.HasSuffix(r.URL.Path, "/containers/create"):
+						_, _ = w.Write([]byte(`{"Id":"gitcontainer"}`))
+					case strings.HasSuffix(r.URL.Path, "/wait") && tt.cancel:
+						// Cancel the deploy while the clone is running.
+						cancel()
+						<-r.Context().Done()
+					case strings.HasSuffix(r.URL.Path, "/wait"):
+						_, _ = fmt.Fprintf(w, `{"StatusCode":%d}`, tt.exitCode)
+					default:
+						_, _ = w.Write([]byte(`{}`))
+					}
+				},
+			))
+			t.Cleanup(srv.Close)
+
+			dockerAPI, err := client.NewClientWithOpts(
+				client.WithHost("tcp://" + srv.Listener.Addr().String()),
+			)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			c := &Client{docker: dockerAPI, log: slog.Default()}
+
+			dir := t.TempDir()
+			cfg := &cloneConfig{
+				repoURL:       "git@example.com:repo.git",
+				branch:        mainBranch,
+				sshPrivateKey: "fake-key",
+				containerDir:  filepath.Join(dir, "repo"),
+				hostDir:       filepath.Join(dir, "repo"),
+				keyFile:       filepath.Join(dir, "deploy_key"),
+				hostKeyFile:   filepath.Join(dir, "deploy_key"),
+			}
+
+			_, _ = c.performClone(ctx, cfg)
+
+			select {
+			case query := <-removeQuery:
+				if query.Get("v") != "1" {
+					t.Errorf("clone container removed without its volumes: %v", query)
+				}
+			default:
+				t.Error("clone container was not removed")
+			}
+		})
+	}
+}

internal/handlers/app.go

@@ -6,9 +6,11 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/fs"
 	"net/http"
 	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -640,7 +642,7 @@ func (h *Handlers) HandleDeploymentLogDownload() http.HandlerFunc {
 		}
 		defer func() { _ = root.Close() }()
 
-		file, openErr := root.Open(relPath)
+		file, openErr := openDeploymentLog(root, relPath)
 		if openErr != nil {
 			http.NotFound(writer, request)
 
@@ -665,6 +667,24 @@ func (h *Handlers) HandleDeploymentLogDownload() http.HandlerFunc {
 	}
 }
 
+// openDeploymentLog opens a deployment log file inside the log root.
+// Logs written by older versions sit one directory deeper, under the
+// hostname of the container that wrote them, so when the file is not at
+// relPath it is looked for under any directory directly below the root.
+func openDeploymentLog(root *os.Root, relPath string) (*os.File, error) {
+	file, err := root.Open(relPath)
+	if err == nil {
+		return file, nil
+	}
+
+	matches, globErr := fs.Glob(root.FS(), path.Join("*", filepath.ToSlash(relPath)))
+	if globErr != nil || len(matches) == 0 {
+		return nil, err
+	}
+
+	return root.Open(matches[0])
+}
+
 // containerLogsAPITail is the default number of log lines for the container logs API.
 const containerLogsAPITail = "100"
 

internal/handlers/log_download_test.go

@@ -69,6 +69,56 @@ func TestHandleDeploymentLogDownloadServesLegitimateFile(t *testing.T) {
 	assert.Contains(t, recorder.Body.String(), "deploy log contents")
 }
 
+// TestGetLogFilePathHasNoHostname verifies a deployment log is stored
+// directly under logs/<appname>/, with no hostname directory in between,
+// so it is still found after the container is recreated with a new
+// hostname.
+func TestGetLogFilePathHasNoHostname(t *testing.T) {
+	t.Parallel()
+
+	testCtx := setupTestHandlers(t)
+	createdApp := createTestApp(t, testCtx, "log-path-app")
+
+	deployment := models.NewDeployment(testCtx.database)
+	deployment.AppID = createdApp.ID
+
+	logPath := testCtx.deploySvc.GetLogFilePath(createdApp, deployment)
+
+	assert.Equal(t,
+		filepath.Join(testCtx.deploySvc.GetLogDir(), createdApp.Name),
+		filepath.Dir(logPath),
+	)
+}
+
+// TestHandleDeploymentLogDownloadServesLogFromOldHostnameDir verifies a
+// log written by an older version, under the hostname of a container that
+// has since been recreated, can still be downloaded.
+func TestHandleDeploymentLogDownloadServesLogFromOldHostnameDir(t *testing.T) {
+	t.Parallel()
+
+	testCtx := setupTestHandlers(t)
+	createdApp := createTestApp(t, testCtx, "log-old-hostname-app")
+
+	deployment := models.NewDeployment(testCtx.database)
+	deployment.AppID = createdApp.ID
+	deployment.Status = models.DeploymentStatusSuccess
+	require.NoError(t, deployment.Save(context.Background()))
+
+	logDir := testCtx.deploySvc.GetLogDir()
+	newPath := testCtx.deploySvc.GetLogFilePath(createdApp, deployment)
+	relPath, relErr := filepath.Rel(logDir, newPath)
+	require.NoError(t, relErr)
+
+	oldPath := filepath.Join(logDir, "old-container-hostname", relPath)
+	require.NoError(t, os.MkdirAll(filepath.Dir(oldPath), 0o750))
+	require.NoError(t, os.WriteFile(oldPath, []byte("old deploy log contents"), 0o600))
+
+	recorder := doLogDownload(t, testCtx, createdApp.ID, deployment.ID)
+
+	assert.Equal(t, http.StatusOK, recorder.Code)
+	assert.Contains(t, recorder.Body.String(), "old deploy log contents")
+}
+
 // TestHandleDeploymentLogDownloadRejectsPathTraversal verifies the
 // os.Root containment guard. A traversal-shaped app name drives the
 // resolved log path out of the deploy log directory onto a sentinel

internal/service/deploy/deploy.go

@@ -9,8 +9,10 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"maps"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -261,15 +263,14 @@ func (svc *Service) GetBuildDir(appName string) string {
 
 // GetLogFilePath returns the path to the log file for a deployment.
 // Returns empty string if the path cannot be determined.
+//
+// The path must not depend on the container's hostname: Docker assigns a
+// new one whenever the container is recreated, and older logs would then
+// no longer be found.
 func (svc *Service) GetLogFilePath(
 	app *models.App,
 	deployment *models.Deployment,
 ) string {
-	hostname, err := os.Hostname()
-	if err != nil {
-		hostname = "unknown"
-	}
-
 	// Get commit SHA
 	sha := ""
 	if deployment.CommitSHA.Valid && deployment.CommitSHA.String != "" {
@@ -291,7 +292,7 @@ func (svc *Service) GetLogFilePath(
 		filename = fmt.Sprintf("%s_%s.log.txt", app.Name, timestamp)
 	}
 
-	return filepath.Join(svc.config.DataDir, "logs", hostname, app.Name, filename)
+	return filepath.Join(svc.config.DataDir, "logs", app.Name, filename)
 }
 
 // GetLogDir returns the root directory under which all deployment log
@@ -525,12 +526,7 @@ func (svc *Service) runBuildAndDeploy(
 		return err
 	}
 
-	// Save current image as previous before updating to new one
+	err = svc.recordDeployedImage(bgCtx, app, deployment, imageID)
-	if app.ImageID.Valid && app.ImageID.String != "" {
-		app.PreviousImageID = app.ImageID
-	}
-
-	err = svc.updateAppRunning(bgCtx, app, imageID)
 	if err != nil {
 		return err
 	}
@@ -715,6 +711,68 @@ func (svc *Service) checkCancelled(
 	return ErrDeployCancelled
 }
 
+// recordDeployedImage runs once the new container has started: it makes
+// imageID the app's current image, keeps the replaced one as the previous
+// image for Rollback, and then removes the app's other images.
+func (svc *Service) recordDeployedImage(
+	ctx context.Context,
+	app *models.App,
+	deployment *models.Deployment,
+	imageID docker.ImageID,
+) error {
+	// Save current image as previous before updating to new one
+	if app.ImageID.Valid && app.ImageID.String != "" {
+		app.PreviousImageID = app.ImageID
+	}
+
+	err := svc.updateAppRunning(ctx, app, imageID)
+	if err != nil {
+		return err
+	}
+
+	svc.removeUnusedImages(ctx, app, deployment)
+
+	return nil
+}
+
+// removeUnusedImages removes the app's tags (upaas-<app>:<deployment>, set by
+// buildImage) except those of the image the running container uses and the
+// one Rollback would start. Docker deletes an image only once no other tag,
+// such as another app's, and no container still uses it.
+func (svc *Service) removeUnusedImages(
+	ctx context.Context,
+	app *models.App,
+	deployment *models.Deployment,
+) {
+	tags, err := svc.docker.ListImageTags(ctx, "upaas-"+app.Name)
+	if err != nil {
+		svc.log.Error("failed to list app images", "error", err, "app", app.Name)
+
+		return
+	}
+
+	for _, tag := range slices.Sorted(maps.Keys(tags)) {
+		imageID := tags[tag].String()
+		if imageID == app.ImageID.String || imageID == app.PreviousImageID.String {
+			continue
+		}
+
+		removeErr := svc.docker.RemoveImageTag(ctx, tag)
+		if removeErr != nil {
+			svc.log.Error("failed to remove old image",
+				"error", removeErr, "app", app.Name, "tag", tag)
+			_ = deployment.AppendLog(
+				ctx,
+				"WARNING: failed to remove old image "+tag+": "+removeErr.Error(),
+			)
+
+			continue
+		}
+
+		_ = deployment.AppendLog(ctx, "Removed old image: "+tag)
+	}
+}
+
 // cleanupCancelledDeploy removes orphan resources left by a cancelled deployment.
 func (svc *Service) cleanupCancelledDeploy(
 	ctx context.Context,
@@ -1294,7 +1352,7 @@ func (svc *Service) failDeployment(
 }
 
 // writeLogsToFile writes the deployment logs to a file on disk.
-// Structure: DataDir/logs/<hostname>/<appname>/<appname>_<sha>_<timestamp>.log.txt
+// Structure: DataDir/logs/<appname>/<appname>_<sha>_<timestamp>.log.txt
 func (svc *Service) writeLogsToFile(app *models.App, deployment *models.Deployment) {
 	if !deployment.Logs.Valid || deployment.Logs.String == "" {
 		return

internal/service/deploy/deploy_images_test.go

@@ -0,0 +1,106 @@
+package deploy_test
+
+import (
+	"context"
+	"database/sql"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/fx/fxtest"
+
+	"sneak.berlin/go/upaas/internal/config"
+	"sneak.berlin/go/upaas/internal/database"
+	"sneak.berlin/go/upaas/internal/docker"
+	"sneak.berlin/go/upaas/internal/logger"
+	"sneak.berlin/go/upaas/internal/models"
+	"sneak.berlin/go/upaas/internal/service/deploy"
+)
+
+// TestRecordDeployedImageRemovesOldImages runs the step after a deploy
+// against a fake Docker API. Image one is also tagged for another app,
+// image two was the previous image, three the current one, four is new.
+func TestRecordDeployedImageRemovesOldImages(t *testing.T) {
+	t.Parallel()
+
+	var (
+		mu      sync.Mutex
+		removed []string
+		forced  bool
+	)
+
+	srv := httptest.NewServer(http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Type", "application/json")
+
+			switch {
+			case r.Method == http.MethodDelete:
+				_, name, _ := strings.Cut(r.URL.Path, "/images/")
+
+				mu.Lock()
+
+				removed = append(removed, name)
+				forced = forced || r.URL.Query().Get("force") != ""
+				mu.Unlock()
+
+				_, _ = w.Write([]byte(`[]`))
+			case strings.HasSuffix(r.URL.Path, "/images/json"):
+				_, _ = w.Write([]byte(`[
+					{"Id":"sha256:one","RepoTags":["upaas-myapp:1","upaas-otherapp:7"]},
+					{"Id":"sha256:two","RepoTags":["upaas-myapp:2"]},
+					{"Id":"sha256:three","RepoTags":["upaas-myapp:3"]},
+					{"Id":"sha256:four","RepoTags":["upaas-myapp:4"]}
+				]`))
+			default:
+				_, _ = w.Write([]byte(`{}`))
+			}
+		},
+	))
+	t.Cleanup(srv.Close)
+
+	log := slog.New(slog.NewTextHandler(os.Stderr, nil))
+	lifecycle := fxtest.NewLifecycle(t)
+
+	dockerClient, err := docker.New(lifecycle, docker.Params{
+		Logger: logger.NewForTest(log),
+		Config: &config.Config{DockerHost: "tcp://" + srv.Listener.Addr().String()},
+	})
+	require.NoError(t, err)
+
+	lifecycle.RequireStart()
+	t.Cleanup(lifecycle.RequireStop)
+
+	db := database.NewTestDatabase(t)
+	ctx := context.Background()
+
+	app := models.NewApp(db)
+	app.ID = "myapp-id"
+	app.Name = "myapp"
+	app.ImageID = sql.NullString{String: "sha256:three", Valid: true}
+	app.PreviousImageID = sql.NullString{String: "sha256:two", Valid: true}
+	require.NoError(t, app.Save(ctx))
+
+	deployment := models.NewDeployment(db)
+	deployment.AppID = app.ID
+	require.NoError(t, deployment.Save(ctx))
+
+	svc := deploy.NewTestServiceWithConfig(log, &config.Config{}, dockerClient)
+
+	err = svc.RecordDeployedImage(ctx, app, deployment, "sha256:four")
+	require.NoError(t, err)
+
+	assert.Equal(t, "sha256:four", app.ImageID.String)
+	assert.Equal(t, "sha256:three", app.PreviousImageID.String)
+
+	mu.Lock()
+	defer mu.Unlock()
+
+	assert.Equal(t, []string{"upaas-myapp:1", "upaas-myapp:2"}, removed)
+	assert.False(t, forced, "old image tags must be removed without force")
+}

internal/service/deploy/export_test.go

@@ -90,6 +90,16 @@ func (svc *Service) GetBuildDirExported(appName string) string {
 	return svc.GetBuildDir(appName)
 }
 
+// RecordDeployedImage exposes recordDeployedImage for testing.
+func (svc *Service) RecordDeployedImage(
+	ctx context.Context,
+	app *models.App,
+	deployment *models.Deployment,
+	imageID docker.ImageID,
+) error {
+	return svc.recordDeployedImage(ctx, app, deployment, imageID)
+}
+
 // BuildContainerOptionsExported exposes buildContainerOptions for testing.
 func (svc *Service) BuildContainerOptionsExported(
 	ctx context.Context,