clawbot
a459de3223
Builds are tagged upaas-<app>:<short hash>, git's own short form of the commit checked out, instead of the deployment number. A redeploy of a commit gives the tag to the new image. The cleanup after a deploy now also finds the app's untagged images among those its deployments recorded, and removes them by ID once the app neither runs them nor would roll back to them. It keeps every image any app uses, since apps that build the same commit can share one. The clone reads the commits from git's output after removing Docker's log headers, which had hidden the COMMIT: line; commit_sha is now saved on update so manual deploys keep the commit they recorded. Model: opus-5-5
325 lines
10 KiB
Go
325 lines
10 KiB
Go
package deploy_test
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"maps"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
|
|
"github.com/docker/docker/pkg/stdcopy"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"go.uber.org/fx/fxtest"
|
|
|
|
"sneak.berlin/go/upaas/internal/config"
|
|
"sneak.berlin/go/upaas/internal/database"
|
|
"sneak.berlin/go/upaas/internal/docker"
|
|
"sneak.berlin/go/upaas/internal/logger"
|
|
"sneak.berlin/go/upaas/internal/models"
|
|
"sneak.berlin/go/upaas/internal/service/deploy"
|
|
)
|
|
|
|
// fakeImageAPI is a fake Docker API that keeps images and their tags as
|
|
// Docker does: a build gives its tag to the image it builds, and removing
|
|
// an image's last tag, or an untagged image by its ID, deletes the image.
|
|
// It also answers the steps of a git clone that reports shortSHA.
|
|
type fakeImageAPI struct {
|
|
mu sync.Mutex
|
|
images map[string][]string // image ID -> tags
|
|
shortSHA string // the commit's short hash the clone reports
|
|
nextID string // ID of the image the next build creates
|
|
built []string // the tag of each build
|
|
removed []string // each tag or ID removed
|
|
forced bool // whether a removal was forced
|
|
}
|
|
|
|
func (api *fakeImageAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
api.mu.Lock()
|
|
defer api.mu.Unlock()
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
|
|
_, name, isImage := strings.Cut(r.URL.Path, "/images/")
|
|
|
|
switch {
|
|
case strings.HasSuffix(r.URL.Path, "/images/json"):
|
|
dangling := strings.Contains(r.URL.Query().Get("filters"), "dangling")
|
|
api.listImages(w, dangling)
|
|
case isImage && r.Method == http.MethodDelete:
|
|
api.forced = api.forced || r.URL.Query().Get("force") != ""
|
|
api.removeImage(w, name)
|
|
case isImage && strings.HasSuffix(name, "/json"):
|
|
api.inspectImage(w, strings.TrimSuffix(name, "/json"))
|
|
case strings.HasSuffix(r.URL.Path, "/build"):
|
|
tag := r.URL.Query().Get("t")
|
|
api.built = append(api.built, tag)
|
|
api.untag(tag)
|
|
api.images[api.nextID] = append(api.images[api.nextID], tag)
|
|
case strings.HasSuffix(r.URL.Path, "/version"):
|
|
_, _ = w.Write([]byte(`{"Version":"27.3.1","ApiVersion":"1.47"}`))
|
|
case strings.HasSuffix(r.URL.Path, "/containers/create"):
|
|
_, _ = w.Write([]byte(`{"Id":"gitcontainer"}`))
|
|
case strings.HasSuffix(r.URL.Path, "/logs"):
|
|
writeCloneOutput(w, api.shortSHA)
|
|
default:
|
|
// The other steps of the git clone, which succeeds.
|
|
_, _ = w.Write([]byte(`{}`))
|
|
}
|
|
}
|
|
|
|
// listImages lists the untagged images, or else the tagged ones.
|
|
func (api *fakeImageAPI) listImages(w http.ResponseWriter, dangling bool) {
|
|
list := []map[string]any{}
|
|
|
|
for _, id := range slices.Sorted(maps.Keys(api.images)) {
|
|
if (len(api.images[id]) == 0) == dangling {
|
|
list = append(list, map[string]any{"Id": id, "RepoTags": api.images[id]})
|
|
}
|
|
}
|
|
|
|
_ = json.NewEncoder(w).Encode(list)
|
|
}
|
|
|
|
func (api *fakeImageAPI) inspectImage(w http.ResponseWriter, name string) {
|
|
for id, tags := range api.images {
|
|
if id == name || slices.Contains(tags, name) {
|
|
_ = json.NewEncoder(w).Encode(map[string]any{"Id": id, "RepoTags": tags})
|
|
|
|
return
|
|
}
|
|
}
|
|
|
|
w.WriteHeader(http.StatusNotFound)
|
|
_, _ = w.Write([]byte(`{"message":"No such image"}`))
|
|
}
|
|
|
|
func (api *fakeImageAPI) removeImage(w http.ResponseWriter, name string) {
|
|
api.removed = append(api.removed, name)
|
|
|
|
id := name
|
|
if _, isID := api.images[name]; !isID {
|
|
id = api.untag(name)
|
|
}
|
|
|
|
if len(api.images[id]) == 0 {
|
|
delete(api.images, id)
|
|
}
|
|
|
|
_, _ = w.Write([]byte(`[]`))
|
|
}
|
|
|
|
// untag removes tag from the image that has it, leaving the image, and
|
|
// returns the image's ID.
|
|
func (api *fakeImageAPI) untag(tag string) string {
|
|
for id, tags := range api.images {
|
|
if slices.Contains(tags, tag) {
|
|
api.images[id] = slices.DeleteFunc(tags, func(t string) bool { return t == tag })
|
|
|
|
return id
|
|
}
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
// state returns each image's tags, the tag of each build and each tag or
|
|
// ID removed.
|
|
func (api *fakeImageAPI) state() (map[string][]string, []string, []string) {
|
|
api.mu.Lock()
|
|
defer api.mu.Unlock()
|
|
|
|
return maps.Clone(api.images), slices.Clone(api.built), slices.Clone(api.removed)
|
|
}
|
|
|
|
// writeCloneOutput writes the line of a git clone's output that gives the
|
|
// commit's short hash, as Docker sends a container's log: after a header.
|
|
func writeCloneOutput(w io.Writer, shortSHA string) {
|
|
out := stdcopy.NewStdWriter(w, stdcopy.Stdout)
|
|
|
|
_, _ = fmt.Fprintf(out, "SHORT_SHA:%s\n", shortSHA)
|
|
}
|
|
|
|
// newImageTestService returns a deploy service that uses api as its
|
|
// Docker API, and its database.
|
|
func newImageTestService(
|
|
t *testing.T,
|
|
api *fakeImageAPI,
|
|
) (*deploy.Service, *database.Database) {
|
|
t.Helper()
|
|
|
|
srv := httptest.NewServer(api)
|
|
t.Cleanup(srv.Close)
|
|
|
|
log := slog.New(slog.NewTextHandler(os.Stderr, nil))
|
|
lifecycle := fxtest.NewLifecycle(t)
|
|
|
|
dockerClient, err := docker.New(lifecycle, docker.Params{
|
|
Logger: logger.NewForTest(log),
|
|
Config: &config.Config{DockerHost: "tcp://" + srv.Listener.Addr().String()},
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
lifecycle.RequireStart()
|
|
t.Cleanup(lifecycle.RequireStop)
|
|
|
|
db := database.NewTestDatabase(t)
|
|
dataDir := t.TempDir()
|
|
cfg := &config.Config{DataDir: dataDir, HostDataDir: dataDir}
|
|
|
|
return deploy.NewTestServiceWithConfig(log, cfg, db, dockerClient), db
|
|
}
|
|
|
|
// saveApp saves an app with the given current and previous image.
|
|
func saveApp(
|
|
t *testing.T,
|
|
db *database.Database,
|
|
name, imageID, previousImageID string,
|
|
) *models.App {
|
|
t.Helper()
|
|
|
|
app := models.NewApp(db)
|
|
app.ID = name + "-id"
|
|
app.Name = name
|
|
app.ImageID = sql.NullString{String: imageID, Valid: true}
|
|
app.PreviousImageID = sql.NullString{String: previousImageID, Valid: true}
|
|
require.NoError(t, app.Save(context.Background()))
|
|
|
|
return app
|
|
}
|
|
|
|
// TestRecordDeployedImageRemovesOldImages runs the step after a deploy
|
|
// against a fake Docker API. Image one has a tag from before images were
|
|
// tagged with their commit, and is also tagged for another app. Image two
|
|
// was the previous image, three the current one, four is new. Image five is
|
|
// the other app's previous image, which a redeploy of its commit left
|
|
// without the other app's tag.
|
|
func TestRecordDeployedImageRemovesOldImages(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
api := &fakeImageAPI{images: map[string][]string{
|
|
"sha256:one": {"upaas-myapp:140", "upaas-otherapp:1a2b3c4"},
|
|
"sha256:two": {"upaas-myapp:2b3c4d5"},
|
|
"sha256:three": {"upaas-myapp:3c4d5e6"},
|
|
"sha256:four": {"upaas-myapp:4d5e6f7"},
|
|
"sha256:five": {"upaas-myapp:5e6f7a8"},
|
|
}}
|
|
svc, db := newImageTestService(t, api)
|
|
ctx := context.Background()
|
|
|
|
app := saveApp(t, db, "myapp", "sha256:three", "sha256:two")
|
|
saveApp(t, db, "otherapp", "sha256:other", "sha256:five")
|
|
|
|
deployment := models.NewDeployment(db)
|
|
deployment.AppID = app.ID
|
|
require.NoError(t, deployment.Save(ctx))
|
|
|
|
err := svc.RecordDeployedImage(ctx, app, deployment, "sha256:four")
|
|
require.NoError(t, err)
|
|
|
|
assert.Equal(t, "sha256:four", app.ImageID.String)
|
|
assert.Equal(t, "sha256:three", app.PreviousImageID.String)
|
|
|
|
images, _, removed := api.state()
|
|
|
|
assert.Equal(t, []string{"upaas-myapp:140", "upaas-myapp:2b3c4d5"}, removed)
|
|
assert.Equal(t, map[string][]string{
|
|
"sha256:one": {"upaas-otherapp:1a2b3c4"},
|
|
"sha256:three": {"upaas-myapp:3c4d5e6"},
|
|
"sha256:four": {"upaas-myapp:4d5e6f7"},
|
|
"sha256:five": {"upaas-myapp:5e6f7a8"},
|
|
}, images)
|
|
assert.False(t, api.forced, "old images must be removed without force")
|
|
}
|
|
|
|
// TestRedeployRemovesImagesLeftWithoutTag deploys commits against a fake
|
|
// Docker API, some of them again. A build takes the commit's tag from the
|
|
// image an earlier build of it made. That image is kept, without a tag,
|
|
// while the app runs it or Rollback would start it, and is removed by its
|
|
// ID once neither does.
|
|
func TestRedeployRemovesImagesLeftWithoutTag(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// The app runs commit abc1234 and would roll back to def5678. The last
|
|
// deploy, of commit 0123abc, failed after its build.
|
|
api := &fakeImageAPI{images: map[string][]string{
|
|
"sha256:built-abc1234": {"upaas-myapp:abc1234"},
|
|
"sha256:built-def5678": {"upaas-myapp:def5678"},
|
|
"sha256:failed-0123abc": {"upaas-myapp:0123abc"},
|
|
}}
|
|
svc, db := newImageTestService(t, api)
|
|
ctx := context.Background()
|
|
|
|
app := saveApp(t, db, "myapp", "sha256:built-abc1234", "sha256:built-def5678")
|
|
|
|
for imageID := range api.images {
|
|
deployment := models.NewDeployment(db)
|
|
deployment.AppID = app.ID
|
|
deployment.ImageID = sql.NullString{String: imageID, Valid: true}
|
|
require.NoError(t, deployment.Save(ctx))
|
|
}
|
|
|
|
deployCommit := func(shortSHA, imageID string) {
|
|
t.Helper()
|
|
|
|
api.mu.Lock()
|
|
api.shortSHA = shortSHA
|
|
api.nextID = imageID
|
|
api.mu.Unlock()
|
|
|
|
deployment := models.NewDeployment(db)
|
|
deployment.AppID = app.ID
|
|
require.NoError(t, deployment.Save(ctx))
|
|
|
|
built, err := svc.BuildImage(ctx, app, deployment)
|
|
require.NoError(t, err)
|
|
require.NoError(t, svc.RecordDeployedImage(ctx, app, deployment, built))
|
|
}
|
|
|
|
// The failed deploy's image loses its tag, and nothing uses it.
|
|
deployCommit("0123abc", "sha256:retried-0123abc")
|
|
|
|
images, _, _ := api.state()
|
|
assert.Equal(t, map[string][]string{
|
|
"sha256:built-abc1234": {"upaas-myapp:abc1234"},
|
|
"sha256:retried-0123abc": {"upaas-myapp:0123abc"},
|
|
}, images)
|
|
|
|
// The running image loses its tag and becomes the one Rollback starts.
|
|
deployCommit("0123abc", "sha256:rebuilt-0123abc")
|
|
|
|
images, _, _ = api.state()
|
|
assert.Equal(t, map[string][]string{
|
|
"sha256:rebuilt-0123abc": {"upaas-myapp:0123abc"},
|
|
"sha256:retried-0123abc": {},
|
|
}, images)
|
|
assert.Equal(t, "sha256:retried-0123abc", app.PreviousImageID.String)
|
|
|
|
// Once Rollback no longer needs it, the untagged image is removed.
|
|
deployCommit("4567def", "sha256:built-4567def")
|
|
|
|
images, built, removed := api.state()
|
|
assert.Equal(t, map[string][]string{
|
|
"sha256:built-4567def": {"upaas-myapp:4567def"},
|
|
"sha256:rebuilt-0123abc": {"upaas-myapp:0123abc"},
|
|
}, images)
|
|
assert.Equal(t, []string{
|
|
"upaas-myapp:0123abc", "upaas-myapp:0123abc", "upaas-myapp:4567def",
|
|
}, built)
|
|
assert.Equal(t, []string{
|
|
"sha256:failed-0123abc", "upaas-myapp:def5678", // first deploy
|
|
"upaas-myapp:abc1234", // second deploy
|
|
"sha256:retried-0123abc", // third deploy
|
|
}, removed)
|
|
assert.False(t, api.forced, "old images must be removed without force")
|
|
}
|