feat: add backup/restore of app configurations
All checks were successful
Check / check (pull_request) Successful in 3m25s

Add export and import functionality for app configurations:

- Export single app or all apps as versioned JSON backup bundle
- Import from backup file with name-conflict detection (skip duplicates)
- Fresh SSH keys and webhook secrets generated on import
- Preserves env vars, labels, volumes, and port mappings
- Web UI: export button on app detail, backup/restore page on dashboard
- REST API: GET /api/v1/apps/{id}/export, GET /api/v1/backup/export,
  POST /api/v1/backup/import
- Comprehensive test coverage for service and handler layers
This commit is contained in:
user
2026-03-17 02:17:34 -07:00
parent fd110e69db
commit bb91f314c5
11 changed files with 1740 additions and 6 deletions

View File

@@ -0,0 +1,391 @@
package app
import (
"context"
"fmt"
"time"
"sneak.berlin/go/upaas/internal/models"
)
// BackupEnvVar represents an environment variable in a backup.
type BackupEnvVar struct {
Key string `json:"key"`
Value string `json:"value"`
}
// BackupLabel represents a Docker label in a backup.
type BackupLabel struct {
Key string `json:"key"`
Value string `json:"value"`
}
// BackupVolume represents a volume mount in a backup.
type BackupVolume struct {
HostPath string `json:"hostPath"`
ContainerPath string `json:"containerPath"`
ReadOnly bool `json:"readOnly"`
}
// BackupPort represents a port mapping in a backup.
type BackupPort struct {
HostPort int `json:"hostPort"`
ContainerPort int `json:"containerPort"`
Protocol string `json:"protocol"`
}
// Backup represents the exported configuration of a single app.
type Backup struct {
Name string `json:"name"`
RepoURL string `json:"repoUrl"`
Branch string `json:"branch"`
DockerfilePath string `json:"dockerfilePath"`
DockerNetwork string `json:"dockerNetwork,omitempty"`
NtfyTopic string `json:"ntfyTopic,omitempty"`
SlackWebhook string `json:"slackWebhook,omitempty"`
EnvVars []BackupEnvVar `json:"envVars"`
Labels []BackupLabel `json:"labels"`
Volumes []BackupVolume `json:"volumes"`
Ports []BackupPort `json:"ports"`
}
// BackupBundle represents a complete backup of one or more apps.
type BackupBundle struct {
Version int `json:"version"`
ExportedAt string `json:"exportedAt"`
Apps []Backup `json:"apps"`
}
// backupVersion is the current backup format version.
const backupVersion = 1
// ExportApp exports a single app's configuration as a BackupBundle.
func (svc *Service) ExportApp(
ctx context.Context,
application *models.App,
) (*BackupBundle, error) {
appBackup, err := svc.buildAppBackup(ctx, application)
if err != nil {
return nil, err
}
return &BackupBundle{
Version: backupVersion,
ExportedAt: time.Now().UTC().Format(time.RFC3339),
Apps: []Backup{appBackup},
}, nil
}
// ExportAllApps exports all app configurations as a BackupBundle.
func (svc *Service) ExportAllApps(ctx context.Context) (*BackupBundle, error) {
apps, err := models.AllApps(ctx, svc.db)
if err != nil {
return nil, fmt.Errorf("listing apps for export: %w", err)
}
backups := make([]Backup, 0, len(apps))
for _, application := range apps {
appBackup, buildErr := svc.buildAppBackup(ctx, application)
if buildErr != nil {
return nil, buildErr
}
backups = append(backups, appBackup)
}
return &BackupBundle{
Version: backupVersion,
ExportedAt: time.Now().UTC().Format(time.RFC3339),
Apps: backups,
}, nil
}
// ImportApps imports app configurations from a BackupBundle.
// It creates new apps (with fresh IDs, SSH keys, and webhook secrets)
// and populates their env vars, labels, volumes, and ports.
// Apps whose names conflict with existing apps are skipped and reported.
func (svc *Service) ImportApps(
ctx context.Context,
bundle *BackupBundle,
) ([]string, []string, error) {
// Build a set of existing app names for conflict detection
existingApps, listErr := models.AllApps(ctx, svc.db)
if listErr != nil {
return nil, nil, fmt.Errorf("listing existing apps: %w", listErr)
}
existingNames := make(map[string]bool, len(existingApps))
for _, a := range existingApps {
existingNames[a.Name] = true
}
var imported, skipped []string
for _, ab := range bundle.Apps {
if existingNames[ab.Name] {
skipped = append(skipped, ab.Name)
continue
}
importErr := svc.importSingleApp(ctx, ab)
if importErr != nil {
return imported, skipped, fmt.Errorf(
"importing app %q: %w", ab.Name, importErr,
)
}
imported = append(imported, ab.Name)
}
return imported, skipped, nil
}
// importSingleApp creates a single app from backup data.
func (svc *Service) importSingleApp(
ctx context.Context,
ab Backup,
) error {
createdApp, createErr := svc.CreateApp(ctx, CreateAppInput{
Name: ab.Name,
RepoURL: ab.RepoURL,
Branch: ab.Branch,
DockerfilePath: ab.DockerfilePath,
DockerNetwork: ab.DockerNetwork,
NtfyTopic: ab.NtfyTopic,
SlackWebhook: ab.SlackWebhook,
})
if createErr != nil {
return fmt.Errorf("creating app: %w", createErr)
}
envErr := svc.importEnvVars(ctx, createdApp.ID, ab.EnvVars)
if envErr != nil {
return envErr
}
labelErr := svc.importLabels(ctx, createdApp.ID, ab.Labels)
if labelErr != nil {
return labelErr
}
volErr := svc.importVolumes(ctx, createdApp.ID, ab.Volumes)
if volErr != nil {
return volErr
}
portErr := svc.importPorts(ctx, createdApp.ID, ab.Ports)
if portErr != nil {
return portErr
}
svc.log.Info("app imported from backup",
"id", createdApp.ID, "name", createdApp.Name)
return nil
}
// importEnvVars adds env vars from backup to an app.
func (svc *Service) importEnvVars(
ctx context.Context,
appID string,
envVars []BackupEnvVar,
) error {
for _, ev := range envVars {
addErr := svc.AddEnvVar(ctx, appID, ev.Key, ev.Value)
if addErr != nil {
return fmt.Errorf("adding env var %q: %w", ev.Key, addErr)
}
}
return nil
}
// importLabels adds labels from backup to an app.
func (svc *Service) importLabels(
ctx context.Context,
appID string,
labels []BackupLabel,
) error {
for _, l := range labels {
addErr := svc.AddLabel(ctx, appID, l.Key, l.Value)
if addErr != nil {
return fmt.Errorf("adding label %q: %w", l.Key, addErr)
}
}
return nil
}
// importVolumes adds volumes from backup to an app.
func (svc *Service) importVolumes(
ctx context.Context,
appID string,
volumes []BackupVolume,
) error {
for _, v := range volumes {
addErr := svc.AddVolume(ctx, appID, v.HostPath, v.ContainerPath, v.ReadOnly)
if addErr != nil {
return fmt.Errorf("adding volume %q: %w", v.ContainerPath, addErr)
}
}
return nil
}
// importPorts adds ports from backup to an app.
func (svc *Service) importPorts(
ctx context.Context,
appID string,
ports []BackupPort,
) error {
for _, p := range ports {
port := models.NewPort(svc.db)
port.AppID = appID
port.HostPort = p.HostPort
port.ContainerPort = p.ContainerPort
port.Protocol = models.PortProtocol(p.Protocol)
if port.Protocol == "" {
port.Protocol = models.PortProtocolTCP
}
saveErr := port.Save(ctx)
if saveErr != nil {
return fmt.Errorf("adding port %d: %w", p.HostPort, saveErr)
}
}
return nil
}
// buildAppBackup collects all configuration for a single app into a Backup.
func (svc *Service) buildAppBackup(
ctx context.Context,
application *models.App,
) (Backup, error) {
envVars, labels, volumes, ports, err := svc.fetchAppResources(ctx, application)
if err != nil {
return Backup{}, err
}
backup := Backup{
Name: application.Name,
RepoURL: application.RepoURL,
Branch: application.Branch,
DockerfilePath: application.DockerfilePath,
EnvVars: convertEnvVars(envVars),
Labels: convertLabels(labels),
Volumes: convertVolumes(volumes),
Ports: convertPorts(ports),
}
if application.DockerNetwork.Valid {
backup.DockerNetwork = application.DockerNetwork.String
}
if application.NtfyTopic.Valid {
backup.NtfyTopic = application.NtfyTopic.String
}
if application.SlackWebhook.Valid {
backup.SlackWebhook = application.SlackWebhook.String
}
return backup, nil
}
// fetchAppResources retrieves all sub-resources for an app.
func (svc *Service) fetchAppResources(
ctx context.Context,
application *models.App,
) ([]*models.EnvVar, []*models.Label, []*models.Volume, []*models.Port, error) {
envVars, err := application.GetEnvVars(ctx)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf(
"getting env vars for %q: %w", application.Name, err,
)
}
labels, err := application.GetLabels(ctx)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf(
"getting labels for %q: %w", application.Name, err,
)
}
volumes, err := application.GetVolumes(ctx)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf(
"getting volumes for %q: %w", application.Name, err,
)
}
ports, err := application.GetPorts(ctx)
if err != nil {
return nil, nil, nil, nil, fmt.Errorf(
"getting ports for %q: %w", application.Name, err,
)
}
return envVars, labels, volumes, ports, nil
}
// convertEnvVars converts model env vars to backup format.
func convertEnvVars(envVars []*models.EnvVar) []BackupEnvVar {
result := make([]BackupEnvVar, 0, len(envVars))
for _, ev := range envVars {
result = append(result, BackupEnvVar{
Key: ev.Key,
Value: ev.Value,
})
}
return result
}
// convertLabels converts model labels to backup format.
func convertLabels(labels []*models.Label) []BackupLabel {
result := make([]BackupLabel, 0, len(labels))
for _, l := range labels {
result = append(result, BackupLabel{
Key: l.Key,
Value: l.Value,
})
}
return result
}
// convertVolumes converts model volumes to backup format.
func convertVolumes(volumes []*models.Volume) []BackupVolume {
result := make([]BackupVolume, 0, len(volumes))
for _, v := range volumes {
result = append(result, BackupVolume{
HostPath: v.HostPath,
ContainerPath: v.ContainerPath,
ReadOnly: v.ReadOnly,
})
}
return result
}
// convertPorts converts model ports to backup format.
func convertPorts(ports []*models.Port) []BackupPort {
result := make([]BackupPort, 0, len(ports))
for _, p := range ports {
result = append(result, BackupPort{
HostPort: p.HostPort,
ContainerPort: p.ContainerPort,
Protocol: string(p.Protocol),
})
}
return result
}

View File

@@ -0,0 +1,379 @@
package app_test
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/fx"
"sneak.berlin/go/upaas/internal/config"
"sneak.berlin/go/upaas/internal/database"
"sneak.berlin/go/upaas/internal/globals"
"sneak.berlin/go/upaas/internal/logger"
"sneak.berlin/go/upaas/internal/models"
"sneak.berlin/go/upaas/internal/service/app"
)
// backupTestContext bundles test dependencies for backup tests.
type backupTestContext struct {
svc *app.Service
db *database.Database
}
func setupBackupTest(t *testing.T) *backupTestContext {
t.Helper()
tmpDir := t.TempDir()
globals.SetAppname("upaas-test")
globals.SetVersion("test")
globalsInst, err := globals.New(fx.Lifecycle(nil))
require.NoError(t, err)
loggerInst, err := logger.New(
fx.Lifecycle(nil),
logger.Params{Globals: globalsInst},
)
require.NoError(t, err)
cfg := &config.Config{
Port: 8080,
DataDir: tmpDir,
SessionSecret: "test-secret-key-at-least-32-chars",
}
dbInst, err := database.New(fx.Lifecycle(nil), database.Params{
Logger: loggerInst,
Config: cfg,
})
require.NoError(t, err)
svc, err := app.New(fx.Lifecycle(nil), app.ServiceParams{
Logger: loggerInst,
Database: dbInst,
})
require.NoError(t, err)
return &backupTestContext{svc: svc, db: dbInst}
}
// createAppWithFullConfig creates an app with env vars, labels, volumes, and ports.
func createAppWithFullConfig(
t *testing.T,
btc *backupTestContext,
name string,
) *models.App {
t.Helper()
createdApp, err := btc.svc.CreateApp(context.Background(), app.CreateAppInput{
Name: name,
RepoURL: "git@example.com:user/" + name + ".git",
Branch: "develop",
NtfyTopic: "https://ntfy.sh/" + name,
DockerNetwork: "test-network",
})
require.NoError(t, err)
require.NoError(t, btc.svc.AddEnvVar(
context.Background(), createdApp.ID, "DB_HOST", "localhost",
))
require.NoError(t, btc.svc.AddEnvVar(
context.Background(), createdApp.ID, "DB_PORT", "5432",
))
require.NoError(t, btc.svc.AddLabel(
context.Background(), createdApp.ID, "traefik.enable", "true",
))
require.NoError(t, btc.svc.AddVolume(
context.Background(), createdApp.ID, "/data", "/app/data", false,
))
port := models.NewPort(btc.db)
port.AppID = createdApp.ID
port.HostPort = 9090
port.ContainerPort = 8080
port.Protocol = models.PortProtocolTCP
require.NoError(t, port.Save(context.Background()))
return createdApp
}
// createAppWithConfigPort creates an app like createAppWithFullConfig but with
// a custom host port to avoid UNIQUE constraint collisions.
func createAppWithConfigPort(
t *testing.T,
btc *backupTestContext,
name string,
hostPort int,
) *models.App {
t.Helper()
createdApp, err := btc.svc.CreateApp(context.Background(), app.CreateAppInput{
Name: name,
RepoURL: "git@example.com:user/" + name + ".git",
Branch: "develop",
NtfyTopic: "https://ntfy.sh/" + name,
DockerNetwork: "test-network",
})
require.NoError(t, err)
require.NoError(t, btc.svc.AddEnvVar(
context.Background(), createdApp.ID, "DB_HOST", "localhost",
))
require.NoError(t, btc.svc.AddLabel(
context.Background(), createdApp.ID, "traefik.enable", "true",
))
require.NoError(t, btc.svc.AddVolume(
context.Background(), createdApp.ID, "/data2", "/app/data2", false,
))
port := models.NewPort(btc.db)
port.AppID = createdApp.ID
port.HostPort = hostPort
port.ContainerPort = 8080
port.Protocol = models.PortProtocolTCP
require.NoError(t, port.Save(context.Background()))
return createdApp
}
func TestExportApp(t *testing.T) {
t.Parallel()
btc := setupBackupTest(t)
createdApp := createAppWithFullConfig(t, btc, "export-svc-test")
bundle, err := btc.svc.ExportApp(context.Background(), createdApp)
require.NoError(t, err)
assert.Equal(t, 1, bundle.Version)
assert.NotEmpty(t, bundle.ExportedAt)
require.Len(t, bundle.Apps, 1)
ab := bundle.Apps[0]
assert.Equal(t, "export-svc-test", ab.Name)
assert.Equal(t, "develop", ab.Branch)
assert.Equal(t, "test-network", ab.DockerNetwork)
assert.Equal(t, "https://ntfy.sh/export-svc-test", ab.NtfyTopic)
assert.Len(t, ab.EnvVars, 2)
assert.Len(t, ab.Labels, 1)
assert.Len(t, ab.Volumes, 1)
assert.Len(t, ab.Ports, 1)
assert.Equal(t, 9090, ab.Ports[0].HostPort)
assert.Equal(t, 8080, ab.Ports[0].ContainerPort)
assert.Equal(t, "tcp", ab.Ports[0].Protocol)
}
func TestExportAllApps(t *testing.T) {
t.Parallel()
btc := setupBackupTest(t)
createAppWithFullConfig(t, btc, "export-all-1")
createAppWithConfigPort(t, btc, "export-all-2", 9091)
bundle, err := btc.svc.ExportAllApps(context.Background())
require.NoError(t, err)
assert.Equal(t, 1, bundle.Version)
assert.Len(t, bundle.Apps, 2)
}
func TestExportAllAppsEmpty(t *testing.T) {
t.Parallel()
btc := setupBackupTest(t)
bundle, err := btc.svc.ExportAllApps(context.Background())
require.NoError(t, err)
assert.Empty(t, bundle.Apps)
}
func TestImportApps(t *testing.T) {
t.Parallel()
btc := setupBackupTest(t)
bundle := &app.BackupBundle{
Version: 1,
ExportedAt: "2025-01-01T00:00:00Z",
Apps: []app.Backup{
{
Name: "imported-test",
RepoURL: "git@example.com:user/imported.git",
Branch: "main",
DockerfilePath: "Dockerfile",
DockerNetwork: "my-network",
EnvVars: []app.BackupEnvVar{
{Key: "FOO", Value: "bar"},
},
Labels: []app.BackupLabel{
{Key: "app", Value: "test"},
},
Volumes: []app.BackupVolume{
{HostPath: "/host", ContainerPath: "/container", ReadOnly: true},
},
Ports: []app.BackupPort{
{HostPort: 3000, ContainerPort: 8080, Protocol: "tcp"},
},
},
},
}
imported, skipped, err := btc.svc.ImportApps(context.Background(), bundle)
require.NoError(t, err)
assert.Equal(t, []string{"imported-test"}, imported)
assert.Empty(t, skipped)
// Verify the app was created
apps, _ := btc.svc.ListApps(context.Background())
require.Len(t, apps, 1)
assert.Equal(t, "imported-test", apps[0].Name)
assert.True(t, apps[0].DockerNetwork.Valid)
assert.Equal(t, "my-network", apps[0].DockerNetwork.String)
// Has fresh secrets
assert.NotEmpty(t, apps[0].WebhookSecret)
assert.NotEmpty(t, apps[0].SSHPublicKey)
// Verify sub-resources
envVars, _ := apps[0].GetEnvVars(context.Background())
assert.Len(t, envVars, 1)
labels, _ := apps[0].GetLabels(context.Background())
assert.Len(t, labels, 1)
volumes, _ := apps[0].GetVolumes(context.Background())
assert.Len(t, volumes, 1)
assert.True(t, volumes[0].ReadOnly)
ports, _ := apps[0].GetPorts(context.Background())
assert.Len(t, ports, 1)
assert.Equal(t, 3000, ports[0].HostPort)
}
func TestImportAppsSkipsDuplicates(t *testing.T) {
t.Parallel()
btc := setupBackupTest(t)
// Create existing app
_, err := btc.svc.CreateApp(context.Background(), app.CreateAppInput{
Name: "existing",
RepoURL: "git@example.com:user/existing.git",
})
require.NoError(t, err)
bundle := &app.BackupBundle{
Version: 1,
ExportedAt: "2025-01-01T00:00:00Z",
Apps: []app.Backup{
{
Name: "existing",
RepoURL: "git@example.com:user/existing.git",
Branch: "main",
DockerfilePath: "Dockerfile",
EnvVars: []app.BackupEnvVar{},
Labels: []app.BackupLabel{},
Volumes: []app.BackupVolume{},
Ports: []app.BackupPort{},
},
{
Name: "brand-new",
RepoURL: "git@example.com:user/new.git",
Branch: "main",
DockerfilePath: "Dockerfile",
EnvVars: []app.BackupEnvVar{},
Labels: []app.BackupLabel{},
Volumes: []app.BackupVolume{},
Ports: []app.BackupPort{},
},
},
}
imported, skipped, err := btc.svc.ImportApps(context.Background(), bundle)
require.NoError(t, err)
assert.Equal(t, []string{"brand-new"}, imported)
assert.Equal(t, []string{"existing"}, skipped)
}
func TestImportAppsPortDefaultProtocol(t *testing.T) {
t.Parallel()
btc := setupBackupTest(t)
bundle := &app.BackupBundle{
Version: 1,
ExportedAt: "2025-01-01T00:00:00Z",
Apps: []app.Backup{
{
Name: "port-default-test",
RepoURL: "git@example.com:user/repo.git",
Branch: "main",
DockerfilePath: "Dockerfile",
EnvVars: []app.BackupEnvVar{},
Labels: []app.BackupLabel{},
Volumes: []app.BackupVolume{},
Ports: []app.BackupPort{
{HostPort: 80, ContainerPort: 80, Protocol: ""},
},
},
},
}
imported, _, err := btc.svc.ImportApps(context.Background(), bundle)
require.NoError(t, err)
assert.Len(t, imported, 1)
apps, _ := btc.svc.ListApps(context.Background())
ports, _ := apps[0].GetPorts(context.Background())
require.Len(t, ports, 1)
assert.Equal(t, models.PortProtocolTCP, ports[0].Protocol)
}
func TestExportImportRoundTripService(t *testing.T) {
t.Parallel()
btc := setupBackupTest(t)
createAppWithFullConfig(t, btc, "roundtrip-svc")
// Export
bundle, err := btc.svc.ExportAllApps(context.Background())
require.NoError(t, err)
require.Len(t, bundle.Apps, 1)
// Delete original
apps, _ := btc.svc.ListApps(context.Background())
for _, a := range apps {
require.NoError(t, btc.svc.DeleteApp(context.Background(), a))
}
// Import
imported, skipped, err := btc.svc.ImportApps(context.Background(), bundle)
require.NoError(t, err)
assert.Len(t, imported, 1)
assert.Empty(t, skipped)
// Verify round-trip fidelity
restored, _ := btc.svc.ListApps(context.Background())
require.Len(t, restored, 1)
assert.Equal(t, "roundtrip-svc", restored[0].Name)
assert.Equal(t, "develop", restored[0].Branch)
assert.Equal(t, "test-network", restored[0].DockerNetwork.String)
envVars, _ := restored[0].GetEnvVars(context.Background())
assert.Len(t, envVars, 2)
labels, _ := restored[0].GetLabels(context.Background())
assert.Len(t, labels, 1)
volumes, _ := restored[0].GetVolumes(context.Background())
assert.Len(t, volumes, 1)
ports, _ := restored[0].GetPorts(context.Background())
assert.Len(t, ports, 1)
}