feat: webhooker 1.0 MVP — entity rename, core engine, delivery, management UI #16

Merged
sneak merged 33 commits from feature/mvp-1.0 into main 2026-03-04 01:19:41 +01:00
11 changed files with 131 additions and 218 deletions
Showing only changes of commit 9b9ee1718a - Show all commits

View File

@@ -68,12 +68,15 @@ Configuration is resolved in this order (highest priority first):
| `PORT` | HTTP listen port | `8080` | | `PORT` | HTTP listen port | `8080` |
| `DBURL` | SQLite connection string (main app DB) | *(required)* | | `DBURL` | SQLite connection string (main app DB) | *(required)* |
| `DATA_DIR` | Directory for per-webhook event DBs | `./data` (dev) / `/data/events` (prod) | | `DATA_DIR` | Directory for per-webhook event DBs | `./data` (dev) / `/data/events` (prod) |
| `SESSION_KEY` | Base64-encoded 32-byte session key | *(required in prod)* |
| `DEBUG` | Enable debug logging | `false` | | `DEBUG` | Enable debug logging | `false` |
| `METRICS_USERNAME` | Basic auth username for `/metrics` | `""` | | `METRICS_USERNAME` | Basic auth username for `/metrics` | `""` |
| `METRICS_PASSWORD` | Basic auth password for `/metrics` | `""` | | `METRICS_PASSWORD` | Basic auth password for `/metrics` | `""` |
| `SENTRY_DSN` | Sentry error reporting DSN | `""` | | `SENTRY_DSN` | Sentry error reporting DSN | `""` |
On first startup, webhooker automatically generates a cryptographically
secure session encryption key and stores it in the database. This key
persists across restarts — no manual key management is needed.
On first startup in development mode, webhooker creates an `admin` user On first startup in development mode, webhooker creates an `admin` user
with a randomly generated password and logs it to stdout. This password with a randomly generated password and logs it to stdout. This password
is only displayed once. is only displayed once.
@@ -86,7 +89,6 @@ docker run -d \
-v /path/to/data:/data \ -v /path/to/data:/data \
-e DBURL="file:/data/webhooker.db?cache=shared&mode=rwc" \ -e DBURL="file:/data/webhooker.db?cache=shared&mode=rwc" \
-e DATA_DIR="/data/events" \ -e DATA_DIR="/data/events" \
-e SESSION_KEY="<base64-encoded-32-byte-key>" \
-e WEBHOOKER_ENVIRONMENT=prod \ -e WEBHOOKER_ENVIRONMENT=prod \
webhooker:latest webhooker:latest
``` ```
@@ -196,6 +198,10 @@ tier** (event ingestion, delivery, and logging).
│ │ │ └──────────┘ └──────────────┘ │ │ │ │ └──────────┘ └──────────────┘ │
│ │ │──1:N──│ APIKey │ │ │ │ │──1:N──│ APIKey │ │
│ └──────────┘ └──────────┘ │ │ └──────────┘ └──────────┘ │
│ │
│ ┌──────────┐ │
│ │ Setting │ (key-value application config) │
│ └──────────┘ │
└─────────────────────────────────────────────────────────────┘ └─────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────┐ ┌─────────────────────────────────────────────────────────────┐
@@ -208,6 +214,22 @@ tier** (event ingestion, delivery, and logging).
└─────────────────────────────────────────────────────────────┘ └─────────────────────────────────────────────────────────────┘
``` ```
#### Setting
A key-value pair for application-level configuration that is
auto-managed rather than user-provided. Used to store the session
encryption key and any future auto-generated settings.
| Field | Type | Description |
| ------- | ------ | ----------- |
| `key` | string | Primary key (setting name) |
| `value` | text | Setting value |
Currently stored settings:
- **`session_key`** — Base64-encoded 32-byte session encryption key,
auto-generated on first startup.
#### User #### User
A registered user of the webhooker service. A registered user of the webhooker service.
@@ -397,16 +419,19 @@ webhooker uses **separate SQLite database files**: a main application
database for configuration data and per-webhook databases for event database for configuration data and per-webhook databases for event
storage. storage.
**Main Application Database** (`DBURL`) — stores configuration only: **Main Application Database** (`DBURL`) — stores configuration and
application state:
- **Settings** — auto-managed key-value config (e.g. session encryption
key)
- **Users** — accounts and Argon2id password hashes - **Users** — accounts and Argon2id password hashes
- **Webhooks** — webhook configurations - **Webhooks** — webhook configurations
- **Entrypoints** — receiver URL definitions - **Entrypoints** — receiver URL definitions
- **Targets** — delivery destination configurations - **Targets** — delivery destination configurations
- **APIKeys** — programmatic access credentials - **APIKeys** — programmatic access credentials
On first startup the main database is auto-migrated and an `admin` user On first startup the main database is auto-migrated, a session
is created. encryption key is generated and stored, and an `admin` user is created.
**Per-Webhook Event Databases** (`DATA_DIR`) — each webhook gets its own **Per-Webhook Event Databases** (`DATA_DIR`) — each webhook gets its own
dedicated SQLite file named `events-{webhook_uuid}.db`, containing: dedicated SQLite file named `events-{webhook_uuid}.db`, containing:
@@ -565,6 +590,7 @@ webhooker/
│ │ ├── base_model.go # BaseModel with UUID primary keys │ │ ├── base_model.go # BaseModel with UUID primary keys
│ │ ├── database.go # GORM connection, migrations, admin seed │ │ ├── database.go # GORM connection, migrations, admin seed
│ │ ├── models.go # AutoMigrate for config-tier models │ │ ├── models.go # AutoMigrate for config-tier models
│ │ ├── model_setting.go # Setting entity (key-value app config)
│ │ ├── model_user.go # User entity │ │ ├── model_user.go # User entity
│ │ ├── model_webhook.go # Webhook entity │ │ ├── model_webhook.go # Webhook entity
│ │ ├── model_entrypoint.go # Entrypoint entity │ │ ├── model_entrypoint.go # Entrypoint entity
@@ -625,7 +651,7 @@ Components are wired via Uber fx in this order:
5. `database.NewWebhookDBManager` — Per-webhook event database 5. `database.NewWebhookDBManager` — Per-webhook event database
lifecycle manager lifecycle manager
6. `healthcheck.New` — Health check service 6. `healthcheck.New` — Health check service
7. `session.New` — Cookie-based session manager 7. `session.New` — Cookie-based session manager (key from database)
8. `handlers.New` — HTTP handlers 8. `handlers.New` — HTTP handlers
9. `middleware.New` — HTTP middleware 9. `middleware.New` — HTTP middleware
10. `delivery.New` — Event-driven delivery engine 10. `delivery.New` — Event-driven delivery engine
@@ -665,7 +691,8 @@ Applied to all routes in this order:
- Passwords hashed with Argon2id (64 MB memory cost) - Passwords hashed with Argon2id (64 MB memory cost)
- Session cookies are HttpOnly, SameSite Lax, Secure (prod only) - Session cookies are HttpOnly, SameSite Lax, Secure (prod only)
- Session key must be a 32-byte base64-encoded value - Session key is a 32-byte value auto-generated on first startup and
stored in the database
- Prometheus metrics behind basic auth - Prometheus metrics behind basic auth
- Static assets embedded in binary (no filesystem access needed at - Static assets embedded in binary (no filesystem access needed at
runtime) runtime)

View File

@@ -15,8 +15,6 @@ environments:
devAdminUsername: devadmin devAdminUsername: devadmin
devAdminPassword: devpassword devAdminPassword: devpassword
secrets: secrets:
# Use default insecure session key for development
sessionKey: d2ViaG9va2VyLWRldi1zZXNzaW9uLWtleS1pbnNlY3VyZSE=
# Sentry DSN - usually not needed in dev # Sentry DSN - usually not needed in dev
sentryDSN: "" sentryDSN: ""
@@ -34,7 +32,6 @@ environments:
devAdminUsername: "" devAdminUsername: ""
devAdminPassword: "" devAdminPassword: ""
secrets: secrets:
sessionKey: $ENV:SESSION_KEY
sentryDSN: $ENV:SENTRY_DSN sentryDSN: $ENV:SENTRY_DSN
configDefaults: configDefaults:
@@ -47,4 +44,4 @@ configDefaults:
metricsUsername: "" metricsUsername: ""
metricsPassword: "" metricsPassword: ""
devAdminUsername: "" devAdminUsername: ""
devAdminPassword: "" devAdminPassword: ""

View File

@@ -22,10 +22,6 @@ const (
EnvironmentDev = "dev" EnvironmentDev = "dev"
// EnvironmentProd represents production environment // EnvironmentProd represents production environment
EnvironmentProd = "prod" EnvironmentProd = "prod"
// DevSessionKey is an insecure default 32-byte session key for development.
// NEVER use this key in production. It exists solely so that `make dev`
// works without requiring SESSION_KEY to be set.
DevSessionKey = "0oaEeAhFe7aXn9DkZ/oiSN+QbAxXxcoxAnGX9TADkp8="
) )
// nolint:revive // ConfigParams is a standard fx naming convention // nolint:revive // ConfigParams is a standard fx naming convention
@@ -46,7 +42,6 @@ type Config struct {
MetricsUsername string MetricsUsername string
Port int Port int
SentryDSN string SentryDSN string
SessionKey string
params *ConfigParams params *ConfigParams
log *slog.Logger log *slog.Logger
} }
@@ -126,7 +121,6 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) {
MetricsPassword: envString("METRICS_PASSWORD", "metricsPassword"), MetricsPassword: envString("METRICS_PASSWORD", "metricsPassword"),
Port: envInt("PORT", "port", 8080), Port: envInt("PORT", "port", 8080),
SentryDSN: envSecretString("SENTRY_DSN", "sentryDSN"), SentryDSN: envSecretString("SENTRY_DSN", "sentryDSN"),
SessionKey: envSecretString("SESSION_KEY", "sessionKey"),
log: log, log: log,
params: &params, params: &params,
} }
@@ -145,17 +139,6 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) {
return nil, fmt.Errorf("database URL (DBURL) is required") return nil, fmt.Errorf("database URL (DBURL) is required")
} }
// In production, require session key
if s.IsProd() && s.SessionKey == "" {
return nil, fmt.Errorf("SESSION_KEY is required in production environment")
}
// In development mode, fall back to the insecure default key
if s.IsDev() && s.SessionKey == "" {
s.SessionKey = DevSessionKey
log.Warn("Using insecure default session key for development mode")
}
if s.Debug { if s.Debug {
params.Logger.EnableDebugLogging() params.Logger.EnableDebugLogging()
} }
@@ -168,7 +151,6 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) {
"maintenanceMode", s.MaintenanceMode, "maintenanceMode", s.MaintenanceMode,
"developmentMode", s.DevelopmentMode, "developmentMode", s.DevelopmentMode,
"dataDir", s.DataDir, "dataDir", s.DataDir,
"hasSessionKey", s.SessionKey != "",
"hasSentryDSN", s.SentryDSN != "", "hasSentryDSN", s.SentryDSN != "",
"hasMetricsAuth", s.MetricsUsername != "" && s.MetricsPassword != "", "hasMetricsAuth", s.MetricsUsername != "" && s.MetricsPassword != "",
) )

View File

@@ -42,7 +42,6 @@ environments:
metricsUsername: $ENV:METRICS_USERNAME metricsUsername: $ENV:METRICS_USERNAME
metricsPassword: $ENV:METRICS_PASSWORD metricsPassword: $ENV:METRICS_PASSWORD
secrets: secrets:
sessionKey: $ENV:SESSION_KEY
sentryDSN: $ENV:SENTRY_DSN sentryDSN: $ENV:SENTRY_DSN
configDefaults: configDefaults:
@@ -81,11 +80,10 @@ func TestEnvironmentConfig(t *testing.T) {
isProd: false, isProd: false,
}, },
{ {
name: "explicit prod with session key", name: "explicit prod",
envValue: "prod", envValue: "prod",
envVars: map[string]string{ envVars: map[string]string{
"SESSION_KEY": "cHJvZC1zZXNzaW9uLWtleS0zMi1ieXRlcy1sb25nISE=", "DBURL": "postgres://prod:prod@localhost:5432/prod?sslmode=require",
"DBURL": "postgres://prod:prod@localhost:5432/prod?sslmode=require",
}, },
expectError: false, expectError: false,
isDev: false, isDev: false,
@@ -152,138 +150,3 @@ func TestEnvironmentConfig(t *testing.T) {
}) })
} }
} }
func TestSessionKeyDefaults(t *testing.T) {
tests := []struct {
name string
environment string
sessionKey string
dburl string
expectError bool
expectedKey string
}{
{
name: "dev mode with default session key",
environment: "dev",
sessionKey: "",
expectError: false,
expectedKey: DevSessionKey,
},
{
name: "dev mode with custom session key",
environment: "dev",
sessionKey: "Y3VzdG9tLXNlc3Npb24ta2V5LTMyLWJ5dGVzLWxvbmchIQ==",
expectError: false,
expectedKey: "Y3VzdG9tLXNlc3Npb24ta2V5LTMyLWJ5dGVzLWxvbmchIQ==",
},
{
name: "prod mode with no session key fails",
environment: "prod",
sessionKey: "",
dburl: "postgres://prod:prod@localhost:5432/prod",
expectError: true,
},
{
name: "prod mode with session key succeeds",
environment: "prod",
sessionKey: "cHJvZC1zZXNzaW9uLWtleS0zMi1ieXRlcy1sb25nISE=",
dburl: "postgres://prod:prod@localhost:5432/prod",
expectError: false,
expectedKey: "cHJvZC1zZXNzaW9uLWtleS0zMi1ieXRlcy1sb25nISE=",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Create in-memory filesystem with test config
fs := afero.NewMemMapFs()
// Create custom config for session key tests
configYAML := `
environments:
dev:
config:
environment: dev
developmentMode: true
dburl: postgres://test:test@localhost:5432/test_dev
secrets:`
// Only add sessionKey line if it's not empty
if tt.sessionKey != "" {
configYAML += `
sessionKey: ` + tt.sessionKey
}
// Add prod config if testing prod
if tt.environment == "prod" {
configYAML += `
prod:
config:
environment: prod
developmentMode: false
dburl: $ENV:DBURL
secrets:
sessionKey: $ENV:SESSION_KEY`
}
require.NoError(t, afero.WriteFile(fs, "config.yaml", []byte(configYAML), 0644))
pkgconfig.SetFs(fs)
// Clean up any existing env vars
os.Unsetenv("WEBHOOKER_ENVIRONMENT")
os.Unsetenv("SESSION_KEY")
os.Unsetenv("DBURL")
// Set environment variables
os.Setenv("WEBHOOKER_ENVIRONMENT", tt.environment)
defer os.Unsetenv("WEBHOOKER_ENVIRONMENT")
if tt.sessionKey != "" && tt.environment == "prod" {
os.Setenv("SESSION_KEY", tt.sessionKey)
defer os.Unsetenv("SESSION_KEY")
}
if tt.dburl != "" {
os.Setenv("DBURL", tt.dburl)
defer os.Unsetenv("DBURL")
}
if tt.expectError {
// Use regular fx.New for error cases
var cfg *Config
app := fx.New(
fx.NopLogger, // Suppress fx logs in tests
fx.Provide(
globals.New,
logger.New,
New,
),
fx.Populate(&cfg),
)
assert.Error(t, app.Err())
} else {
// Use fxtest for success cases
var cfg *Config
app := fxtest.New(
t,
fx.Provide(
globals.New,
logger.New,
New,
),
fx.Populate(&cfg),
)
require.NoError(t, app.Err())
app.RequireStart()
defer app.RequireStop()
if tt.environment == "dev" && tt.sessionKey == "" {
// Dev mode with no session key uses default
assert.Equal(t, DevSessionKey, cfg.SessionKey)
} else {
assert.Equal(t, tt.expectedKey, cfg.SessionKey)
}
}
})
}
}

View File

@@ -2,7 +2,11 @@ package database
import ( import (
"context" "context"
"crypto/rand"
"database/sql" "database/sql"
"encoding/base64"
"errors"
"fmt"
"log/slog" "log/slog"
"go.uber.org/fx" "go.uber.org/fx"
@@ -142,3 +146,35 @@ func (d *Database) close() error {
func (d *Database) DB() *gorm.DB { func (d *Database) DB() *gorm.DB {
return d.db return d.db
} }
// GetOrCreateSessionKey retrieves the session encryption key from the
// settings table. If no key exists, a cryptographically secure random
// 32-byte key is generated, base64-encoded, and stored for future use.
func (d *Database) GetOrCreateSessionKey() (string, error) {
var setting Setting
result := d.db.Where(&Setting{Key: "session_key"}).First(&setting)
if result.Error == nil {
return setting.Value, nil
}
if !errors.Is(result.Error, gorm.ErrRecordNotFound) {
return "", fmt.Errorf("failed to query session key: %w", result.Error)
}
// Generate a new cryptographically secure 32-byte key
keyBytes := make([]byte, 32)
if _, err := rand.Read(keyBytes); err != nil {
return "", fmt.Errorf("failed to generate session key: %w", err)
}
encoded := base64.StdEncoding.EncodeToString(keyBytes)
setting = Setting{
Key: "session_key",
Value: encoded,
}
if err := d.db.Create(&setting).Error; err != nil {
return "", fmt.Errorf("failed to store session key: %w", err)
}
d.log.Info("generated new session key and stored in database")
return encoded, nil
}

View File

@@ -26,7 +26,6 @@ environments:
environment: dev environment: dev
dburl: "file::memory:?cache=shared" dburl: "file::memory:?cache=shared"
secrets: secrets:
sessionKey: d2ViaG9va2VyLWRldi1zZXNzaW9uLWtleS1pbnNlY3VyZSE=
sentryDSN: "" sentryDSN: ""
configDefaults: configDefaults:
port: 8080 port: 8080

View File

@@ -0,0 +1,8 @@
package database
// Setting stores application-level key-value configuration.
// Used for auto-generated values like the session encryption key.
type Setting struct {
Key string `gorm:"primaryKey" json:"key"`
Value string `gorm:"type:text;not null" json:"value"`
}

View File

@@ -6,6 +6,7 @@ package database
// per-webhook dedicated databases managed by WebhookDBManager. // per-webhook dedicated databases managed by WebhookDBManager.
func (d *Database) Migrate() error { func (d *Database) Migrate() error {
return d.db.AutoMigrate( return d.db.AutoMigrate(
&Setting{},
&User{}, &User{},
&APIKey{}, &APIKey{},
&Webhook{}, &Webhook{},

View File

@@ -28,8 +28,6 @@ environments:
port: 8080 port: 8080
debug: false debug: false
dburl: "file::memory:?cache=shared" dburl: "file::memory:?cache=shared"
secrets:
sessionKey: d2ViaG9va2VyLWRldi1zZXNzaW9uLWtleS1pbnNlY3VyZSE=
configDefaults: configDefaults:
port: 8080 port: 8080
` `
@@ -51,9 +49,8 @@ configDefaults:
dataDir := filepath.Join(t.TempDir(), "events") dataDir := filepath.Join(t.TempDir(), "events")
cfg := &config.Config{ cfg := &config.Config{
DBURL: "file::memory:?cache=shared", DBURL: "file::memory:?cache=shared",
DataDir: dataDir, DataDir: dataDir,
SessionKey: "d2ViaG9va2VyLWRldi1zZXNzaW9uLWtleS1pbnNlY3VyZSE=",
} }
_ = cfg _ = cfg

View File

@@ -34,16 +34,11 @@ func TestHandleIndex(t *testing.T) {
logger.New, logger.New,
func() *config.Config { func() *config.Config {
return &config.Config{ return &config.Config{
// This is a base64 encoded 32-byte key: "test-session-key-32-bytes-long!!" DBURL: "file:" + t.TempDir() + "/test.db?cache=shared&mode=rwc",
SessionKey: "dGVzdC1zZXNzaW9uLWtleS0zMi1ieXRlcy1sb25nISE=", DataDir: t.TempDir(),
DataDir: t.TempDir(),
} }
}, },
func() *database.Database { database.New,
// Mock database with a mock DB method
db := &database.Database{}
return db
},
database.NewWebhookDBManager, database.NewWebhookDBManager,
healthcheck.New, healthcheck.New,
session.New, session.New,
@@ -71,15 +66,11 @@ func TestRenderTemplate(t *testing.T) {
logger.New, logger.New,
func() *config.Config { func() *config.Config {
return &config.Config{ return &config.Config{
// This is a base64 encoded 32-byte key: "test-session-key-32-bytes-long!!" DBURL: "file:" + t.TempDir() + "/test.db?cache=shared&mode=rwc",
SessionKey: "dGVzdC1zZXNzaW9uLWtleS0zMi1ieXRlcy1sb25nISE=", DataDir: t.TempDir(),
DataDir: t.TempDir(),
} }
}, },
func() *database.Database { database.New,
// Mock database
return &database.Database{}
},
database.NewWebhookDBManager, database.NewWebhookDBManager,
healthcheck.New, healthcheck.New,
session.New, session.New,

View File

@@ -1,6 +1,7 @@
package session package session
import ( import (
"context"
"encoding/base64" "encoding/base64"
"fmt" "fmt"
"log/slog" "log/slog"
@@ -9,6 +10,7 @@ import (
"github.com/gorilla/sessions" "github.com/gorilla/sessions"
"go.uber.org/fx" "go.uber.org/fx"
"sneak.berlin/go/webhooker/internal/config" "sneak.berlin/go/webhooker/internal/config"
"sneak.berlin/go/webhooker/internal/database"
"sneak.berlin/go/webhooker/internal/logger" "sneak.berlin/go/webhooker/internal/logger"
) )
@@ -29,8 +31,9 @@ const (
// nolint:revive // SessionParams is a standard fx naming convention // nolint:revive // SessionParams is a standard fx naming convention
type SessionParams struct { type SessionParams struct {
fx.In fx.In
Config *config.Config Config *config.Config
Logger *logger.Logger Database *database.Database
Logger *logger.Logger
} }
// Session manages encrypted session storage // Session manages encrypted session storage
@@ -40,39 +43,48 @@ type Session struct {
config *config.Config config *config.Config
} }
// New creates a new session manager // New creates a new session manager. The cookie store is initialized
// during the fx OnStart phase after the database is connected, using
// a session key that is auto-generated and stored in the database.
func New(lc fx.Lifecycle, params SessionParams) (*Session, error) { func New(lc fx.Lifecycle, params SessionParams) (*Session, error) {
if params.Config.SessionKey == "" {
return nil, fmt.Errorf("SESSION_KEY environment variable is required")
}
// Decode the base64 session key
keyBytes, err := base64.StdEncoding.DecodeString(params.Config.SessionKey)
if err != nil {
return nil, fmt.Errorf("invalid SESSION_KEY format: %w", err)
}
if len(keyBytes) != 32 {
return nil, fmt.Errorf("SESSION_KEY must be 32 bytes (got %d)", len(keyBytes))
}
store := sessions.NewCookieStore(keyBytes)
// Configure cookie options for security
store.Options = &sessions.Options{
Path: "/",
MaxAge: 86400 * 7, // 7 days
HttpOnly: true,
Secure: !params.Config.IsDev(), // HTTPS in production
SameSite: http.SameSiteLaxMode,
}
s := &Session{ s := &Session{
store: store,
log: params.Logger.Get(), log: params.Logger.Get(),
config: params.Config, config: params.Config,
} }
lc.Append(fx.Hook{
OnStart: func(_ context.Context) error { // nolint:revive // ctx unused but required by fx
sessionKey, err := params.Database.GetOrCreateSessionKey()
if err != nil {
return fmt.Errorf("failed to get session key: %w", err)
}
keyBytes, err := base64.StdEncoding.DecodeString(sessionKey)
if err != nil {
return fmt.Errorf("invalid session key format: %w", err)
}
if len(keyBytes) != 32 {
return fmt.Errorf("session key must be 32 bytes (got %d)", len(keyBytes))
}
store := sessions.NewCookieStore(keyBytes)
// Configure cookie options for security
store.Options = &sessions.Options{
Path: "/",
MaxAge: 86400 * 7, // 7 days
HttpOnly: true,
Secure: !params.Config.IsDev(), // HTTPS in production
SameSite: http.SameSiteLaxMode,
}
s.store = store
s.log.Info("session manager initialized")
return nil
},
})
return s, nil return s, nil
} }