Compare commits
3 Commits
93be4d2846
...
c568441dc6
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c568441dc6 | ||
| c3b6623be1 | |||
| 39064a3d6c |
@@ -19,9 +19,14 @@ RUN go mod download
|
|||||||
# .dockerignore.
|
# .dockerignore.
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# Run formatting check and linter
|
# Run formatting check and linter. golangci-lint is invoked directly rather
|
||||||
|
# than through `make lint`: this stage is already the pinned linter image, and
|
||||||
|
# script/lint is a wrapper that builds Dockerfile.lint, so calling it here
|
||||||
|
# would need a docker daemon inside the build. Keep these steps in step with
|
||||||
|
# Dockerfile.lint, including --network=none (see its header for why).
|
||||||
RUN make fmt-check
|
RUN make fmt-check
|
||||||
RUN make lint
|
RUN --network=none golangci-lint config verify --config .golangci.yml
|
||||||
|
RUN --network=none golangci-lint run --config .golangci.yml ./...
|
||||||
|
|
||||||
# Build stage
|
# Build stage
|
||||||
# golang:1.26.1-bookworm (Debian-based), 2026-03-17
|
# golang:1.26.1-bookworm (Debian-based), 2026-03-17
|
||||||
|
|||||||
37
Dockerfile.lint
Normal file
37
Dockerfile.lint
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
# Lint-only image, built by script/lint. golangci-lint is never installed on
|
||||||
|
# the host: the repo is COPYed into the pinned image and linted as a build
|
||||||
|
# step, so a successful build IS a clean lint. This works even when the docker
|
||||||
|
# daemon is remote and bind mounts are impossible.
|
||||||
|
#
|
||||||
|
# script/lint passes --no-cache-filter=lint. Without it an unchanged tree
|
||||||
|
# replays the lint stage from cache and the build succeeds in under a second
|
||||||
|
# having run no linter at all. Do not drop that flag.
|
||||||
|
#
|
||||||
|
# The lint steps run with --network=none. `golangci-lint config verify` is
|
||||||
|
# documented as fetching its JSON schema over HTTPS, which would make linting
|
||||||
|
# depend on an unpinned remote artifact; this pinned image resolves the schema
|
||||||
|
# without any network, and --network=none enforces that rather than trusting
|
||||||
|
# it. It also proves no linter reaches out at analysis time. If a future image
|
||||||
|
# bump makes either step need the network, this build fails loudly instead of
|
||||||
|
# quietly acquiring an unpinned dependency.
|
||||||
|
|
||||||
|
# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-07
|
||||||
|
# Using Debian-based image because mattn/go-sqlite3 (CGO) does not
|
||||||
|
# compile on Alpine musl (off64_t is a glibc type).
|
||||||
|
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS deps
|
||||||
|
|
||||||
|
WORKDIR /src
|
||||||
|
|
||||||
|
# Copy go mod files first for better layer caching. This stage is cacheable;
|
||||||
|
# only the lint stage below is forced to re-execute.
|
||||||
|
COPY go.mod go.sum ./
|
||||||
|
RUN go mod download
|
||||||
|
|
||||||
|
FROM deps AS lint
|
||||||
|
|
||||||
|
COPY . .
|
||||||
|
|
||||||
|
# `run` silently ignores config keys it does not recognize, so a typo would
|
||||||
|
# disable a setting without a word. `config verify` is what catches that.
|
||||||
|
RUN --network=none golangci-lint config verify --config .golangci.yml
|
||||||
|
RUN --network=none golangci-lint run --config .golangci.yml ./...
|
||||||
399
README.md
399
README.md
@@ -11,9 +11,16 @@ with retry support, logging, and observability. Category: infrastructure
|
|||||||
|
|
||||||
### Prerequisites
|
### Prerequisites
|
||||||
|
|
||||||
- Go 1.26+
|
- Go 1.26.1+ (the version in `go.mod`)
|
||||||
- golangci-lint v2.11+
|
- Docker (for linting, for the test stage of the CI gate, and for
|
||||||
- Docker (for containerized deployment)
|
containerized deployment)
|
||||||
|
- `curl`, used by `script/fetch-assets` to download the third-party
|
||||||
|
browser assets, which are not committed (`make bootstrap` installs
|
||||||
|
it if missing)
|
||||||
|
|
||||||
|
golangci-lint is not a prerequisite and must not be installed on the
|
||||||
|
host: `script/bootstrap` does not install it, and `make lint` runs the
|
||||||
|
digest-pinned linter image via `Dockerfile.lint`.
|
||||||
|
|
||||||
### Quick Start
|
### Quick Start
|
||||||
|
|
||||||
@@ -22,14 +29,18 @@ with retry support, logging, and observability. Category: infrastructure
|
|||||||
git clone https://git.eeqj.de/sneak/webhooker.git
|
git clone https://git.eeqj.de/sneak/webhooker.git
|
||||||
cd webhooker
|
cd webhooker
|
||||||
|
|
||||||
# Install Go dependencies
|
# Install Go dependencies and the third-party browser assets.
|
||||||
make deps
|
# `make deps` alone is not enough: it only runs go mod download/tidy,
|
||||||
|
# and the checks below need the fetched assets.
|
||||||
|
make bootstrap
|
||||||
|
|
||||||
# Run all checks (format, lint, test, build)
|
# Run all checks (test, lint, format check)
|
||||||
make check
|
make check
|
||||||
|
|
||||||
# Run in development mode (uses SQLite in current directory)
|
# Run in development mode. DATA_DIR defaults to /var/lib/webhooker in
|
||||||
make dev
|
# every environment, so set it (in .env or the shell) to a writable
|
||||||
|
# directory when running from a clone.
|
||||||
|
DATA_DIR=./data make dev
|
||||||
|
|
||||||
# Build Docker image
|
# Build Docker image
|
||||||
make docker
|
make docker
|
||||||
@@ -42,13 +53,18 @@ make bootstrap # Install all dependencies (idempotent)
|
|||||||
make setup # Bootstrap + install git pre-commit hook
|
make setup # Bootstrap + install git pre-commit hook
|
||||||
make assets # Fetch + verify third-party browser assets
|
make assets # Fetch + verify third-party browser assets
|
||||||
make fmt # Format code (gofmt + goimports)
|
make fmt # Format code (gofmt + goimports)
|
||||||
make lint # Run golangci-lint
|
make fmt-check # Fail if gofmt would change anything (writes nothing)
|
||||||
|
make lint # Run golangci-lint in Docker (Dockerfile.lint)
|
||||||
make test # Run tests with race detection
|
make test # Run tests with race detection
|
||||||
make check # test + lint + fmt-check (CI gate)
|
make check # test + lint + fmt-check (CI gate)
|
||||||
make build # Build binary to bin/webhooker
|
make build # Build binary to bin/webhooker
|
||||||
|
make run # build, then run ./bin/webhooker
|
||||||
make dev # go run ./cmd/webhooker
|
make dev # go run ./cmd/webhooker
|
||||||
|
make deps # go mod download + go mod tidy
|
||||||
make docker # Build Docker image
|
make docker # Build Docker image
|
||||||
make hooks # Install git pre-commit hook that runs script/precommit
|
make hooks # Install git pre-commit hook that runs script/precommit
|
||||||
|
make css # Regenerate static/css/tailwind.css (needs tailwindcss)
|
||||||
|
make clean # Remove bin/
|
||||||
```
|
```
|
||||||
|
|
||||||
### Configuration
|
### Configuration
|
||||||
@@ -90,7 +106,7 @@ TTY detection, and security headers are always applied.
|
|||||||
| `PORT` | HTTP listen port | `8080` |
|
| `PORT` | HTTP listen port | `8080` |
|
||||||
| `DATA_DIR` | Directory for all SQLite databases | `/var/lib/webhooker` |
|
| `DATA_DIR` | Directory for all SQLite databases | `/var/lib/webhooker` |
|
||||||
| `DEBUG` | Enable debug logging | `false` |
|
| `DEBUG` | Enable debug logging | `false` |
|
||||||
| `MAINTENANCE_MODE` | Serve the maintenance page | `false` |
|
| `MAINTENANCE_MODE` | Report `maintenanceMode: true` in the healthcheck JSON. It does not change how any request is served — no maintenance page exists | `false` |
|
||||||
| `METRICS_USERNAME` | Basic auth username for `/metrics` | `""` |
|
| `METRICS_USERNAME` | Basic auth username for `/metrics` | `""` |
|
||||||
| `METRICS_PASSWORD` | Basic auth password for `/metrics` | `""` |
|
| `METRICS_PASSWORD` | Basic auth password for `/metrics` | `""` |
|
||||||
| `SENTRY_DSN` | Sentry error reporting DSN | `""` |
|
| `SENTRY_DSN` | Sentry error reporting DSN | `""` |
|
||||||
@@ -130,8 +146,13 @@ sustained trickle re-locks them immediately.
|
|||||||
|
|
||||||
The remedy is to set `TRUSTED_PROXIES` to your reverse proxy's
|
The remedy is to set `TRUSTED_PROXIES` to your reverse proxy's
|
||||||
address, which restores per-client buckets. webhooker logs a warning
|
address, which restores per-client buckets. webhooker logs a warning
|
||||||
at startup when `WEBHOOKER_ENVIRONMENT=prod` and `TRUSTED_PROXIES` is
|
at startup whenever `TRUSTED_PROXIES` is empty, in every environment —
|
||||||
empty. See [Rate Limiting](#rate-limiting) for what each limit shares.
|
not only when `WEBHOOKER_ENVIRONMENT=prod`, because that variable
|
||||||
|
defaults to `dev` and an operator who never set it is precisely the
|
||||||
|
one at risk. The warning is informational when nothing proxies to the
|
||||||
|
process: with no proxy in front, the peer address is the client's own
|
||||||
|
and the buckets are already per-client. See
|
||||||
|
[Rate Limiting](#rate-limiting) for what each limit shares.
|
||||||
|
|
||||||
`X-Real-IP` and `True-Client-IP` are **never** read, from any peer.
|
`X-Real-IP` and `True-Client-IP` are **never** read, from any peer.
|
||||||
Reverse proxies append to `X-Forwarded-For` but forward other client
|
Reverse proxies append to `X-Forwarded-For` but forward other client
|
||||||
@@ -163,6 +184,8 @@ Two operator requirements follow:
|
|||||||
makes all three limits, including the unauthenticated webhook
|
makes all three limits, including the unauthenticated webhook
|
||||||
receiver, silently bypassable by every client in the block.
|
receiver, silently bypassable by every client in the block.
|
||||||
|
|
||||||
|
#### Sessions
|
||||||
|
|
||||||
Sessions are bounded by two independent clocks, and end at whichever
|
Sessions are bounded by two independent clocks, and end at whichever
|
||||||
one runs out first:
|
one runs out first:
|
||||||
|
|
||||||
@@ -232,17 +255,20 @@ docker run -d \
|
|||||||
The container runs as a non-root user (`webhooker`, UID 1000), exposes
|
The container runs as a non-root user (`webhooker`, UID 1000), exposes
|
||||||
port 8080, and includes a health check against
|
port 8080, and includes a health check against
|
||||||
`/.well-known/healthcheck`. The `/var/lib/webhooker` volume holds all
|
`/.well-known/healthcheck`. The `/var/lib/webhooker` volume holds all
|
||||||
SQLite databases: the main application database (`webhooker.db`) and
|
SQLite databases: the main application database (`webhooker.db`), the
|
||||||
the per-webhook event databases (`events-{uuid}.db`). Mount this as a
|
per-webhook event databases (`events-{uuid}.db`), and any archive
|
||||||
persistent volume to preserve data across container restarts.
|
databases written by `database` targets (`archive-{uuid}.db`). Mount
|
||||||
|
this as a persistent volume to preserve data across container
|
||||||
|
restarts.
|
||||||
|
|
||||||
## Entrypoints
|
## Entrypoints
|
||||||
|
|
||||||
This repository adheres to the
|
This repository adheres to the
|
||||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
||||||
standard: normalized scripts in `script/` are the entrypoints for the
|
standard: normalized scripts in `script/` are the entrypoints for the
|
||||||
development workflow, and the Makefile targets are thin shims that call
|
development workflow. Ten of the Makefile's sixteen targets are thin
|
||||||
them. We provide:
|
shims that call them; `build`, `run`, `dev`, `deps`, `clean` and `css`
|
||||||
|
are inline commands with no script behind them. We provide:
|
||||||
|
|
||||||
- `script/bootstrap` — install all dependencies (idempotent)
|
- `script/bootstrap` — install all dependencies (idempotent)
|
||||||
- `script/setup` — make a fresh clone ready for development
|
- `script/setup` — make a fresh clone ready for development
|
||||||
@@ -251,7 +277,7 @@ them. We provide:
|
|||||||
- `script/fetch-assets` — download the third-party browser assets into
|
- `script/fetch-assets` — download the third-party browser assets into
|
||||||
`static/`, verifying each against its pinned sha256
|
`static/`, verifying each against its pinned sha256
|
||||||
- `script/test` — run the test suite
|
- `script/test` — run the test suite
|
||||||
- `script/lint` — run golangci-lint
|
- `script/lint` — run golangci-lint in Docker (see Linting below)
|
||||||
- `script/fmt` — format all code (writes)
|
- `script/fmt` — format all code (writes)
|
||||||
- `script/fmt-check` — check formatting (read-only)
|
- `script/fmt-check` — check formatting (read-only)
|
||||||
- `script/check` — run test, lint, and fmt-check
|
- `script/check` — run test, lint, and fmt-check
|
||||||
@@ -354,7 +380,11 @@ It uses:
|
|||||||
- **[gorilla/csrf](https://github.com/gorilla/csrf)** for CSRF
|
- **[gorilla/csrf](https://github.com/gorilla/csrf)** for CSRF
|
||||||
protection (cookie-based double-submit tokens)
|
protection (cookie-based double-submit tokens)
|
||||||
- **[go-chi/httprate](https://github.com/go-chi/httprate)** for
|
- **[go-chi/httprate](https://github.com/go-chi/httprate)** for
|
||||||
per-IP login rate limiting (sliding window counter)
|
sliding-window rate limiting of the login, password-change and
|
||||||
|
webhook receiver endpoints. The bucket is per client IP only when
|
||||||
|
`TRUSTED_PROXIES` names the reverse proxy; unset, every client
|
||||||
|
behind that proxy shares one bucket per limit (see
|
||||||
|
[Rate Limiting](#rate-limiting))
|
||||||
- **[Prometheus](https://prometheus.io)** for metrics, served at
|
- **[Prometheus](https://prometheus.io)** for metrics, served at
|
||||||
`/metrics` behind basic auth
|
`/metrics` behind basic auth
|
||||||
- **[Sentry](https://sentry.io)** for optional error reporting
|
- **[Sentry](https://sentry.io)** for optional error reporting
|
||||||
@@ -372,7 +402,7 @@ The codebase uses consistent naming throughout (rename completed in
|
|||||||
|
|
||||||
### Data Model
|
### Data Model
|
||||||
|
|
||||||
webhooker's data model has eight entities organized into two tiers: the
|
webhooker's data model has nine entities organized into two tiers: the
|
||||||
**application tier** (user and webhook configuration) and the **event
|
**application tier** (user and webhook configuration) and the **event
|
||||||
tier** (event ingestion, delivery, and logging).
|
tier** (event ingestion, delivery, and logging).
|
||||||
|
|
||||||
@@ -464,9 +494,25 @@ days (`database.RetentionForeverDays`). The retention reaper recognises
|
|||||||
that sentinel and skips the webhook entirely, and the web UI displays
|
that sentinel and skips the webhook entirely, and the web UI displays
|
||||||
such a webhook's retention as "forever" rather than as a day count.
|
such a webhook's retention as "forever" rather than as a day count.
|
||||||
|
|
||||||
A *finite* retention is capped at `database.MaxFiniteRetentionDays`
|
Submitted `retention_days` values therefore fall into three bands, not
|
||||||
(106751 days, about 292 years), and a larger one is rejected with a
|
two:
|
||||||
400. The cap is not arbitrary: the reaper computes its cutoff as a
|
|
||||||
|
- `1` up to `database.MaxFiniteRetentionDays` (106751 days, about 292
|
||||||
|
years) is accepted as a finite retention.
|
||||||
|
- Above that ceiling but below the retain-forever sentinel of 365000
|
||||||
|
(`database.RetentionForeverDays`) is rejected with a 400. This is the
|
||||||
|
band the cap exists for.
|
||||||
|
- `0`, and `365000` or above, are accepted and mean retain forever,
|
||||||
|
collapsing to the sentinel — `0` in `Webhook.BeforeSave`, the large
|
||||||
|
values in `parseRetentionDays`. The large values are not out of
|
||||||
|
range: the edit form pre-fills the sentinel for a retain-forever
|
||||||
|
webhook, so submitting that form back unchanged has to keep meaning
|
||||||
|
"forever".
|
||||||
|
|
||||||
|
A negative value is in none of the three: `parseRetentionDays` rejects
|
||||||
|
it with a 400 before `BeforeSave` ever sees it.
|
||||||
|
|
||||||
|
The cap is not arbitrary: the reaper computes its cutoff as a
|
||||||
`time.Duration`, an int64 nanosecond count, and a longer period
|
`time.Duration`, an int64 nanosecond count, and a longer period
|
||||||
overflows it. An overflowed cutoff lands in the future, where it
|
overflows it. An overflowed cutoff lands in the future, where it
|
||||||
matches every row, so the sweep would delete every event the webhook
|
matches every row, so the sweep would delete every event the webhook
|
||||||
@@ -484,7 +530,7 @@ the full request and creates an Event.
|
|||||||
| -------------- | ------- | ----------- |
|
| -------------- | ------- | ----------- |
|
||||||
| `id` | UUID | Primary key |
|
| `id` | UUID | Primary key |
|
||||||
| `webhook_id` | UUID | Foreign key → Webhook |
|
| `webhook_id` | UUID | Foreign key → Webhook |
|
||||||
| `path` | string | Unique URL path (UUID-based, e.g. `/webhook/{uuid}`) |
|
| `path` | string | Unique bare UUID, generated at creation. The `/webhook/` prefix is route only and is not stored: the receiver matches this column against the raw `{uuid}` path segment |
|
||||||
| `description` | string | Optional description |
|
| `description` | string | Optional description |
|
||||||
| `active` | boolean | Whether this entrypoint accepts events (default: true) |
|
| `active` | boolean | Whether this entrypoint accepts events (default: true) |
|
||||||
|
|
||||||
@@ -508,8 +554,8 @@ events should be forwarded.
|
|||||||
| `type` | TargetType | One of: `http`, `slack`, `database`, `log` |
|
| `type` | TargetType | One of: `http`, `slack`, `database`, `log` |
|
||||||
| `active` | boolean | Whether deliveries are enabled (default: true) |
|
| `active` | boolean | Whether deliveries are enabled (default: true) |
|
||||||
| `config` | JSON text | Type-specific configuration |
|
| `config` | JSON text | Type-specific configuration |
|
||||||
| `max_retries` | integer | Maximum retry attempts for HTTP targets (0 = fire-and-forget, >0 = retries with backoff) |
|
| `max_retries` | integer | Maximum retry attempts for `http` and `slack` targets (0 = fire-and-forget, >0 = retries with backoff and a circuit breaker). Ignored by `database` and `log` targets |
|
||||||
| `max_queue_size` | integer | Maximum queued deliveries (for HTTP targets with retries) |
|
| `max_queue_size` | integer | Stored and shown on the target's detail view, but not enforced anywhere yet: nothing in the delivery engine consults it. Queue depth is set by the two fixed 10,000-entry channels |
|
||||||
|
|
||||||
**Relations:** Belongs to Webhook. Has many Deliveries.
|
**Relations:** Belongs to Webhook. Has many Deliveries.
|
||||||
|
|
||||||
@@ -522,6 +568,11 @@ events should be forwarded.
|
|||||||
greater than 0, failed deliveries are retried with exponential backoff
|
greater than 0, failed deliveries are retried with exponential backoff
|
||||||
up to `max_retries` attempts, protected by a per-target circuit
|
up to `max_retries` attempts, protected by a per-target circuit
|
||||||
breaker.
|
breaker.
|
||||||
|
- **`slack`** — Post the event as a formatted message to a
|
||||||
|
Slack-compatible incoming webhook URL (`webhookUrl` in `config`). It
|
||||||
|
is built on the same HTTP core as `http` and honours `max_retries`
|
||||||
|
identically, circuit breaker included. See the Slack target section
|
||||||
|
under "Per-Webhook Event Databases" for the message format.
|
||||||
- **`database`** — Archive the full event as a row into a separate
|
- **`database`** — Archive the full event as a row into a separate
|
||||||
per-webhook archive database (`archive-{webhookID}.db`) for long-term
|
per-webhook archive database (`archive-{webhookID}.db`) for long-term
|
||||||
retention, with an optional creation-validated expiry (default: keep
|
retention, with an optional creation-validated expiry (default: keep
|
||||||
@@ -558,7 +609,7 @@ data for auditing and for the planned replay capability.
|
|||||||
| `id` | UUID | Primary key |
|
| `id` | UUID | Primary key |
|
||||||
| `webhook_id` | UUID | Foreign key → Webhook |
|
| `webhook_id` | UUID | Foreign key → Webhook |
|
||||||
| `entrypoint_id` | UUID | Foreign key → Entrypoint |
|
| `entrypoint_id` | UUID | Foreign key → Entrypoint |
|
||||||
| `method` | string | HTTP method (POST, PUT, etc.) |
|
| `method` | string | HTTP method of the captured request. Always `POST`: the receiver answers every other method with 405 before an Event is created |
|
||||||
| `headers` | JSON | Complete request headers |
|
| `headers` | JSON | Complete request headers |
|
||||||
| `body` | text | Raw request body |
|
| `body` | text | Raw request body |
|
||||||
| `content_type` | string | Content-Type header value |
|
| `content_type` | string | Content-Type header value |
|
||||||
@@ -613,7 +664,9 @@ retries) is individually logged for full observability.
|
|||||||
|
|
||||||
#### Common Fields
|
#### Common Fields
|
||||||
|
|
||||||
All entities include these fields from `BaseModel`:
|
Every entity except `Setting` includes these fields from `BaseModel`.
|
||||||
|
`Setting` is a bare key-value row with no `id`, no timestamps and no
|
||||||
|
soft delete:
|
||||||
|
|
||||||
| Field | Type | Description |
|
| Field | Type | Description |
|
||||||
| ------------ | --------- | ----------- |
|
| ------------ | --------- | ----------- |
|
||||||
@@ -659,7 +712,7 @@ handles connection pooling, lazy opening, migrations, and cleanup.
|
|||||||
This separation provides:
|
This separation provides:
|
||||||
|
|
||||||
- **Isolation** — a high-volume webhook won't cause lock contention or
|
- **Isolation** — a high-volume webhook won't cause lock contention or
|
||||||
WAL bloat affecting the main application or other webhooks.
|
journal growth affecting the main application or other webhooks.
|
||||||
- **Independent lifecycle** — event databases can be independently
|
- **Independent lifecycle** — event databases can be independently
|
||||||
backed up, archived, rotated, or size-limited without impacting the
|
backed up, archived, rotated, or size-limited without impacting the
|
||||||
application.
|
application.
|
||||||
@@ -669,9 +722,12 @@ This separation provides:
|
|||||||
- **Per-webhook retention** — the `retention_days` field on each webhook
|
- **Per-webhook retention** — the `retention_days` field on each webhook
|
||||||
controls automatic cleanup of old events in that webhook's database
|
controls automatic cleanup of old events in that webhook's database
|
||||||
only, or disables cleanup entirely when set to `0` (retain forever).
|
only, or disables cleanup entirely when set to `0` (retain forever).
|
||||||
- **Performance** — each webhook's database has its own WAL, its own
|
- **Performance** — each webhook's database has its own page cache and
|
||||||
page cache, and its own lock, so concurrent event ingestion across
|
its own lock, so concurrent event ingestion across webhooks won't
|
||||||
webhooks won't contend.
|
contend. No write-ahead log is involved: both DSNs are
|
||||||
|
`file:{path}?cache=shared&mode=rwc` and no `journal_mode` pragma is
|
||||||
|
ever issued, so every database runs on SQLite's default rollback
|
||||||
|
journal.
|
||||||
|
|
||||||
The **database target type** builds on this architecture to provide
|
The **database target type** builds on this architecture to provide
|
||||||
long-term archiving, separate from the per-webhook event database (which
|
long-term archiving, separate from the per-webhook event database (which
|
||||||
@@ -727,8 +783,9 @@ and other compatible services). Each message includes event metadata
|
|||||||
pretty-printed in a code block. JSON payloads are automatically
|
pretty-printed in a code block. JSON payloads are automatically
|
||||||
formatted with indentation for readability; non-JSON payloads are shown
|
formatted with indentation for readability; non-JSON payloads are shown
|
||||||
as raw text. Large payloads are truncated to keep messages reasonable.
|
as raw text. Large payloads are truncated to keep messages reasonable.
|
||||||
Config stores `webhook_url` — the Slack/Mattermost incoming webhook
|
Config stores `webhookUrl` — the Slack/Mattermost incoming webhook
|
||||||
endpoint.
|
endpoint. That is the JSON key; the error text for a missing one reads
|
||||||
|
`webhook_url is required`, which is the message, not the key.
|
||||||
|
|
||||||
The database uses the
|
The database uses the
|
||||||
[modernc.org/sqlite](https://pkg.go.dev/modernc.org/sqlite) driver at
|
[modernc.org/sqlite](https://pkg.go.dev/modernc.org/sqlite) driver at
|
||||||
@@ -750,8 +807,9 @@ External Service
|
|||||||
1. Look up Entrypoint by UUID
|
1. Look up Entrypoint by UUID
|
||||||
2. Capture full request as Event
|
2. Capture full request as Event
|
||||||
3. Create Delivery records for each active Target
|
3. Create Delivery records for each active Target
|
||||||
4. Build self-contained DeliveryTask structs
|
4. Build self-contained delivery.Task structs
|
||||||
(target config + event data inline for ≤16KB)
|
(target config + event data inline for
|
||||||
|
bodies < 16 KiB)
|
||||||
5. Notify Engine via channel (no DB read needed)
|
5. Notify Engine via channel (no DB read needed)
|
||||||
│
|
│
|
||||||
▼
|
▼
|
||||||
@@ -786,7 +844,7 @@ at any time, preventing goroutine explosions regardless of queue depth.
|
|||||||
a delivery channel (new tasks from the webhook handler) and a retry
|
a delivery channel (new tasks from the webhook handler) and a retry
|
||||||
channel (tasks from backoff timers). Both are buffered to 10,000.
|
channel (tasks from backoff timers). Both are buffered to 10,000.
|
||||||
- **Fan-out via channel, not goroutines:** When an event arrives with
|
- **Fan-out via channel, not goroutines:** When an event arrives with
|
||||||
multiple targets, each `DeliveryTask` is sent to the delivery channel.
|
multiple targets, each `delivery.Task` is sent to the delivery channel.
|
||||||
Workers pick them up and process them — no goroutine-per-target.
|
Workers pick them up and process them — no goroutine-per-target.
|
||||||
- **Worker goroutines:** A fixed number of worker goroutines select from
|
- **Worker goroutines:** A fixed number of worker goroutines select from
|
||||||
both channels. Each worker processes one task at a time, then picks up
|
both channels. Each worker processes one task at a time, then picks up
|
||||||
@@ -810,7 +868,12 @@ This means:
|
|||||||
- **Independent results** — each worker records its own delivery result
|
- **Independent results** — each worker records its own delivery result
|
||||||
in the per-webhook database without coordination.
|
in the per-webhook database without coordination.
|
||||||
- **Graceful shutdown** — cancel the context, workers finish their
|
- **Graceful shutdown** — cancel the context, workers finish their
|
||||||
current task and exit. `WaitGroup.Wait()` ensures clean shutdown.
|
current task and exit. The stop hook waits for the pool via
|
||||||
|
`lifecycle.WaitForShutdown`, which bounds that wait by fx's stop
|
||||||
|
timeout rather than blocking forever on a wedged worker. On timeout
|
||||||
|
it logs at `ERROR` and returns an error, and the goroutines that
|
||||||
|
did not finish are still running — an unclean shutdown is reported
|
||||||
|
rather than hidden.
|
||||||
|
|
||||||
**Recovery paths:**
|
**Recovery paths:**
|
||||||
|
|
||||||
@@ -838,12 +901,13 @@ remains stored in the per-webhook event database, there is no way to
|
|||||||
redeliver it: manual redelivery is planned, not implemented (see
|
redeliver it: manual redelivery is planned, not implemented (see
|
||||||
[TODO.md](TODO.md)).
|
[TODO.md](TODO.md)).
|
||||||
|
|
||||||
### Circuit Breaker (HTTP Targets with Retries)
|
### Circuit Breaker (HTTP and Slack Targets with Retries)
|
||||||
|
|
||||||
HTTP targets with `max_retries` > 0 are protected by a **per-target circuit breaker** that
|
`http` and `slack` targets with `max_retries` > 0 are protected by a
|
||||||
prevents hammering a down target with repeated failed delivery attempts.
|
**per-target circuit breaker** that prevents hammering a down target
|
||||||
The circuit breaker is in-memory only and resets on restart (which is
|
with repeated failed delivery attempts. The circuit breaker is
|
||||||
fine — startup recovery rescans the database anyway).
|
in-memory only and resets on restart (which is fine — startup recovery
|
||||||
|
rescans the database anyway).
|
||||||
|
|
||||||
**States:**
|
**States:**
|
||||||
|
|
||||||
@@ -879,10 +943,12 @@ fine — startup recovery rescans the database anyway).
|
|||||||
- **Failure threshold:** 5 consecutive failures before opening
|
- **Failure threshold:** 5 consecutive failures before opening
|
||||||
- **Cooldown:** 30 seconds in open state before probing
|
- **Cooldown:** 30 seconds in open state before probing
|
||||||
|
|
||||||
**Scope:** Circuit breakers only apply to **HTTP targets with
|
**Scope:** Circuit breakers apply to **`http` and `slack` targets with
|
||||||
`max_retries` > 0**. Fire-and-forget HTTP targets (`max_retries` == 0),
|
`max_retries` > 0**. The Slack target is built on the same HTTP core
|
||||||
Slack targets, database targets (local operations), and log
|
and hands its own `max_retries` to the same retry path, so it gets a
|
||||||
targets (stdout) do not use circuit breakers.
|
breaker with the same 5-failure / 30-second defaults. Fire-and-forget
|
||||||
|
targets of either type (`max_retries` == 0), database targets (local
|
||||||
|
operations), and log targets (stdout) do not use circuit breakers.
|
||||||
|
|
||||||
When a circuit is open and a new delivery arrives, the engine marks the
|
When a circuit is open and a new delivery arrives, the engine marks the
|
||||||
delivery as `retrying` and schedules a retry timer for after the
|
delivery as `retrying` and schedules a retry timer for after the
|
||||||
@@ -898,9 +964,11 @@ unpredictable rates, and blanket limits shared with other routes would
|
|||||||
cause legitimate deliveries to be dropped.
|
cause legitimate deliveries to be dropped.
|
||||||
|
|
||||||
The receiver instead has its own dedicated abuse limit, scoped to the
|
The receiver instead has its own dedicated abuse limit, scoped to the
|
||||||
`/webhook/{uuid}` route only and keyed per client IP per entrypoint: one
|
`/webhook/{uuid}` route only and keyed per client IP per request path
|
||||||
misbehaving sender is throttled without affecting other senders of the
|
(`httprate.KeyByEndpoint`): one misbehaving sender is throttled without
|
||||||
same entrypoint or the same sender's other entrypoints. The limit is
|
affecting other senders of the same entrypoint or the same sender's
|
||||||
|
other entrypoints. Keying on the path rather than on the entrypoint
|
||||||
|
matters — see the aggregate limit below. The limit is
|
||||||
`RECEIVER_RATE_LIMIT` requests per minute (default 120, generous for
|
`RECEIVER_RATE_LIMIT` requests per minute (default 120, generous for
|
||||||
legitimate webhook senders). Requests over the limit receive HTTP 429
|
legitimate webhook senders). Requests over the limit receive HTTP 429
|
||||||
with a `Retry-After` header. A set-but-invalid `RECEIVER_RATE_LIMIT`
|
with a `Retry-After` header. A set-but-invalid `RECEIVER_RATE_LIMIT`
|
||||||
@@ -935,7 +1003,14 @@ Every limiter here — receiver, login, and password change — identifies
|
|||||||
the client the same way, through one shared key function: the
|
the client the same way, through one shared key function: the
|
||||||
connection's own address, unless the peer is listed in
|
connection's own address, unless the peer is listed in
|
||||||
`TRUSTED_PROXIES`, in which case the forwarded client address is used
|
`TRUSTED_PROXIES`, in which case the forwarded client address is used
|
||||||
instead. See [Trusted proxies](#trusted-proxies). Deployed without that
|
instead. That address becomes a bucket by family: IPv4 keys on the full
|
||||||
|
address, IPv6 on its `/64` prefix. A routed `/64` is the normal
|
||||||
|
residential and mobile IPv6 allocation, so keying IPv6 per address would
|
||||||
|
let one subscriber rotate source addresses and mint a fresh bucket per
|
||||||
|
request, evading these limits at the network layer without spoofing
|
||||||
|
anything; the cost is that distinct clients inside one `/64` share a
|
||||||
|
bucket. IPv4-mapped addresses (`::ffff:1.2.3.4`) key as the IPv4 address
|
||||||
|
they carry. See [Trusted proxies](#trusted-proxies). Deployed without that
|
||||||
variable set, a client behind a reverse proxy shares one bucket with
|
variable set, a client behind a reverse proxy shares one bucket with
|
||||||
every other client behind the same proxy. Set `TRUSTED_PROXIES` to the
|
every other client behind the same proxy. Set `TRUSTED_PROXIES` to the
|
||||||
proxy's address to get per-client limits back. What the shared bucket
|
proxy's address to get per-client limits back. What the shared bucket
|
||||||
@@ -958,8 +1033,8 @@ opposite directions:
|
|||||||
login bucket full, and the operator's own login returns HTTP 429 for
|
login bucket full, and the operator's own login returns HTTP 429 for
|
||||||
as long as that trickle continues. A restart clears the in-memory
|
as long as that trickle continues. A restart clears the in-memory
|
||||||
buckets and a resumed trickle re-locks them. Production deployments
|
buckets and a resumed trickle re-locks them. Production deployments
|
||||||
must set `TRUSTED_PROXIES`; webhooker warns at startup when it is
|
must set `TRUSTED_PROXIES`; webhooker warns at startup whenever it is
|
||||||
empty in `prod`.
|
empty, in any environment.
|
||||||
|
|
||||||
Finer-grained per-webhook rate limits (configured in the web UI and
|
Finer-grained per-webhook rate limits (configured in the web UI and
|
||||||
enforced in the webhook handler) can layer on top of this env-level
|
enforced in the webhook handler) can layer on top of this env-level
|
||||||
@@ -971,17 +1046,17 @@ abuse limit later; they are tracked as future work.
|
|||||||
|
|
||||||
| Method | Path | Description |
|
| Method | Path | Description |
|
||||||
| ------ | --------------------------- | ----------- |
|
| ------ | --------------------------- | ----------- |
|
||||||
| `GET` | `/` | Root redirect (authenticated → `/sources`, unauthenticated → `/pages/login`) |
|
| `GET` | `/` | Root redirect, 303 (authenticated → `/sources`, unauthenticated → `/pages/login`) |
|
||||||
| `GET` | `/.well-known/healthcheck` | Health check (JSON: status, uptime, version) |
|
| `GET` | `/.well-known/healthcheck` | Health check (JSON: `status`, `now`, `uptimeSeconds`, `uptimeHuman`, `version`, `appname`, `maintenanceMode`) |
|
||||||
| `GET` | `/s/*` | Static file serving (embedded CSS, JS) |
|
| any | `/s/*` | Static file serving (embedded CSS, JS). Mounted for every method, not just `GET`/`HEAD`: chi's `Mount` registers all methods and `http.FileServer` special-cases only `HEAD` (by omitting the body), so a `POST` or `DELETE` to an asset is answered `200` with the file. Pinned by `TestStaticServesEveryMethod` |
|
||||||
| `ANY` | `/webhook/{uuid}` | Webhook receiver endpoint (accepts all methods) |
|
| `POST` | `/webhook/{uuid}` | Webhook receiver endpoint. `POST` only — every other method is answered `405 Method Not Allowed` with `Allow: POST`. Rate limited (see [Rate Limiting](#rate-limiting)) |
|
||||||
|
|
||||||
#### Authentication Endpoints
|
#### Authentication Endpoints
|
||||||
|
|
||||||
| Method | Path | Description |
|
| Method | Path | Description |
|
||||||
| ------ | --------------- | ----------- |
|
| ------ | --------------- | ----------- |
|
||||||
| `GET` | `/pages/login` | Login page |
|
| `GET` | `/pages/login` | Login page (not rate limited; the limiter applies to POST only) |
|
||||||
| `POST` | `/pages/login` | Login form submission |
|
| `POST` | `/pages/login` | Login form submission (5 per minute per bucket, then 429) |
|
||||||
| `POST` | `/pages/logout` | Logout (destroys session) |
|
| `POST` | `/pages/logout` | Logout (destroys session) |
|
||||||
|
|
||||||
#### Authenticated Endpoints
|
#### Authenticated Endpoints
|
||||||
@@ -989,6 +1064,7 @@ abuse limit later; they are tracked as future work.
|
|||||||
| Method | Path | Description |
|
| Method | Path | Description |
|
||||||
| ------ | ------------------------ | ----------- |
|
| ------ | ------------------------ | ----------- |
|
||||||
| `GET` | `/user/{username}` | User profile page |
|
| `GET` | `/user/{username}` | User profile page |
|
||||||
|
| `POST` | `/user/{username}/password` | Change the user's password (5 per minute per bucket, then 429) |
|
||||||
| `GET` | `/sources` | List user's webhooks |
|
| `GET` | `/sources` | List user's webhooks |
|
||||||
| `GET` | `/sources/new` | Create webhook form |
|
| `GET` | `/sources/new` | Create webhook form |
|
||||||
| `POST` | `/sources/new` | Create webhook submission |
|
| `POST` | `/sources/new` | Create webhook submission |
|
||||||
@@ -998,13 +1074,17 @@ abuse limit later; they are tracked as future work.
|
|||||||
| `POST` | `/source/{id}/delete` | Delete webhook |
|
| `POST` | `/source/{id}/delete` | Delete webhook |
|
||||||
| `GET` | `/source/{id}/logs` | Webhook event logs |
|
| `GET` | `/source/{id}/logs` | Webhook event logs |
|
||||||
| `POST` | `/source/{id}/entrypoints` | Add entrypoint to webhook |
|
| `POST` | `/source/{id}/entrypoints` | Add entrypoint to webhook |
|
||||||
|
| `POST` | `/source/{id}/entrypoints/{entrypointID}/delete` | Delete an entrypoint |
|
||||||
|
| `POST` | `/source/{id}/entrypoints/{entrypointID}/toggle` | Enable or disable an entrypoint |
|
||||||
| `POST` | `/source/{id}/targets` | Add target to webhook |
|
| `POST` | `/source/{id}/targets` | Add target to webhook |
|
||||||
|
| `POST` | `/source/{id}/targets/{targetID}/delete` | Delete a target |
|
||||||
|
| `POST` | `/source/{id}/targets/{targetID}/toggle` | Enable or disable a target |
|
||||||
|
|
||||||
#### Infrastructure Endpoints
|
#### Infrastructure Endpoints
|
||||||
|
|
||||||
| Method | Path | Description |
|
| Method | Path | Description |
|
||||||
| ------ | ---------- | ----------- |
|
| ------ | ---------- | ----------- |
|
||||||
| `GET` | `/metrics` | Prometheus metrics (requires basic auth) |
|
| `GET` | `/metrics` | Prometheus metrics, behind basic auth. The route is registered only when `METRICS_USERNAME` is set; otherwise it does not exist and returns 404 |
|
||||||
|
|
||||||
#### API (Planned)
|
#### API (Planned)
|
||||||
|
|
||||||
@@ -1018,8 +1098,10 @@ abuse limit later; they are tracked as future work.
|
|||||||
| `GET` | `/api/v1/webhooks/{id}/events` | List events for webhook |
|
| `GET` | `/api/v1/webhooks/{id}/events` | List events for webhook |
|
||||||
| `POST` | `/api/v1/events/{id}/redeliver`| Redeliver an event |
|
| `POST` | `/api/v1/events/{id}/redeliver`| Redeliver an event |
|
||||||
|
|
||||||
API authentication will use API keys passed via `Authorization: Bearer
|
None of these exist yet. `/api/v1` is mounted with no routes, so every
|
||||||
<key>` header.
|
path under it returns 404 today. API authentication will use API keys
|
||||||
|
passed via `Authorization: Bearer <key>` header; no Bearer middleware
|
||||||
|
is implemented either.
|
||||||
|
|
||||||
### Package Layout
|
### Package Layout
|
||||||
|
|
||||||
@@ -1047,16 +1129,30 @@ webhooker/
|
|||||||
│ │ ├── model_delivery_result.go # DeliveryResult entity (per-webhook DB)
|
│ │ ├── model_delivery_result.go # DeliveryResult entity (per-webhook DB)
|
||||||
│ │ ├── model_apikey.go # APIKey entity
|
│ │ ├── model_apikey.go # APIKey entity
|
||||||
│ │ ├── password.go # Argon2id hashing and verification
|
│ │ ├── password.go # Argon2id hashing and verification
|
||||||
|
│ │ ├── retention.go # Retention reaper (per-webhook event expiry)
|
||||||
|
│ │ ├── testing.go # NewTestDatabase: wrapper for tests, no fx lifecycle
|
||||||
│ │ └── webhook_db_manager.go # Per-webhook DB lifecycle manager
|
│ │ └── webhook_db_manager.go # Per-webhook DB lifecycle manager
|
||||||
│ ├── globals/
|
│ ├── globals/
|
||||||
│ │ └── globals.go # Build-time variables (appname, version, arch)
|
│ │ └── globals.go # Build-time variables (appname, version, arch)
|
||||||
│ ├── delivery/
|
│ ├── delivery/
|
||||||
│ │ ├── engine.go # Event-driven delivery engine (channel + timer based)
|
│ │ ├── engine.go # Event-driven delivery engine (channel + timer based)
|
||||||
│ │ ├── circuit_breaker.go # Per-target circuit breaker for HTTP targets with retries
|
│ │ ├── circuit_breaker.go # Per-target circuit breaker for http/slack targets with retries
|
||||||
|
│ │ ├── target.go # Target interface, Task, Scheduler
|
||||||
|
│ │ ├── target_http.go # HTTP target (retries, circuit breaker)
|
||||||
|
│ │ ├── target_slack.go # Slack/Mattermost incoming-webhook target
|
||||||
|
│ │ ├── target_database.go # Database archive target
|
||||||
|
│ │ ├── target_database_archive.go # Archive file lifecycle and pruning
|
||||||
|
│ │ ├── target_log.go # Log target (stdout)
|
||||||
|
│ │ ├── target_config_view.go # Masked target config for templates
|
||||||
|
│ │ ├── archive_sweeper.go # Periodic pruning of idle archives
|
||||||
|
│ │ ├── url_mask.go # Strips credentials from *url.Error
|
||||||
│ │ └── ssrf.go # SSRF prevention (IP validation, safe HTTP transport)
|
│ │ └── ssrf.go # SSRF prevention (IP validation, safe HTTP transport)
|
||||||
|
│ ├── lifecycle/
|
||||||
|
│ │ └── lifecycle.go # Shared fx start/stop hook helpers
|
||||||
│ ├── handlers/
|
│ ├── handlers/
|
||||||
│ │ ├── handlers.go # Base handler struct, JSON helpers, template rendering
|
│ │ ├── handlers.go # Base handler struct, JSON helpers, template rendering
|
||||||
│ │ ├── auth.go # Login, logout handlers
|
│ │ ├── auth.go # Login, logout handlers
|
||||||
|
│ │ ├── event_log_view.go # Event log projection, byte-capped in SQL
|
||||||
│ │ ├── healthcheck.go # Health check handler
|
│ │ ├── healthcheck.go # Health check handler
|
||||||
│ │ ├── index.go # Index page handler
|
│ │ ├── index.go # Index page handler
|
||||||
│ │ ├── profile.go # User profile handler
|
│ │ ├── profile.go # User profile handler
|
||||||
@@ -1069,20 +1165,28 @@ webhooker/
|
|||||||
│ ├── middleware/
|
│ ├── middleware/
|
||||||
│ │ ├── middleware.go # Logging, CORS, Auth, Metrics, MetricsAuth, SecurityHeaders, MaxBodySize
|
│ │ ├── middleware.go # Logging, CORS, Auth, Metrics, MetricsAuth, SecurityHeaders, MaxBodySize
|
||||||
│ │ ├── csrf.go # CSRF protection middleware (gorilla/csrf)
|
│ │ ├── csrf.go # CSRF protection middleware (gorilla/csrf)
|
||||||
│ │ └── ratelimit.go # Per-IP rate limiting middleware (go-chi/httprate)
|
│ │ ├── ratelimit.go # Per-IP rate limiting middleware (go-chi/httprate)
|
||||||
|
│ │ └── testing.go # NewForTest: Middleware without the fx lifecycle
|
||||||
│ ├── server/
|
│ ├── server/
|
||||||
│ │ ├── server.go # Server struct, fx lifecycle, signal handling
|
│ │ ├── server.go # Server struct, fx lifecycle, signal handling
|
||||||
│ │ ├── http.go # HTTP server setup with timeouts
|
│ │ ├── http.go # HTTP server setup with timeouts
|
||||||
│ │ └── routes.go # All route definitions
|
│ │ └── routes.go # All route definitions
|
||||||
│ └── session/
|
│ └── session/
|
||||||
│ └── session.go # Cookie-based session management
|
│ ├── session.go # Cookie-based session management
|
||||||
|
│ └── testing.go # NewForTest: Session without the fx lifecycle
|
||||||
├── static/
|
├── static/
|
||||||
│ ├── static.go # //go:embed directive
|
│ ├── static.go # //go:embed directive
|
||||||
│ ├── css/style.css # Custom stylesheet (system font stack, card effects, layout)
|
│ ├── css/input.css # Tailwind input, source for tailwind.css (make css)
|
||||||
│ └── js/app.js # Client-side JavaScript (minimal bootstrap)
|
│ ├── css/tailwind.css # Generated stylesheet the pages load
|
||||||
├── templates/ # Go HTML templates (base, index, login, etc.)
|
│ ├── css/style.css # Older hand-written stylesheet, no longer loaded
|
||||||
├── Dockerfile # Multi-stage: lint, build+test, then Alpine runtime
|
│ ├── js/app.js # Progressive-enhancement copy-to-clipboard
|
||||||
├── Makefile # fmt, lint, test, check, build, docker targets
|
│ ├── js/alpine.min.js # Alpine.js, fetched by script/fetch-assets, not committed
|
||||||
|
│ └── vendor.sha256 # Pinned hashes the fetched assets are verified against
|
||||||
|
├── templates/ # Go HTML templates (base, login, sources, etc.)
|
||||||
|
├── script/ # Scripts to Rule Them All entrypoints
|
||||||
|
├── Dockerfile # Three stages: lint, test+build, Alpine runtime
|
||||||
|
├── Dockerfile.lint # Lint-only image built by script/lint
|
||||||
|
├── Makefile # 10 of 16 targets shim script/; 6 are inline
|
||||||
├── go.mod / go.sum
|
├── go.mod / go.sum
|
||||||
└── .golangci.yml # Linter configuration
|
└── .golangci.yml # Linter configuration
|
||||||
```
|
```
|
||||||
@@ -1098,21 +1202,27 @@ Components are wired via Uber fx in this order:
|
|||||||
user seed
|
user seed
|
||||||
5. `database.NewWebhookDBManager` — Per-webhook event database
|
5. `database.NewWebhookDBManager` — Per-webhook event database
|
||||||
lifecycle manager
|
lifecycle manager
|
||||||
6. `healthcheck.New` — Health check service
|
6. `database.NewRetentionReaper` — Per-webhook event retention sweep
|
||||||
7. `session.New` — Cookie-based session manager (key from database)
|
7. `healthcheck.New` — Health check service
|
||||||
8. `handlers.New` — HTTP handlers
|
8. `session.New` — Cookie-based session manager (key from database)
|
||||||
9. `middleware.New` — HTTP middleware
|
9. `handlers.New` — HTTP handlers
|
||||||
10. `delivery.New` — Event-driven delivery engine
|
10. `middleware.New` — HTTP middleware
|
||||||
11. `delivery.Engine` → `handlers.DeliveryNotifier` — interface bridge
|
11. `delivery.New` — Event-driven delivery engine
|
||||||
12. `server.New` — HTTP server and router
|
12. `delivery.NewArchiveSweeper` — Periodic pruning of idle archives
|
||||||
|
13. `delivery.Engine` → `delivery.Notifier` — interface bridge
|
||||||
|
14. `delivery.Engine` → `delivery.WebhookEvictor` — interface bridge so
|
||||||
|
deleting a webhook releases its archive writer
|
||||||
|
15. `server.New` — HTTP server and router
|
||||||
|
|
||||||
The server starts via `fx.Invoke(func(*server.Server, *delivery.Engine)
|
The server starts via `fx.Invoke(func(*server.Server, *delivery.Engine,
|
||||||
{})` which triggers the fx lifecycle hooks in dependency order. The
|
*database.RetentionReaper, *delivery.ArchiveSweeper) {})`, which
|
||||||
`DeliveryNotifier` interface allows the webhook handler to send
|
triggers the fx lifecycle hooks in dependency order. The
|
||||||
self-contained `DeliveryTask` slices to the engine without a direct
|
`delivery.Notifier` interface allows the webhook handler to send
|
||||||
|
self-contained `delivery.Task` slices to the engine without a direct
|
||||||
package dependency. Each task carries all target config and event data
|
package dependency. Each task carries all target config and event data
|
||||||
inline (for bodies ≤16KB), so the engine can deliver without reading
|
inline (for bodies under 16 KiB, `delivery.MaxInlineBodySize`), so the
|
||||||
from any database — it only writes to record results.
|
engine can deliver without reading from any database — it only writes
|
||||||
|
to record results.
|
||||||
|
|
||||||
### Middleware Stack
|
### Middleware Stack
|
||||||
|
|
||||||
@@ -1138,16 +1248,32 @@ CSRF middleware in every one of those route groups, because
|
|||||||
gorilla/csrf parses the form; if the cap were installed after it, form
|
gorilla/csrf parses the form; if the cap were installed after it, form
|
||||||
parsing would run under net/http's 10 MB default and the 1 MB limit
|
parsing would run under net/http's 10 MB default and the 1 MB limit
|
||||||
would never apply. A request that declares a `Content-Length` over the
|
would never apply. A request that declares a `Content-Length` over the
|
||||||
limit is answered with `413 Request Entity Too Large` before any other
|
limit is answered with `413 Request Entity Too Large` without its body
|
||||||
middleware or handler runs; a chunked request, or one that lies about
|
being read and without reaching CSRF, the route group's remaining
|
||||||
its length, is hard-capped by `http.MaxBytesReader` and fails
|
middleware, or the handler. It is not rejected before *any* other
|
||||||
downstream at form-parse time.
|
middleware, though: the global entries listed above all run first, so
|
||||||
|
such a request is still logged and given the security headers — and
|
||||||
|
counted in the metrics, on a deployment where `METRICS_USERNAME` is
|
||||||
|
set and the Metrics middleware is therefore registered at all. The
|
||||||
|
rejection itself is logged at `WARN` with the method, path and
|
||||||
|
declared length. A chunked request, or
|
||||||
|
one that lies about its length, is hard-capped by
|
||||||
|
`http.MaxBytesReader` and fails downstream at form-parse time.
|
||||||
|
|
||||||
|
Those same four route groups then apply **CSRF** and **NoCache**
|
||||||
|
(`Cache-Control: no-store`, `Pragma: no-cache`), and every group except
|
||||||
|
`/pages` applies **RequireAuth**. The rate limiters are per-route
|
||||||
|
rather than global: **LoginRateLimit** on `/pages/login`,
|
||||||
|
**PasswordChangeRateLimit** on `/user/{username}/password`, and
|
||||||
|
**ReceiverRateLimit** on `/webhook/{uuid}`.
|
||||||
|
|
||||||
### Authentication
|
### Authentication
|
||||||
|
|
||||||
- **Web UI:** Cookie-based sessions using gorilla/sessions with
|
- **Web UI:** Cookie-based sessions using gorilla/sessions with
|
||||||
encrypted cookies. Sessions are configured with HttpOnly, SameSite
|
encrypted cookies. Sessions are configured with HttpOnly, SameSite
|
||||||
Lax, and Secure (in production). Session lifetime is 7 days.
|
Lax, and Secure (in production). Absolute session lifetime is 7 days,
|
||||||
|
with a sliding idle timeout on top of it (see
|
||||||
|
[Sessions](#sessions)).
|
||||||
- **API (planned):** API key authentication via `Authorization: Bearer`
|
- **API (planned):** API key authentication via `Authorization: Bearer`
|
||||||
header. API keys are stored per-user with usage tracking
|
header. API keys are stored per-user with usage tracking
|
||||||
(`last_used_at`).
|
(`last_used_at`).
|
||||||
@@ -1179,33 +1305,79 @@ downstream at form-parse time.
|
|||||||
IPs before connecting, preventing DNS rebinding attacks)
|
IPs before connecting, preventing DNS rebinding attacks)
|
||||||
- **Login rate limiting** via [go-chi/httprate](https://github.com/go-chi/httprate):
|
- **Login rate limiting** via [go-chi/httprate](https://github.com/go-chi/httprate):
|
||||||
sliding-window rate limiter on the login endpoint, 5 POST attempts
|
sliding-window rate limiter on the login endpoint, 5 POST attempts
|
||||||
per minute per bucket, to slow brute-force attacks. The bucket is per
|
per minute per bucket, to slow brute-force attacks. GET requests to
|
||||||
client IP only when `TRUSTED_PROXIES` names the reverse proxy;
|
the login page are not limited. The password-change endpoint carries
|
||||||
unset, every client shares one bucket and the login becomes remotely
|
the same 5-per-minute limit. The bucket is per client IP only when
|
||||||
deniable (see [Rate Limiting](#rate-limiting))
|
`TRUSTED_PROXIES` names the reverse proxy; unset, every client
|
||||||
|
shares one bucket and the login becomes remotely deniable (see
|
||||||
|
[Rate Limiting](#rate-limiting)). webhooker warns at startup
|
||||||
|
whenever `TRUSTED_PROXIES` is empty
|
||||||
- Prometheus metrics behind basic auth
|
- Prometheus metrics behind basic auth
|
||||||
- Static assets embedded in binary (no filesystem access needed at
|
- Static assets embedded in binary (no filesystem access needed at
|
||||||
runtime)
|
runtime)
|
||||||
- Container runs as non-root user (UID 1000)
|
- Container runs as non-root user (UID 1000)
|
||||||
- GORM soft deletes on all entities (data preserved for audit)
|
- GORM soft deletes on every entity that carries `BaseModel`, which is
|
||||||
|
all of them but `Setting` (data preserved for audit)
|
||||||
|
|
||||||
|
### Linting
|
||||||
|
|
||||||
|
golangci-lint never runs on the host. `script/lint` builds
|
||||||
|
`Dockerfile.lint`, which copies the repo into the digest-pinned
|
||||||
|
golangci-lint image and lints as a build step, so a successful build is
|
||||||
|
a clean lint. A host binary would share one cache and one lock with
|
||||||
|
every other checkout on the machine, which has produced both invented
|
||||||
|
findings attributed to other worktrees and unearned passes.
|
||||||
|
|
||||||
|
Two properties are load-bearing:
|
||||||
|
|
||||||
|
- `script/lint` passes `--no-cache-filter=lint`. Without it an unchanged
|
||||||
|
tree replays the lint layer from cache and the build exits 0 in under
|
||||||
|
a second having linted nothing. The `deps` stage stays cacheable, so
|
||||||
|
module downloads are not repeated. Invalidation is scoped to the one
|
||||||
|
stage; never prune the shared build cache.
|
||||||
|
- Both lint steps use `RUN --network=none`. `golangci-lint config
|
||||||
|
verify` is documented as fetching its JSON schema over HTTPS, which
|
||||||
|
would be an unpinned remote dependency; the pinned image resolves the
|
||||||
|
schema without network access, and `--network=none` enforces that
|
||||||
|
instead of trusting it. Verify is worth keeping because
|
||||||
|
`golangci-lint run` silently ignores config keys it does not
|
||||||
|
recognize, so a typo would disable a setting with no warning.
|
||||||
|
|
||||||
### Docker
|
### Docker
|
||||||
|
|
||||||
The Dockerfile uses a multi-stage build:
|
The Dockerfile uses a three-stage build. Each stage is pinned by
|
||||||
|
digest, and the two check stages are separate images so the linter's
|
||||||
|
version is fixed independently of the compiler's:
|
||||||
|
|
||||||
1. **Builder stage** (Debian-based `golang:1.24`) — installs
|
1. **Lint stage** (`golangci/golangci-lint:v2.12.2`, Debian-based) —
|
||||||
golangci-lint, downloads dependencies, copies source, runs `make
|
installs `make`, downloads dependencies, copies the source, and runs
|
||||||
check` (format verification, linting, tests, compilation).
|
`make fmt-check`, then `golangci-lint config verify` and
|
||||||
2. **Runtime stage** (`alpine:3.21`) — copies the binary, creates the
|
`golangci-lint run`, both with `--network=none`.
|
||||||
`/var/lib/webhooker` directory for all SQLite databases, runs as
|
2. **Builder stage** (`golang:1.26.1-bookworm`) — depends on the lint
|
||||||
non-root user, exposes port 8080, includes a health check.
|
stage passing (it copies a file from it), runs `script/fetch-assets`
|
||||||
|
to download and verify the third-party browser assets, then runs
|
||||||
|
`make test` and `make build`, and finally rebuilds the binary with
|
||||||
|
`CGO_ENABLED=1` and static linking so it runs on musl.
|
||||||
|
3. **Runtime stage** (`alpine:3.21`) — copies the static binary,
|
||||||
|
creates the `/var/lib/webhooker` directory for all SQLite databases,
|
||||||
|
runs as the non-root `webhooker` user (UID 1000), exposes port 8080,
|
||||||
|
and includes a health check against `/.well-known/healthcheck`.
|
||||||
|
|
||||||
The builder uses Debian rather than Alpine because GORM's SQLite
|
The lint stage invokes `golangci-lint` directly rather than `make lint`:
|
||||||
dialect pulls in CGO-dependent headers at compile time. The runtime
|
it is already the pinned linter image, and `make lint` builds
|
||||||
binary is statically linked and runs on Alpine.
|
`Dockerfile.lint`, which would need a docker daemon inside this build.
|
||||||
|
|
||||||
`docker build .` is the CI gate — if it passes, the code is formatted,
|
Both check stages use Debian rather than Alpine because
|
||||||
linted, tested, and compiled.
|
`gorm.io/driver/sqlite` pulls in `mattn/go-sqlite3`, which needs CGO
|
||||||
|
and does not compile against musl. Only the final binary is statically
|
||||||
|
linked, which is what lets it run on the Alpine runtime image.
|
||||||
|
|
||||||
|
`script/cibuild` — `docker build .` — is the CI gate: the four check
|
||||||
|
targets run inside the image, so a build that succeeds is a repo that
|
||||||
|
is formatted, linted, tested and compiled. `script/lint` also uses
|
||||||
|
Docker (`Dockerfile.lint`, see Linting above), so `make lint` and
|
||||||
|
`make check` run the same pinned linter version the gate does; only
|
||||||
|
`script/test` and `script/fmt-check` run on the host.
|
||||||
|
|
||||||
#### CI gate honesty
|
#### CI gate honesty
|
||||||
|
|
||||||
@@ -1218,19 +1390,20 @@ the hash of the last commit that touched the build context, so:
|
|||||||
- Any commit that changes code (including a squash merge whose tree
|
- Any commit that changes code (including a squash merge whose tree
|
||||||
matches an already-built branch) gets a new fingerprint, invalidates
|
matches an already-built branch) gets a new fingerprint, invalidates
|
||||||
the `COPY . .` layer of both check stages, and really runs
|
the `COPY . .` layer of both check stages, and really runs
|
||||||
`make fmt-check`, `make lint`, `make test`, and `make build`. A run
|
`make fmt-check`, `golangci-lint`, `make test`, and `make build`. A
|
||||||
that reports success ran them.
|
run that reports success ran them.
|
||||||
- A docs-only commit leaves the fingerprint unchanged — `.dockerignore`
|
- A docs-only commit leaves the fingerprint unchanged — `.dockerignore`
|
||||||
excludes `*.md` and `LICENSE` from the context anyway — so the image
|
excludes `*.md`, `LICENSE` and `.editorconfig` from the context
|
||||||
replays from cache and costs seconds.
|
anyway — so the image replays from cache and costs seconds.
|
||||||
|
|
||||||
The module download layer sits above `COPY . .` and stays cached either
|
The module download layer sits above `COPY . .` and stays cached either
|
||||||
way.
|
way.
|
||||||
|
|
||||||
The workflow's first step covers a second way the gate lied: Gitea
|
A separate workflow step, run before the fingerprint is written, covers
|
||||||
cancels an in-flight run when a newer commit lands on the same branch
|
a second way the gate lied: Gitea cancels an in-flight run when a newer
|
||||||
and records that cancellation as a `failure` status, marking a commit
|
commit lands on the same branch and records that cancellation as a
|
||||||
red that was never tested. Cancellation is unconditional server-side for
|
`failure` status, marking a commit red that was never tested.
|
||||||
|
Cancellation is unconditional server-side for
|
||||||
push events, so the superseding run rewrites the exact
|
push events, so the superseding run rewrites the exact
|
||||||
`Has been cancelled` status to `skipped`. Genuine failures are never
|
`Has been cancelled` status to `skipped`. Genuine failures are never
|
||||||
touched.
|
touched.
|
||||||
|
|||||||
22
TODO.md
22
TODO.md
@@ -25,13 +25,17 @@ password change flow (#65), policy compliance (#6), pinned lint tooling
|
|||||||
(#55), and fail-loud configuration parsing (#80).
|
(#55), and fail-loud configuration parsing (#80).
|
||||||
|
|
||||||
`next` holds the completed 1.0.0 milestone: every issue in it is closed,
|
`next` holds the completed 1.0.0 milestone: every issue in it is closed,
|
||||||
and it is verified green both by CI and by cache-defeated container
|
and it is verified green by cache-defeated container runs
|
||||||
runs. The two were only made to mean the same thing this cycle — before
|
(`docker build --no-cache-filter=lint --no-cache-filter=builder`). The
|
||||||
#119, a warm layer cache let the gate report success without executing
|
CI status is not independently claimed here: a superseded run is
|
||||||
anything, and replayed the previous build's console log so the lie
|
recorded as `skipped` and still rolls up green, so a commit status on
|
||||||
looked like a real run. Note: TODO.md was deliberately deleted from this
|
`next` does not by itself evidence an executed check (#152). Before
|
||||||
repo in f9a9569 (2026-03-01, #6); its content was folded into the README
|
#119, a warm layer cache also let the gate report success without
|
||||||
TODO section, which this draft reconstructs as of 2026-07-06.
|
executing anything, and replayed the previous build's console log so
|
||||||
|
the lie looked like a real run. Note: `TODO.md` was deliberately
|
||||||
|
deleted from this repo in f9a9569 (2026-03-01, #6); its content was
|
||||||
|
folded into the README TODO section, which this draft reconstructs as
|
||||||
|
of 2026-07-06.
|
||||||
|
|
||||||
# Next Step
|
# Next Step
|
||||||
|
|
||||||
@@ -190,7 +194,9 @@ rate-limit keys should bucket by `/64`).
|
|||||||
- OpenAPI specification
|
- OpenAPI specification
|
||||||
- Analytics dashboard: success rates, response times, volume
|
- Analytics dashboard: success rates, response times, volume
|
||||||
- A remember-me option at login
|
- A remember-me option at login
|
||||||
- Password change and reset flow
|
- Password reset flow for a forgotten password. The authenticated
|
||||||
|
password *change* flow already landed on `main` (#65); reset does not
|
||||||
|
exist
|
||||||
- Later, nice to have
|
- Later, nice to have
|
||||||
- email delivery target type
|
- email delivery target type
|
||||||
- SNS and S3 delivery targets
|
- SNS and S3 delivery targets
|
||||||
|
|||||||
@@ -422,33 +422,43 @@ func loadFromEnv() (*Config, error) {
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// warnSharedRateLimitBucket logs a startup warning when a production
|
// warnSharedRateLimitBucket logs a startup warning whenever
|
||||||
// deployment leaves TRUSTED_PROXIES empty.
|
// TRUSTED_PROXIES is empty, in any environment.
|
||||||
//
|
//
|
||||||
// With no trusted proxies every rate limiter keys on the connecting
|
// With no trusted proxies every rate limiter keys on the connecting
|
||||||
// peer's address. A production deployment is required to run behind a
|
// peer's address. Whether that is harmless or dangerous depends on
|
||||||
// TLS-terminating reverse proxy, and the peer is then that proxy for
|
// what is in front of the process, which this code cannot observe:
|
||||||
// every request, so all clients share one bucket per limiter. The
|
// with nothing in front, the peer is the client and the limits are
|
||||||
|
// per-client as intended; behind a reverse proxy the peer is the proxy
|
||||||
|
// for every request, so all clients share one bucket per limiter. The
|
||||||
// login limiter's bucket is the dangerous one: any remote client can
|
// login limiter's bucket is the dangerous one: any remote client can
|
||||||
// keep it full, which denies the only administrative login to
|
// keep it full, which denies the only administrative login to everyone
|
||||||
// everyone until the process restarts.
|
// until the process restarts.
|
||||||
|
//
|
||||||
|
// The warning is deliberately not gated on WEBHOOKER_ENVIRONMENT. That
|
||||||
|
// variable defaults to dev, so gating on it would silence the warning
|
||||||
|
// for exactly the operator who forgot to configure the deployment —
|
||||||
|
// the case it exists to catch.
|
||||||
//
|
//
|
||||||
// The default of trusting nobody is deliberate — trusting forwarded
|
// The default of trusting nobody is deliberate — trusting forwarded
|
||||||
// headers from arbitrary peers lets any client choose its own bucket —
|
// headers from arbitrary peers lets any client choose its own bucket —
|
||||||
// so this warns rather than failing startup or changing the key.
|
// so this warns rather than failing startup or changing the key.
|
||||||
func (c *Config) warnSharedRateLimitBucket(log *slog.Logger) {
|
func (c *Config) warnSharedRateLimitBucket(log *slog.Logger) {
|
||||||
if !c.IsProd() || len(c.TrustedProxies) > 0 {
|
if len(c.TrustedProxies) > 0 {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Warn(
|
log.Warn(
|
||||||
"TRUSTED_PROXIES is empty: rate limits key on the "+
|
"TRUSTED_PROXIES is empty: every rate limit keys on the "+
|
||||||
"connecting peer, so behind the reverse proxy a "+
|
"connecting peer's address. With nothing proxying to "+
|
||||||
"production deployment runs behind, every client "+
|
"this process that is the client itself and the limits "+
|
||||||
"shares one bucket per limit. Any remote client can "+
|
"are per-client as intended. Behind a reverse proxy the "+
|
||||||
"then keep the login limit full and deny the admin "+
|
"peer is the proxy on every request, so all clients "+
|
||||||
"login, the only administrative path, until restart. "+
|
"share one bucket per limit and any remote client can "+
|
||||||
"Set TRUSTED_PROXIES to your reverse proxy's address.",
|
"keep the login limit full, denying the admin login — "+
|
||||||
|
"the only administrative path — until restart. If "+
|
||||||
|
"anything proxies to this process, set TRUSTED_PROXIES "+
|
||||||
|
"to its address.",
|
||||||
"environment", c.Environment,
|
"environment", c.Environment,
|
||||||
"trustedProxies", len(c.TrustedProxies),
|
"trustedProxies", len(c.TrustedProxies),
|
||||||
)
|
)
|
||||||
@@ -491,6 +501,10 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) {
|
|||||||
"maintenanceMode", s.MaintenanceMode,
|
"maintenanceMode", s.MaintenanceMode,
|
||||||
"dataDir", s.DataDir,
|
"dataDir", s.DataDir,
|
||||||
"retentionSweepInterval", s.RetentionSweepInterval.String(),
|
"retentionSweepInterval", s.RetentionSweepInterval.String(),
|
||||||
|
// Logged because a perfectly valid non-positive value here
|
||||||
|
// disables idle expiry entirely, and that is worth showing
|
||||||
|
// back to the operator.
|
||||||
|
"sessionIdleTimeout", s.SessionIdleTimeout.String(),
|
||||||
"receiverRateLimit", s.ReceiverRateLimit,
|
"receiverRateLimit", s.ReceiverRateLimit,
|
||||||
"trustedProxies", len(s.TrustedProxies),
|
"trustedProxies", len(s.TrustedProxies),
|
||||||
"hasSentryDSN", s.SentryDSN != "",
|
"hasSentryDSN", s.SentryDSN != "",
|
||||||
|
|||||||
@@ -628,10 +628,12 @@ func testTrustedProxiesSuccess(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestSharedRateLimitBucketWarning covers the startup warning that
|
// TestSharedRateLimitBucketWarning covers the startup warning that
|
||||||
// tells an operator their production deployment shares one rate-limit
|
// tells an operator a deployment behind a reverse proxy shares one
|
||||||
// bucket between every client, which makes the admin login remotely
|
// rate-limit bucket between every client, which makes the admin login
|
||||||
// deniable. It must fire when TRUSTED_PROXIES is empty in production
|
// remotely deniable. It must fire whenever TRUSTED_PROXIES is empty,
|
||||||
// and stay quiet otherwise.
|
// in any environment: WEBHOOKER_ENVIRONMENT defaults to dev, so gating
|
||||||
|
// on it would silence the warning for exactly the operator who never
|
||||||
|
// configured the deployment. It stays quiet once proxies are named.
|
||||||
func TestSharedRateLimitBucketWarning(t *testing.T) {
|
func TestSharedRateLimitBucketWarning(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -651,11 +653,18 @@ func TestSharedRateLimitBucketWarning(t *testing.T) {
|
|||||||
expectWarning: false,
|
expectWarning: false,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
// Development is not required to run behind a
|
// The default environment. An internet-exposed
|
||||||
// reverse proxy, so the shared bucket the warning
|
// deployment whose operator never set
|
||||||
// describes is not the expected shape there.
|
// WEBHOOKER_ENVIRONMENT lands here and has exactly
|
||||||
name: "dev without trusted proxies is quiet",
|
// the exposure the warning announces.
|
||||||
|
name: "dev without trusted proxies warns",
|
||||||
environment: config.EnvironmentDev,
|
environment: config.EnvironmentDev,
|
||||||
|
expectWarning: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "dev with trusted proxies is quiet",
|
||||||
|
environment: config.EnvironmentDev,
|
||||||
|
trustedProxies: cidrPrivateV4,
|
||||||
expectWarning: false,
|
expectWarning: false,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -697,8 +706,14 @@ func TestSharedRateLimitBucketWarning(t *testing.T) {
|
|||||||
|
|
||||||
assert.Contains(t, logged, `"level":"WARN"`)
|
assert.Contains(t, logged, `"level":"WARN"`)
|
||||||
assert.Contains(t, logged, "TRUSTED_PROXIES")
|
assert.Contains(t, logged, "TRUSTED_PROXIES")
|
||||||
assert.Contains(t, logged, "shares one bucket")
|
assert.Contains(t, logged, "share one bucket")
|
||||||
assert.Contains(t, logged, "deny the admin login")
|
assert.Contains(t, logged, "denying the admin login")
|
||||||
|
// The text must stay accurate for a developer with
|
||||||
|
// nothing in front of the process, where an empty
|
||||||
|
// list costs nothing.
|
||||||
|
assert.Contains(
|
||||||
|
t, logged, "nothing proxying to this process",
|
||||||
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -48,6 +48,12 @@ const (
|
|||||||
// bound every request pays a walk proportional to whatever the
|
// bound every request pays a walk proportional to whatever the
|
||||||
// client sent.
|
// client sent.
|
||||||
maxForwardedHops = 64
|
maxForwardedHops = 64
|
||||||
|
|
||||||
|
// ipv6BucketBits is the prefix length IPv6 clients are bucketed
|
||||||
|
// on. A routed /64 is the normal residential and mobile
|
||||||
|
// allocation, so it is the unit an attacker gets addresses in
|
||||||
|
// and therefore the unit worth limiting.
|
||||||
|
ipv6BucketBits = 64
|
||||||
)
|
)
|
||||||
|
|
||||||
// normalizeAddr strips the IPv4-in-IPv6 wrapper and any zone from
|
// normalizeAddr strips the IPv4-in-IPv6 wrapper and any zone from
|
||||||
@@ -56,6 +62,40 @@ func normalizeAddr(addr netip.Addr) netip.Addr {
|
|||||||
return addr.Unmap().WithZone("")
|
return addr.Unmap().WithZone("")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// bucketKey is the rate-limit bucket identity of a client address.
|
||||||
|
// IPv4 keys on the full address; IPv6 keys on its /64 prefix,
|
||||||
|
// because keying IPv6 per /128 lets one ordinary subscriber rotate
|
||||||
|
// source addresses inside its own routed /64 and mint a fresh bucket
|
||||||
|
// per request — evading every limiter here at the network layer,
|
||||||
|
// with no spoofing and nothing to detect.
|
||||||
|
//
|
||||||
|
// An IPv4-mapped address (::ffff:1.2.3.4) is keyed as the IPv4
|
||||||
|
// address it carries, never masked to a /64: mapped form all shares
|
||||||
|
// the ::ffff:0:0/96 prefix, so masking would collapse every IPv4
|
||||||
|
// client reaching a proxy that emits it into one bucket. Callers
|
||||||
|
// pass addresses through normalizeAddr, which already unmaps; the
|
||||||
|
// unmap here keeps the property true of the key function itself.
|
||||||
|
//
|
||||||
|
// The two families cannot collide: an IPv4 key is a bare dotted
|
||||||
|
// quad, and an IPv6 key always carries a "/64" suffix.
|
||||||
|
func bucketKey(addr netip.Addr) string {
|
||||||
|
addr = addr.Unmap()
|
||||||
|
|
||||||
|
if addr.Is4() {
|
||||||
|
return addr.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prefix errors only on a negative bit count, on over 32 bits
|
||||||
|
// for an IPv4 address, or on over 128 for IPv6. The count here
|
||||||
|
// is the constant 64 and the IPv4 case returned above, so the
|
||||||
|
// error is unreachable. (The zero Addr does not error either: it
|
||||||
|
// yields the zero Prefix. Neither call site can produce one,
|
||||||
|
// since both parse the address first.)
|
||||||
|
prefix, _ := addr.Prefix(ipv6BucketBits)
|
||||||
|
|
||||||
|
return prefix.String()
|
||||||
|
}
|
||||||
|
|
||||||
// isTrustedProxy reports whether addr belongs to a network the
|
// isTrustedProxy reports whether addr belongs to a network the
|
||||||
// operator listed in TRUSTED_PROXIES. The list is empty by default,
|
// operator listed in TRUSTED_PROXIES. The list is empty by default,
|
||||||
// so by default nothing is trusted.
|
// so by default nothing is trusted.
|
||||||
@@ -143,6 +183,9 @@ func (m *Middleware) forwardedClientAddr(
|
|||||||
// another client's bucket, by picking an X-Forwarded-For value —
|
// another client's bucket, by picking an X-Forwarded-For value —
|
||||||
// which makes every limit here decorative against a deliberate
|
// which makes every limit here decorative against a deliberate
|
||||||
// attacker.
|
// attacker.
|
||||||
|
//
|
||||||
|
// The address that identifies the client is then reduced to a bucket
|
||||||
|
// by bucketKey: full address for IPv4, /64 prefix for IPv6.
|
||||||
func (m *Middleware) rateLimitKey(r *http.Request) (string, error) {
|
func (m *Middleware) rateLimitKey(r *http.Request) (string, error) {
|
||||||
return m.clientKey(r), nil
|
return m.clientKey(r), nil
|
||||||
}
|
}
|
||||||
@@ -152,23 +195,25 @@ func (m *Middleware) clientKey(r *http.Request) string {
|
|||||||
peer, err := netip.ParseAddr(ipFromHostPort(r.RemoteAddr))
|
peer, err := netip.ParseAddr(ipFromHostPort(r.RemoteAddr))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Not an address we can reason about; key on the raw
|
// Not an address we can reason about; key on the raw
|
||||||
// value, the most specific identity left. On a
|
// value, the most specific identity left. Distinct
|
||||||
// Unix-socket listener every peer carries the same
|
// RemoteAddr values stay in distinct buckets, so this
|
||||||
// RemoteAddr and so shares one bucket, which is the
|
// path cannot silently collapse unrelated clients
|
||||||
// fail-closed direction.
|
// together. On a Unix-socket listener every peer
|
||||||
|
// carries the same RemoteAddr and so shares one bucket,
|
||||||
|
// which is the fail-closed direction.
|
||||||
return r.RemoteAddr
|
return r.RemoteAddr
|
||||||
}
|
}
|
||||||
|
|
||||||
peer = normalizeAddr(peer)
|
peer = normalizeAddr(peer)
|
||||||
if !m.isTrustedProxy(peer) {
|
if !m.isTrustedProxy(peer) {
|
||||||
return peer.String()
|
return bucketKey(peer)
|
||||||
}
|
}
|
||||||
|
|
||||||
if addr, ok := m.forwardedClientAddr(r); ok {
|
if addr, ok := m.forwardedClientAddr(r); ok {
|
||||||
return addr.String()
|
return bucketKey(addr)
|
||||||
}
|
}
|
||||||
|
|
||||||
return peer.String()
|
return bucketKey(peer)
|
||||||
}
|
}
|
||||||
|
|
||||||
// tooManyRequests returns the 429 handler used by the login,
|
// tooManyRequests returns the 429 handler used by the login,
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/middleware"
|
"sneak.berlin/go/webhooker/internal/middleware"
|
||||||
)
|
)
|
||||||
@@ -370,6 +371,30 @@ const (
|
|||||||
headerXFF = "X-Forwarded-For"
|
headerXFF = "X-Forwarded-For"
|
||||||
headerReal = "X-Real-IP"
|
headerReal = "X-Real-IP"
|
||||||
headerTrue = "True-Client-IP"
|
headerTrue = "True-Client-IP"
|
||||||
|
|
||||||
|
// clientIPv4 is the sample IPv4 client address these tests key
|
||||||
|
// on, both directly and in IPv4-mapped form. clientIPv4Alt is
|
||||||
|
// its neighbour, used to show the two do not share a bucket.
|
||||||
|
clientIPv4 = "198.51.100.7"
|
||||||
|
clientIPv4Alt = "198.51.100.8"
|
||||||
|
|
||||||
|
// clientIPv6 and clientIPv6Same are two addresses inside one
|
||||||
|
// routed /64, so both must key on clientBucketV6.
|
||||||
|
// clientIPv6Other is a different allocation and must key on
|
||||||
|
// clientOtherBucketV6.
|
||||||
|
clientIPv6 = "2001:db8:1:2:3:4:5:6"
|
||||||
|
clientIPv6Same = "2001:db8:1:2:aaaa:bbbb:cccc:dddd"
|
||||||
|
clientIPv6Other = "2001:db8:1:3::1"
|
||||||
|
clientBucketV6 = "2001:db8:1:2::/64"
|
||||||
|
clientOtherBucketV6 = "2001:db8:1:3::/64"
|
||||||
|
|
||||||
|
// trustedProxyCIDR is the proxy network the forwarded-path
|
||||||
|
// tests configure, and trustedPeer an address inside it. A
|
||||||
|
// production deployment is required to run behind a reverse
|
||||||
|
// proxy with TRUSTED_PROXIES set, so this is the shape the
|
||||||
|
// bucketing has to hold in.
|
||||||
|
trustedProxyCIDR = "10.0.0.0/8"
|
||||||
|
trustedPeer = "10.0.0.1:44444"
|
||||||
)
|
)
|
||||||
|
|
||||||
// assertSharedBucket drives the login limiter from peer with the
|
// assertSharedBucket drives the login limiter from peer with the
|
||||||
@@ -458,8 +483,8 @@ func TestRateLimitKey_SingleValuedHeadersIgnoredFromTrustedPeer(
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
assertSharedBucket(
|
assertSharedBucket(
|
||||||
t, trustedProxies("10.0.0.0/8"),
|
t, trustedProxies(trustedProxyCIDR),
|
||||||
"10.0.0.1:44444",
|
trustedPeer,
|
||||||
func(i int) map[string]string {
|
func(i int) map[string]string {
|
||||||
return map[string]string{
|
return map[string]string{
|
||||||
header: fmt.Sprintf(
|
header: fmt.Sprintf(
|
||||||
@@ -495,8 +520,8 @@ func TestRateLimitKey_MalformedRightmostHopFallsBackToPeer(
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
assertSharedBucket(
|
assertSharedBucket(
|
||||||
t, trustedProxies("10.0.0.0/8"),
|
t, trustedProxies(trustedProxyCIDR),
|
||||||
"10.0.0.1:44444",
|
trustedPeer,
|
||||||
func(i int) map[string]string {
|
func(i int) map[string]string {
|
||||||
return map[string]string{
|
return map[string]string{
|
||||||
headerXFF: fmt.Sprintf(
|
headerXFF: fmt.Sprintf(
|
||||||
@@ -522,13 +547,13 @@ func TestRateLimitKey_ForwardedHonouredFromTrustedPeer(
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
m := rateLimitMiddleware(t, &config.Config{
|
m := rateLimitMiddleware(t, &config.Config{
|
||||||
TrustedProxies: trustedProxies("10.0.0.0/8"),
|
TrustedProxies: trustedProxies(trustedProxyCIDR),
|
||||||
})
|
})
|
||||||
handler := m.LoginRateLimit()(okHandler())
|
handler := m.LoginRateLimit()(okHandler())
|
||||||
|
|
||||||
const peer = "10.0.0.1:44444"
|
const peer = trustedPeer
|
||||||
|
|
||||||
first := map[string]string{headerXFF: "198.51.100.7"}
|
first := map[string]string{headerXFF: clientIPv4}
|
||||||
|
|
||||||
for range middleware.LoginRateLimitConst {
|
for range middleware.LoginRateLimitConst {
|
||||||
postWithHeaders(handler, peer, loginPath, first)
|
postWithHeaders(handler, peer, loginPath, first)
|
||||||
@@ -542,7 +567,7 @@ func TestRateLimitKey_ForwardedHonouredFromTrustedPeer(
|
|||||||
|
|
||||||
w = postWithHeaders(
|
w = postWithHeaders(
|
||||||
handler, peer, loginPath,
|
handler, peer, loginPath,
|
||||||
map[string]string{headerXFF: "198.51.100.8"},
|
map[string]string{headerXFF: clientIPv4Alt},
|
||||||
)
|
)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, http.StatusOK, w.Code,
|
t, http.StatusOK, w.Code,
|
||||||
@@ -559,7 +584,7 @@ func TestRateLimitKey_ChainWalkSkipsClientPrepended(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
assertSharedBucket(
|
assertSharedBucket(
|
||||||
t, trustedProxies("10.0.0.0/8"), "10.0.0.1:44444",
|
t, trustedProxies(trustedProxyCIDR), trustedPeer,
|
||||||
func(i int) map[string]string {
|
func(i int) map[string]string {
|
||||||
return map[string]string{
|
return map[string]string{
|
||||||
headerXFF: fmt.Sprintf(
|
headerXFF: fmt.Sprintf(
|
||||||
@@ -594,7 +619,7 @@ func TestRateLimitKey_LongChainCapsWalkAndFallsBackToPeer(
|
|||||||
start := time.Now()
|
start := time.Now()
|
||||||
|
|
||||||
assertSharedBucket(
|
assertSharedBucket(
|
||||||
t, trustedProxies("10.0.0.0/8"), "10.0.0.1:44444",
|
t, trustedProxies(trustedProxyCIDR), trustedPeer,
|
||||||
func(i int) map[string]string {
|
func(i int) map[string]string {
|
||||||
return map[string]string{
|
return map[string]string{
|
||||||
headerXFF: fmt.Sprintf("9.9.9.%d%s", i+1, padding),
|
headerXFF: fmt.Sprintf("9.9.9.%d%s", i+1, padding),
|
||||||
@@ -633,13 +658,13 @@ func TestRateLimitKey_LongChainAllocationIsBounded(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
m := rateLimitMiddleware(t, &config.Config{
|
m := rateLimitMiddleware(t, &config.Config{
|
||||||
TrustedProxies: trustedProxies("10.0.0.0/8"),
|
TrustedProxies: trustedProxies(trustedProxyCIDR),
|
||||||
})
|
})
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(
|
req := httptest.NewRequestWithContext(
|
||||||
context.Background(), http.MethodPost, loginPath, nil,
|
context.Background(), http.MethodPost, loginPath, nil,
|
||||||
)
|
)
|
||||||
req.RemoteAddr = "10.0.0.1:44444"
|
req.RemoteAddr = trustedPeer
|
||||||
req.Header.Set(
|
req.Header.Set(
|
||||||
headerXFF, "9.9.9.9"+strings.Repeat(", 10.0.0.2", hops),
|
headerXFF, "9.9.9.9"+strings.Repeat(", 10.0.0.2", hops),
|
||||||
)
|
)
|
||||||
@@ -835,3 +860,369 @@ func TestReceiverRateLimit_IgnoresForwardedFromUntrustedPeer(
|
|||||||
"not mint a fresh receiver bucket",
|
"not mint a fresh receiver bucket",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// clientKeyFor returns the bucket key m computes for a request whose
|
||||||
|
// direct peer is remoteAddr and which carries no forwarded headers.
|
||||||
|
func clientKeyFor(
|
||||||
|
t *testing.T, m *middleware.Middleware, remoteAddr string,
|
||||||
|
) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
req := httptest.NewRequestWithContext(
|
||||||
|
context.Background(), http.MethodPost, loginPath, nil,
|
||||||
|
)
|
||||||
|
req.RemoteAddr = remoteAddr
|
||||||
|
|
||||||
|
return middleware.ClientKeyForTest(m, req)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRateLimitKey_IPv6BucketsByPrefix pins the key function's
|
||||||
|
// address-family behaviour. IPv6 clients must bucket by /64 — a
|
||||||
|
// routed /64 is the normal residential and mobile allocation, so
|
||||||
|
// per-/128 keying lets one subscriber rotate source addresses and
|
||||||
|
// mint a fresh bucket per request — while IPv4 keeps keying on the
|
||||||
|
// full address and IPv4-mapped form is keyed as the IPv4 address it
|
||||||
|
// carries.
|
||||||
|
func TestRateLimitKey_IPv6BucketsByPrefix(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
m := rateLimitMiddleware(t, &config.Config{})
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
peer string
|
||||||
|
want string
|
||||||
|
about string
|
||||||
|
}{{
|
||||||
|
name: "ipv6",
|
||||||
|
peer: "[" + clientIPv6 + "]:44444",
|
||||||
|
want: clientBucketV6,
|
||||||
|
about: "an IPv6 peer must key on its /64",
|
||||||
|
}, {
|
||||||
|
name: "ipv6-other-in-same-64",
|
||||||
|
peer: "[" + clientIPv6Same + "]:1",
|
||||||
|
want: clientBucketV6,
|
||||||
|
about: "another address in the same /64 must key the same",
|
||||||
|
}, {
|
||||||
|
name: "ipv6-different-64",
|
||||||
|
peer: "[" + clientIPv6Other + "]:44444",
|
||||||
|
want: clientOtherBucketV6,
|
||||||
|
about: "a different /64 must key differently",
|
||||||
|
}, {
|
||||||
|
name: "ipv4",
|
||||||
|
peer: clientIPv4 + ":44444",
|
||||||
|
want: clientIPv4,
|
||||||
|
about: "IPv4 must keep keying on the full address",
|
||||||
|
}, {
|
||||||
|
name: "ipv4-neighbour",
|
||||||
|
peer: clientIPv4Alt + ":44444",
|
||||||
|
want: clientIPv4Alt,
|
||||||
|
about: "adjacent IPv4 addresses must not share a bucket",
|
||||||
|
}, {
|
||||||
|
name: "ipv4-mapped",
|
||||||
|
peer: "[::ffff:" + clientIPv4 + "]:44444",
|
||||||
|
want: clientIPv4,
|
||||||
|
about: "IPv4-mapped form must key as the IPv4 address, " +
|
||||||
|
"not be masked to a /64: mapped addresses all share " +
|
||||||
|
"::ffff:0:0/96, so masking would collapse every IPv4 " +
|
||||||
|
"client behind a mapping proxy into one bucket",
|
||||||
|
}} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
assert.Equal(
|
||||||
|
t, tc.want, clientKeyFor(t, m, tc.peer), tc.about,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRateLimitKey_FamiliesDoNotCollide pins the structure the
|
||||||
|
// no-collision property rests on, rather than one sample pair: every
|
||||||
|
// IPv4 key is a bare address and every IPv6 key is a /64 in CIDR
|
||||||
|
// form, so the two name spaces are disjoint by shape. Dropping the
|
||||||
|
// masking strips the suffix that guarantees it, which is why this
|
||||||
|
// asserts the form of each key and not just that two of them differ.
|
||||||
|
func TestRateLimitKey_FamiliesDoNotCollide(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// Restated here rather than imported from the package under
|
||||||
|
// test, so that changing the production bucket width fails this
|
||||||
|
// test instead of silently moving with it.
|
||||||
|
const wantBits = 64
|
||||||
|
|
||||||
|
m := rateLimitMiddleware(t, &config.Config{})
|
||||||
|
|
||||||
|
v4Keys := map[string]bool{}
|
||||||
|
|
||||||
|
for _, peer := range []string{
|
||||||
|
clientIPv4 + ":44444",
|
||||||
|
clientIPv4Alt + ":44444",
|
||||||
|
"[::ffff:" + clientIPv4 + "]:44444",
|
||||||
|
} {
|
||||||
|
key := clientKeyFor(t, m, peer)
|
||||||
|
|
||||||
|
addr, err := netip.ParseAddr(key)
|
||||||
|
require.NoError(
|
||||||
|
t, err, "%s: an IPv4 key must be a bare address", peer,
|
||||||
|
)
|
||||||
|
assert.True(
|
||||||
|
t, addr.Is4(),
|
||||||
|
"%s: an IPv4 key must be a dotted quad, got %q", peer, key,
|
||||||
|
)
|
||||||
|
|
||||||
|
v4Keys[key] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, peer := range []string{
|
||||||
|
"[" + clientIPv6 + "]:44444",
|
||||||
|
"[" + clientIPv6Same + "]:44444",
|
||||||
|
"[" + clientIPv6Other + "]:44444",
|
||||||
|
"[2001:db8::" + clientIPv4 + "]:44444",
|
||||||
|
} {
|
||||||
|
key := clientKeyFor(t, m, peer)
|
||||||
|
|
||||||
|
prefix, err := netip.ParsePrefix(key)
|
||||||
|
require.NoError(
|
||||||
|
t, err, "%s: an IPv6 key must be a CIDR prefix", peer,
|
||||||
|
)
|
||||||
|
assert.Equal(
|
||||||
|
t, wantBits, prefix.Bits(),
|
||||||
|
"%s: an IPv6 key must name a /64", peer,
|
||||||
|
)
|
||||||
|
assert.False(
|
||||||
|
t, v4Keys[key],
|
||||||
|
"%s: an IPv6 key must never equal an IPv4 key", peer,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRateLimitKey_UnparseablePeerKeepsDistinctBuckets covers the
|
||||||
|
// fallback path. A RemoteAddr that is not an address must not panic,
|
||||||
|
// and must not drop unrelated clients into one shared bucket by
|
||||||
|
// accident: the raw value is the most specific identity left, so
|
||||||
|
// distinct values stay in distinct buckets.
|
||||||
|
func TestRateLimitKey_UnparseablePeerKeepsDistinctBuckets(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
m := rateLimitMiddleware(t, &config.Config{})
|
||||||
|
|
||||||
|
first := clientKeyFor(t, m, "not-an-address")
|
||||||
|
second := clientKeyFor(t, m, "also-not-an-address:1234")
|
||||||
|
|
||||||
|
assert.NotEmpty(t, first)
|
||||||
|
assert.NotEqual(
|
||||||
|
t, first, second,
|
||||||
|
"unparseable peers must not collapse into one bucket",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLoginRateLimit_IPv6SharesBucketWithinSlash64 is the behavioural
|
||||||
|
// half, and the regression test for the bypass itself: a client that
|
||||||
|
// rotates source addresses inside its own routed /64 must stay in one
|
||||||
|
// bucket. Reverting the masking makes this test fail, because each
|
||||||
|
// rotated address would mint a fresh bucket and nothing would be
|
||||||
|
// rejected.
|
||||||
|
func TestLoginRateLimit_IPv6SharesBucketWithinSlash64(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
m := rateLimitMiddleware(t, &config.Config{})
|
||||||
|
handler := m.LoginRateLimit()(okHandler())
|
||||||
|
|
||||||
|
for i := range middleware.LoginRateLimitConst {
|
||||||
|
w := postWithHeaders(
|
||||||
|
handler,
|
||||||
|
fmt.Sprintf("[2001:db8:1:2::%d]:44444", i+1),
|
||||||
|
loginPath, nil,
|
||||||
|
)
|
||||||
|
assert.Equal(
|
||||||
|
t, http.StatusOK, w.Code, "request %d should pass", i,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
w := postWithHeaders(
|
||||||
|
handler, "[2001:db8:1:2::ffff]:44444", loginPath, nil,
|
||||||
|
)
|
||||||
|
assert.Equal(
|
||||||
|
t, http.StatusTooManyRequests, w.Code,
|
||||||
|
"rotating source addresses inside one routed /64 must not "+
|
||||||
|
"mint fresh buckets",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLoginRateLimit_IPv6IndependentAcrossSlash64 is the other side
|
||||||
|
// of the trade: bucketing by /64 must not merge separate allocations,
|
||||||
|
// so a client in a different /64 keeps its own limit.
|
||||||
|
func TestLoginRateLimit_IPv6IndependentAcrossSlash64(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
m := rateLimitMiddleware(t, &config.Config{})
|
||||||
|
handler := m.LoginRateLimit()(okHandler())
|
||||||
|
|
||||||
|
for range middleware.LoginRateLimitConst + 1 {
|
||||||
|
postWithHeaders(
|
||||||
|
handler, "[2001:db8:1:2::1]:44444", loginPath, nil,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
w := postWithHeaders(
|
||||||
|
handler, "[2001:db8:1:3::1]:44444", loginPath, nil,
|
||||||
|
)
|
||||||
|
assert.Equal(
|
||||||
|
t, http.StatusOK, w.Code,
|
||||||
|
"a different /64 must have its own bucket",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLoginRateLimit_IPv4IndependentPerAddress guards against the
|
||||||
|
// masking leaking into IPv4: two addresses one apart must still hold
|
||||||
|
// separate buckets.
|
||||||
|
func TestLoginRateLimit_IPv4IndependentPerAddress(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
m := rateLimitMiddleware(t, &config.Config{})
|
||||||
|
handler := m.LoginRateLimit()(okHandler())
|
||||||
|
|
||||||
|
for range middleware.LoginRateLimitConst + 1 {
|
||||||
|
postWithHeaders(
|
||||||
|
handler, clientIPv4+":44444", loginPath, nil,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
w := postWithHeaders(
|
||||||
|
handler, clientIPv4Alt+":44444", loginPath, nil,
|
||||||
|
)
|
||||||
|
assert.Equal(
|
||||||
|
t, http.StatusOK, w.Code,
|
||||||
|
"a second IPv4 address must have its own bucket",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// forwardedKeyFor returns the bucket key m computes for a request
|
||||||
|
// that arrives from trustedPeer — a configured trusted proxy — and
|
||||||
|
// names forwarded as its client in X-Forwarded-For. That is the
|
||||||
|
// production path: a deployment is required to run behind a reverse
|
||||||
|
// proxy with TRUSTED_PROXIES set, so the forwarded address, not the
|
||||||
|
// peer, is what the limiters bucket on there.
|
||||||
|
func forwardedKeyFor(
|
||||||
|
t *testing.T, m *middleware.Middleware, forwarded string,
|
||||||
|
) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
req := httptest.NewRequestWithContext(
|
||||||
|
context.Background(), http.MethodPost, loginPath, nil,
|
||||||
|
)
|
||||||
|
req.RemoteAddr = trustedPeer
|
||||||
|
req.Header.Set(headerXFF, forwarded)
|
||||||
|
|
||||||
|
return middleware.ClientKeyForTest(m, req)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRateLimitKey_ForwardedIPv6BucketsByPrefix pins the /64
|
||||||
|
// bucketing on the trusted-proxy branch. The direct-peer tests above
|
||||||
|
// cannot reach it, so without this the masking could be reverted for
|
||||||
|
// forwarded clients alone — the only shape a production deployment
|
||||||
|
// runs in — and the rest of the suite would stay green.
|
||||||
|
func TestRateLimitKey_ForwardedIPv6BucketsByPrefix(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
m := rateLimitMiddleware(t, &config.Config{
|
||||||
|
TrustedProxies: trustedProxies(trustedProxyCIDR),
|
||||||
|
})
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
forwarded string
|
||||||
|
want string
|
||||||
|
about string
|
||||||
|
}{{
|
||||||
|
name: "ipv6",
|
||||||
|
forwarded: clientIPv6,
|
||||||
|
want: clientBucketV6,
|
||||||
|
about: "a forwarded IPv6 client must key on its /64",
|
||||||
|
}, {
|
||||||
|
name: "ipv6-other-in-same-64",
|
||||||
|
forwarded: clientIPv6Same,
|
||||||
|
want: clientBucketV6,
|
||||||
|
about: "another forwarded address in the same /64 must " +
|
||||||
|
"key the same",
|
||||||
|
}, {
|
||||||
|
name: "ipv6-different-64",
|
||||||
|
forwarded: clientIPv6Other,
|
||||||
|
want: clientOtherBucketV6,
|
||||||
|
about: "a forwarded address in another /64 must differ",
|
||||||
|
}, {
|
||||||
|
name: "ipv4",
|
||||||
|
forwarded: clientIPv4,
|
||||||
|
want: clientIPv4,
|
||||||
|
about: "a forwarded IPv4 client must key on the address",
|
||||||
|
}, {
|
||||||
|
name: "ipv4-mapped",
|
||||||
|
forwarded: "::ffff:" + clientIPv4,
|
||||||
|
want: clientIPv4,
|
||||||
|
about: "a proxy that forwards IPv4-mapped form must key as " +
|
||||||
|
"the IPv4 address it carries, not be masked to a /64: " +
|
||||||
|
"mapped addresses all share ::ffff:0:0/96",
|
||||||
|
}} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
assert.Equal(
|
||||||
|
t, tc.want,
|
||||||
|
forwardedKeyFor(t, m, tc.forwarded), tc.about,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLoginRateLimit_ForwardedIPv6SharesBucketWithinSlash64 is the
|
||||||
|
// behavioural half on the production path: behind a trusted proxy, a
|
||||||
|
// client rotating source addresses inside its own routed /64 must
|
||||||
|
// stay in one bucket.
|
||||||
|
func TestLoginRateLimit_ForwardedIPv6SharesBucketWithinSlash64(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
assertSharedBucket(
|
||||||
|
t, trustedProxies(trustedProxyCIDR), trustedPeer,
|
||||||
|
func(i int) map[string]string {
|
||||||
|
return map[string]string{
|
||||||
|
headerXFF: fmt.Sprintf("2001:db8:1:2::%d", i+1),
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"rotating forwarded source addresses inside one routed /64 "+
|
||||||
|
"must not mint fresh buckets",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLoginRateLimit_ForwardedIPv6IndependentAcrossSlash64 is the
|
||||||
|
// other side of that trade on the same path: bucketing by /64 must
|
||||||
|
// not merge two allocations reaching the proxy.
|
||||||
|
func TestLoginRateLimit_ForwardedIPv6IndependentAcrossSlash64(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
m := rateLimitMiddleware(t, &config.Config{
|
||||||
|
TrustedProxies: trustedProxies(trustedProxyCIDR),
|
||||||
|
})
|
||||||
|
handler := m.LoginRateLimit()(okHandler())
|
||||||
|
|
||||||
|
spent := map[string]string{headerXFF: clientIPv6}
|
||||||
|
for range middleware.LoginRateLimitConst + 1 {
|
||||||
|
postWithHeaders(handler, trustedPeer, loginPath, spent)
|
||||||
|
}
|
||||||
|
|
||||||
|
w := postWithHeaders(
|
||||||
|
handler, trustedPeer, loginPath,
|
||||||
|
map[string]string{headerXFF: clientIPv6Other},
|
||||||
|
)
|
||||||
|
assert.Equal(
|
||||||
|
t, http.StatusOK, w.Code,
|
||||||
|
"a forwarded client in a different /64 must have its own "+
|
||||||
|
"bucket",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ import (
|
|||||||
"sneak.berlin/go/webhooker/internal/middleware"
|
"sneak.berlin/go/webhooker/internal/middleware"
|
||||||
"sneak.berlin/go/webhooker/internal/server"
|
"sneak.berlin/go/webhooker/internal/server"
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
"sneak.berlin/go/webhooker/internal/session"
|
||||||
|
"sneak.berlin/go/webhooker/static"
|
||||||
)
|
)
|
||||||
|
|
||||||
// csrfCookieName is the cookie gorilla/csrf issues when it runs. Its
|
// csrfCookieName is the cookie gorilla/csrf issues when it runs. Its
|
||||||
@@ -246,6 +247,56 @@ func (e *testEnv) storedHash(t *testing.T, username string) string {
|
|||||||
return user.Password
|
return user.Password
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- /s static group ---
|
||||||
|
|
||||||
|
// TestStaticServesEveryMethod pins what the static mount actually
|
||||||
|
// answers. chi's Mount registers the handler for all methods and
|
||||||
|
// http.FileServer only special-cases HEAD (by suppressing the body),
|
||||||
|
// so a POST or a DELETE to an asset is served the file rather than
|
||||||
|
// refused. The README documents this; the test is what keeps the two
|
||||||
|
// from drifting.
|
||||||
|
func TestStaticServesEveryMethod(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := newTestEnv(t)
|
||||||
|
|
||||||
|
body, err := static.Static.ReadFile("js/app.js")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotEmpty(t, body)
|
||||||
|
|
||||||
|
for _, method := range []string{
|
||||||
|
http.MethodGet,
|
||||||
|
http.MethodHead,
|
||||||
|
http.MethodPost,
|
||||||
|
http.MethodPut,
|
||||||
|
http.MethodDelete,
|
||||||
|
} {
|
||||||
|
t.Run(method, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
req := httptest.NewRequestWithContext(
|
||||||
|
context.Background(), method,
|
||||||
|
"/s/js/app.js", nil,
|
||||||
|
)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
env.router.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
assert.Equal(t, http.StatusOK, w.Code,
|
||||||
|
"static mount answers every method")
|
||||||
|
|
||||||
|
if method == http.MethodHead {
|
||||||
|
assert.Empty(t, w.Body.Bytes(),
|
||||||
|
"HEAD must not carry a body")
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Equal(t, body, w.Body.Bytes(),
|
||||||
|
"the asset itself is returned")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// --- /pages group ---
|
// --- /pages group ---
|
||||||
|
|
||||||
// TestPagesLogin_OversizeBody_RejectedBeforeCSRF proves the cap runs
|
// TestPagesLogin_OversizeBody_RejectedBeforeCSRF proves the cap runs
|
||||||
|
|||||||
@@ -3,20 +3,14 @@
|
|||||||
# this repo. Idempotent: every install is guarded by a check so already
|
# this repo. Idempotent: every install is guarded by a check so already
|
||||||
# installed tools are skipped. Base tooling comes from nix, apt, brew,
|
# installed tools are skipped. Base tooling comes from nix, apt, brew,
|
||||||
# or apk (detected in that order); assumes NOTHING is present (not git,
|
# or apk (detected in that order); assumes NOTHING is present (not git,
|
||||||
# make, or go). golangci-lint is packaged in nix, brew, and apk; on apt
|
# make, or go). golangci-lint is deliberately not installed: linting runs
|
||||||
# it is installed from a hash-verified GitHub release archive (never
|
# only in docker, via script/lint and Dockerfile.lint. Finishes by running
|
||||||
# curl | sh). Finishes by running script/fetch-assets, which installs the
|
# script/fetch-assets, which installs the hash-pinned third-party browser
|
||||||
# hash-pinned third-party browser assets the repo does not commit.
|
# assets the repo does not commit.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
# Pinned versions, 2026-08-07. Never "latest"; exact versions only.
|
|
||||||
GOLANGCI_LINT_VERSION="2.12.2"
|
|
||||||
# sha256 of golangci-lint-2.12.2-linux-<arch>.tar.gz release archives
|
|
||||||
GOLANGCI_LINT_SHA256_AMD64="8df580d2670fed8fa984aac0507099af8df275e665215f5c7a2ae3943893a553"
|
|
||||||
GOLANGCI_LINT_SHA256_ARM64="44cd40a8c76c86755375adfeea52cfd3533cb43d7bd647771e0ae065e166df3a"
|
|
||||||
|
|
||||||
PKGMGR=""
|
PKGMGR=""
|
||||||
SUDO=""
|
SUDO=""
|
||||||
|
|
||||||
@@ -57,52 +51,6 @@ missing() {
|
|||||||
! command -v "$1" >/dev/null 2>&1
|
! command -v "$1" >/dev/null 2>&1
|
||||||
}
|
}
|
||||||
|
|
||||||
# verify_sha256 <file> <expected-hash>
|
|
||||||
verify_sha256() {
|
|
||||||
if command -v sha256sum >/dev/null 2>&1; then
|
|
||||||
actual="$(sha256sum "$1" | cut -d' ' -f1)"
|
|
||||||
else
|
|
||||||
actual="$(shasum -a 256 "$1" | cut -d' ' -f1)"
|
|
||||||
fi
|
|
||||||
if [ "$actual" != "$2" ]; then
|
|
||||||
echo "bootstrap: sha256 mismatch for $1" >&2
|
|
||||||
echo " expected: $2" >&2
|
|
||||||
echo " actual: $actual" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# apt has no golangci-lint package: install a pinned release archive
|
|
||||||
# from GitHub, verified by hardcoded sha256 (never curl | sh).
|
|
||||||
install_golangci_lint_release() {
|
|
||||||
case "$(uname -m)" in
|
|
||||||
x86_64) goarch="amd64"; sha="$GOLANGCI_LINT_SHA256_AMD64" ;;
|
|
||||||
aarch64|arm64) goarch="arm64"; sha="$GOLANGCI_LINT_SHA256_ARM64" ;;
|
|
||||||
*)
|
|
||||||
echo "bootstrap: unsupported architecture $(uname -m)" >&2
|
|
||||||
exit 1
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
if missing curl; then pkg_install curl curl curl curl; fi
|
|
||||||
name="golangci-lint-${GOLANGCI_LINT_VERSION}-linux-${goarch}"
|
|
||||||
tmp="$(mktemp -d)"
|
|
||||||
curl -fsSL -o "$tmp/$name.tar.gz" \
|
|
||||||
"https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCI_LINT_VERSION}/${name}.tar.gz"
|
|
||||||
verify_sha256 "$tmp/$name.tar.gz" "$sha"
|
|
||||||
tar -xzf "$tmp/$name.tar.gz" -C "$tmp"
|
|
||||||
$SUDO install -m 0755 "$tmp/$name/golangci-lint" /usr/local/bin/golangci-lint
|
|
||||||
rm -rf "$tmp"
|
|
||||||
}
|
|
||||||
|
|
||||||
ensure_golangci_lint() {
|
|
||||||
if ! missing golangci-lint; then return 0; fi
|
|
||||||
detect_pkgmgr
|
|
||||||
case "$PKGMGR" in
|
|
||||||
apt) install_golangci_lint_release ;;
|
|
||||||
*) pkg_install golangci-lint golangci-lint golangci-lint golangci-lint ;;
|
|
||||||
esac
|
|
||||||
}
|
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
|
|
||||||
@@ -110,9 +58,14 @@ main() {
|
|||||||
if missing git; then pkg_install git git git git; fi
|
if missing git; then pkg_install git git git git; fi
|
||||||
if missing make; then pkg_install gnumake make make make; fi
|
if missing make; then pkg_install gnumake make make make; fi
|
||||||
|
|
||||||
# Go toolchain and linter
|
# Go toolchain
|
||||||
if missing go; then pkg_install go golang go go; fi
|
if missing go; then pkg_install go golang go go; fi
|
||||||
ensure_golangci_lint
|
|
||||||
|
# Not installed here: docker is platform-specific and out of scope for a
|
||||||
|
# package-manager bootstrap, but script/lint needs it.
|
||||||
|
if missing docker; then
|
||||||
|
echo "bootstrap: docker not found; script/lint requires it" >&2
|
||||||
|
fi
|
||||||
|
|
||||||
go mod download
|
go mod download
|
||||||
|
|
||||||
|
|||||||
47
script/lint
47
script/lint
@@ -1,12 +1,55 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/lint: run the linter.
|
# script/lint: run the linter. golangci-lint is never installed locally: it
|
||||||
|
# runs via docker only, one way, everywhere — script/lint builds
|
||||||
|
# Dockerfile.lint, which COPYs the repo into the pinned golangci-lint image
|
||||||
|
# and lints as a build step. This works even when the docker daemon is remote
|
||||||
|
# and bind mounts are impossible, and it removes the host linter's shared
|
||||||
|
# cache, which has attributed other checkouts' findings to this one.
|
||||||
|
#
|
||||||
|
# --no-cache-filter=lint forces the lint stage to re-execute on every run; a
|
||||||
|
# cached lint stage exits 0 in under a second having linted nothing. The deps
|
||||||
|
# stage keeps its cache, so module downloads are not repeated.
|
||||||
|
# --progress=plain keeps the linter's own output visible on success, so a
|
||||||
|
# passing run shows the issue count rather than nothing.
|
||||||
|
# --output=type=cacheonly leaves no image behind to clean up.
|
||||||
|
#
|
||||||
|
# docker silently ignores --no-cache-filter for a stage name that does not
|
||||||
|
# match, so a rename or a typo would restore the cached false green with no
|
||||||
|
# warning and a fast exit 0. The flag is therefore not trusted: the build
|
||||||
|
# output is teed to a log and a run is only a pass if golangci-lint's own
|
||||||
|
# summary line ("N issues." / "N issues:") is in it. No summary, no lint,
|
||||||
|
# whatever the exit code says.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
cd "$ROOT"
|
cd "$ROOT"
|
||||||
golangci-lint run --config .golangci.yml ./...
|
|
||||||
|
log="$(mktemp -t webhooker-lint.XXXXXXXX)"
|
||||||
|
rcfile="$(mktemp -t webhooker-lint-rc.XXXXXXXX)"
|
||||||
|
trap 'rm -f "$log" "$rcfile"' EXIT INT TERM
|
||||||
|
|
||||||
|
# The pipeline's status is tee's, and POSIX sh has no pipefail, so the
|
||||||
|
# build's status travels via a file. Output still streams live.
|
||||||
|
{
|
||||||
|
docker build \
|
||||||
|
-f Dockerfile.lint \
|
||||||
|
--no-cache-filter=lint \
|
||||||
|
--progress=plain \
|
||||||
|
--output=type=cacheonly \
|
||||||
|
. 2>&1 && echo 0 >"$rcfile" || echo $? >"$rcfile"
|
||||||
|
} | tee "$log" >&2
|
||||||
|
|
||||||
|
rc="$(cat "$rcfile")"
|
||||||
|
[ "$rc" -eq 0 ] || exit "$rc"
|
||||||
|
|
||||||
|
if ! grep -qE '[0-9]+ issues[.:]' "$log"; then
|
||||||
|
echo "script/lint: golangci-lint printed no summary line; the linter" >&2
|
||||||
|
echo " did not run. Check that the stage named in --no-cache-filter" >&2
|
||||||
|
echo " still matches a stage in Dockerfile.lint." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
Reference in New Issue
Block a user