Compare commits
1 Commits
92f3a016e1
...
9ac8ff28ec
| Author | SHA1 | Date | |
|---|---|---|---|
| 9ac8ff28ec |
@@ -3,11 +3,6 @@
|
|||||||
# stage of the Dockerfile.
|
# stage of the Dockerfile.
|
||||||
.git/
|
.git/
|
||||||
bin/
|
bin/
|
||||||
# Third-party browser assets are fetched and hash-verified inside the build by
|
|
||||||
# script/fetch-assets. Excluding any host copy keeps a developer's working tree
|
|
||||||
# from supplying the bytes that get shipped. The script and its
|
|
||||||
# static/vendor.sha256 manifest stay in the context.
|
|
||||||
static/js/alpine.min.js
|
|
||||||
*.md
|
*.md
|
||||||
LICENSE
|
LICENSE
|
||||||
.editorconfig
|
.editorconfig
|
||||||
|
|||||||
7
.gitignore
vendored
7
.gitignore
vendored
@@ -44,9 +44,4 @@ tmp/
|
|||||||
temp/
|
temp/
|
||||||
|
|
||||||
# CI cache barrier, written into the build context by the check workflow
|
# CI cache barrier, written into the build context by the check workflow
|
||||||
.ci-fingerprint
|
.ci-fingerprint
|
||||||
|
|
||||||
# Third-party browser assets, fetched and hash-verified by
|
|
||||||
# script/fetch-assets against static/vendor.sha256. Not committed:
|
|
||||||
# REPO_POLICIES.md forbids minified bundles in version control.
|
|
||||||
/static/js/alpine.min.js
|
|
||||||
10
Dockerfile
10
Dockerfile
@@ -32,7 +32,7 @@ FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a349228
|
|||||||
# Depend on lint stage passing
|
# Depend on lint stage passing
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends make curl ca-certificates && rm -rf /var/lib/apt/lists/*
|
RUN apt-get update && apt-get install -y --no-install-recommends make && rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
WORKDIR /build
|
WORKDIR /build
|
||||||
|
|
||||||
@@ -44,14 +44,6 @@ RUN go mod download
|
|||||||
# the lint stage above.
|
# the lint stage above.
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# Fetch the third-party browser assets the UI serves. They are not committed
|
|
||||||
# (REPO_POLICIES.md forbids minified bundles in version control) and
|
|
||||||
# .dockerignore keeps any host copy out of the build context, so this step is
|
|
||||||
# the only way they enter the image. Each download is checked against a
|
|
||||||
# hardcoded sha256 and the build fails on mismatch; make test re-checks the
|
|
||||||
# hashes against the bytes go:embed actually put in the binary.
|
|
||||||
RUN script/fetch-assets
|
|
||||||
|
|
||||||
# Run tests and build
|
# Run tests and build
|
||||||
RUN make test
|
RUN make test
|
||||||
RUN make build
|
RUN make build
|
||||||
|
|||||||
5
Makefile
5
Makefile
@@ -1,4 +1,4 @@
|
|||||||
.PHONY: bootstrap setup assets test lint fmt fmt-check check build run dev deps docker clean hooks css
|
.PHONY: bootstrap setup test lint fmt fmt-check check build run dev deps docker clean hooks css
|
||||||
|
|
||||||
# Default target
|
# Default target
|
||||||
.DEFAULT_GOAL := check
|
.DEFAULT_GOAL := check
|
||||||
@@ -9,9 +9,6 @@ bootstrap:
|
|||||||
setup:
|
setup:
|
||||||
@script/setup
|
@script/setup
|
||||||
|
|
||||||
assets:
|
|
||||||
@script/fetch-assets
|
|
||||||
|
|
||||||
test:
|
test:
|
||||||
@script/test
|
@script/test
|
||||||
|
|
||||||
|
|||||||
185
README.md
185
README.md
@@ -16,9 +16,6 @@ with retry support, logging, and observability. Category: infrastructure
|
|||||||
in the `Dockerfile`'s lint stage; `make bootstrap` installs it)
|
in the `Dockerfile`'s lint stage; `make bootstrap` installs it)
|
||||||
- Docker (for containerized deployment, and for the lint and test
|
- Docker (for containerized deployment, and for the lint and test
|
||||||
stages of the CI gate)
|
stages of the CI gate)
|
||||||
- `curl`, used by `script/fetch-assets` to download the third-party
|
|
||||||
browser assets, which are not committed (`make bootstrap` installs
|
|
||||||
it if missing)
|
|
||||||
|
|
||||||
### Quick Start
|
### Quick Start
|
||||||
|
|
||||||
@@ -27,12 +24,10 @@ with retry support, logging, and observability. Category: infrastructure
|
|||||||
git clone https://git.eeqj.de/sneak/webhooker.git
|
git clone https://git.eeqj.de/sneak/webhooker.git
|
||||||
cd webhooker
|
cd webhooker
|
||||||
|
|
||||||
# Install Go dependencies, the pinned linter, and the third-party
|
# Install Go dependencies
|
||||||
# browser assets. `make deps` alone is not enough: it only runs
|
make deps
|
||||||
# go mod download/tidy, and the checks below need the fetched assets.
|
|
||||||
make bootstrap
|
|
||||||
|
|
||||||
# Run all checks (test, lint, format check)
|
# Run all checks (format, lint, test, build)
|
||||||
make check
|
make check
|
||||||
|
|
||||||
# Run in development mode. DATA_DIR defaults to /var/lib/webhooker in
|
# Run in development mode. DATA_DIR defaults to /var/lib/webhooker in
|
||||||
@@ -49,9 +44,7 @@ make docker
|
|||||||
```bash
|
```bash
|
||||||
make bootstrap # Install all dependencies (idempotent)
|
make bootstrap # Install all dependencies (idempotent)
|
||||||
make setup # Bootstrap + install git pre-commit hook
|
make setup # Bootstrap + install git pre-commit hook
|
||||||
make assets # Fetch + verify third-party browser assets
|
|
||||||
make fmt # Format code (gofmt + goimports)
|
make fmt # Format code (gofmt + goimports)
|
||||||
make fmt-check # Fail if gofmt would change anything (writes nothing)
|
|
||||||
make lint # Run golangci-lint
|
make lint # Run golangci-lint
|
||||||
make test # Run tests with race detection
|
make test # Run tests with race detection
|
||||||
make check # test + lint + fmt-check (CI gate)
|
make check # test + lint + fmt-check (CI gate)
|
||||||
@@ -264,16 +257,13 @@ restarts.
|
|||||||
This repository adheres to the
|
This repository adheres to the
|
||||||
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
[Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all)
|
||||||
standard: normalized scripts in `script/` are the entrypoints for the
|
standard: normalized scripts in `script/` are the entrypoints for the
|
||||||
development workflow. Ten of the Makefile's sixteen targets are thin
|
development workflow, and the Makefile targets are thin shims that call
|
||||||
shims that call them; `build`, `run`, `dev`, `deps`, `clean` and `css`
|
them. We provide:
|
||||||
are inline commands with no script behind them. We provide:
|
|
||||||
|
|
||||||
- `script/bootstrap` — install all dependencies (idempotent)
|
- `script/bootstrap` — install all dependencies (idempotent)
|
||||||
- `script/setup` — make a fresh clone ready for development
|
- `script/setup` — make a fresh clone ready for development
|
||||||
(bootstrap, then install-precommit)
|
(bootstrap, then install-precommit)
|
||||||
- `script/projectname` — output the project name ("webhooker")
|
- `script/projectname` — output the project name ("webhooker")
|
||||||
- `script/fetch-assets` — download the third-party browser assets into
|
|
||||||
`static/`, verifying each against its pinned sha256
|
|
||||||
- `script/test` — run the test suite
|
- `script/test` — run the test suite
|
||||||
- `script/lint` — run golangci-lint
|
- `script/lint` — run golangci-lint
|
||||||
- `script/fmt` — format all code (writes)
|
- `script/fmt` — format all code (writes)
|
||||||
@@ -287,27 +277,6 @@ are inline commands with no script behind them. We provide:
|
|||||||
- `script/install-precommit` — install the git pre-commit hook that
|
- `script/install-precommit` — install the git pre-commit hook that
|
||||||
runs `script/precommit`
|
runs `script/precommit`
|
||||||
|
|
||||||
## Third-party browser assets
|
|
||||||
|
|
||||||
The web UI serves one third-party script, Alpine.js. It is **not** committed:
|
|
||||||
a minified bundle in the tree is unreviewable, and `REPO_POLICIES.md` bars
|
|
||||||
both committed build artifacts and unpinned external references.
|
|
||||||
|
|
||||||
Instead `script/fetch-assets` downloads it from a pinned URL, checks the
|
|
||||||
download against a hardcoded sha256, and installs it under `static/`. The
|
|
||||||
sha256 of every installed asset is recorded in `static/vendor.sha256`, and
|
|
||||||
`static/vendor_test.go` re-hashes the bytes `go:embed` put in the binary
|
|
||||||
against that manifest — so the pin is enforced on what actually ships, not
|
|
||||||
merely written down. Any mismatch fails the build.
|
|
||||||
|
|
||||||
`make bootstrap` runs the fetch for local development, and the Dockerfile
|
|
||||||
runs it in the build stage; `.gitignore` and `.dockerignore` keep the
|
|
||||||
artifact out of both the repo and the build context.
|
|
||||||
|
|
||||||
To move to a new version: update the version, URL, and tarball sha256 in
|
|
||||||
`script/fetch-assets` and the asset sha256 in `static/vendor.sha256`, then
|
|
||||||
run `make assets && make check`.
|
|
||||||
|
|
||||||
## Rationale
|
## Rationale
|
||||||
|
|
||||||
Webhook integrations between services are inherently fragile. The
|
Webhook integrations between services are inherently fragile. The
|
||||||
@@ -400,7 +369,7 @@ The codebase uses consistent naming throughout (rename completed in
|
|||||||
|
|
||||||
### Data Model
|
### Data Model
|
||||||
|
|
||||||
webhooker's data model has nine entities organized into two tiers: the
|
webhooker's data model has eight entities organized into two tiers: the
|
||||||
**application tier** (user and webhook configuration) and the **event
|
**application tier** (user and webhook configuration) and the **event
|
||||||
tier** (event ingestion, delivery, and logging).
|
tier** (event ingestion, delivery, and logging).
|
||||||
|
|
||||||
@@ -495,20 +464,15 @@ such a webhook's retention as "forever" rather than as a day count.
|
|||||||
Submitted `retention_days` values therefore fall into three bands, not
|
Submitted `retention_days` values therefore fall into three bands, not
|
||||||
two:
|
two:
|
||||||
|
|
||||||
- `1` up to `database.MaxFiniteRetentionDays` (106751 days, about 292
|
- `0` up to `database.MaxFiniteRetentionDays` (106751 days, about 292
|
||||||
years) is accepted as a finite retention.
|
years) is accepted as a finite retention.
|
||||||
- Above that ceiling but below the retain-forever sentinel of 365000
|
- Above that ceiling but below the retain-forever sentinel of 365000
|
||||||
(`database.RetentionForeverDays`) is rejected with a 400. This is the
|
(`database.RetentionForeverDays`) is rejected with a 400. This is the
|
||||||
band the cap exists for.
|
band the cap exists for.
|
||||||
- `0`, and `365000` or above, are accepted and mean retain forever,
|
- `365000` or above is accepted and means retain forever, collapsing to
|
||||||
collapsing to the sentinel — `0` in `Webhook.BeforeSave`, the large
|
the sentinel. It is not out of range: the edit form pre-fills the
|
||||||
values in `parseRetentionDays`. The large values are not out of
|
sentinel for a retain-forever webhook, so submitting that form back
|
||||||
range: the edit form pre-fills the sentinel for a retain-forever
|
unchanged has to keep meaning "forever".
|
||||||
webhook, so submitting that form back unchanged has to keep meaning
|
|
||||||
"forever".
|
|
||||||
|
|
||||||
A negative value is in none of the three: `parseRetentionDays` rejects
|
|
||||||
it with a 400 before `BeforeSave` ever sees it.
|
|
||||||
|
|
||||||
The cap is not arbitrary: the reaper computes its cutoff as a
|
The cap is not arbitrary: the reaper computes its cutoff as a
|
||||||
`time.Duration`, an int64 nanosecond count, and a longer period
|
`time.Duration`, an int64 nanosecond count, and a longer period
|
||||||
@@ -528,7 +492,7 @@ the full request and creates an Event.
|
|||||||
| -------------- | ------- | ----------- |
|
| -------------- | ------- | ----------- |
|
||||||
| `id` | UUID | Primary key |
|
| `id` | UUID | Primary key |
|
||||||
| `webhook_id` | UUID | Foreign key → Webhook |
|
| `webhook_id` | UUID | Foreign key → Webhook |
|
||||||
| `path` | string | Unique bare UUID, generated at creation. The `/webhook/` prefix is route only and is not stored: the receiver matches this column against the raw `{uuid}` path segment |
|
| `path` | string | Unique URL path (UUID-based, e.g. `/webhook/{uuid}`) |
|
||||||
| `description` | string | Optional description |
|
| `description` | string | Optional description |
|
||||||
| `active` | boolean | Whether this entrypoint accepts events (default: true) |
|
| `active` | boolean | Whether this entrypoint accepts events (default: true) |
|
||||||
|
|
||||||
@@ -552,8 +516,8 @@ events should be forwarded.
|
|||||||
| `type` | TargetType | One of: `http`, `slack`, `database`, `log` |
|
| `type` | TargetType | One of: `http`, `slack`, `database`, `log` |
|
||||||
| `active` | boolean | Whether deliveries are enabled (default: true) |
|
| `active` | boolean | Whether deliveries are enabled (default: true) |
|
||||||
| `config` | JSON text | Type-specific configuration |
|
| `config` | JSON text | Type-specific configuration |
|
||||||
| `max_retries` | integer | Maximum retry attempts for `http` and `slack` targets (0 = fire-and-forget, >0 = retries with backoff and a circuit breaker). Ignored by `database` and `log` targets |
|
| `max_retries` | integer | Maximum retry attempts for HTTP targets (0 = fire-and-forget, >0 = retries with backoff) |
|
||||||
| `max_queue_size` | integer | Stored and shown on the target's detail view, but not enforced anywhere yet: nothing in the delivery engine consults it. Queue depth is set by the two fixed 10,000-entry channels |
|
| `max_queue_size` | integer | Maximum queued deliveries (for HTTP targets with retries) |
|
||||||
|
|
||||||
**Relations:** Belongs to Webhook. Has many Deliveries.
|
**Relations:** Belongs to Webhook. Has many Deliveries.
|
||||||
|
|
||||||
@@ -566,11 +530,6 @@ events should be forwarded.
|
|||||||
greater than 0, failed deliveries are retried with exponential backoff
|
greater than 0, failed deliveries are retried with exponential backoff
|
||||||
up to `max_retries` attempts, protected by a per-target circuit
|
up to `max_retries` attempts, protected by a per-target circuit
|
||||||
breaker.
|
breaker.
|
||||||
- **`slack`** — Post the event as a formatted message to a
|
|
||||||
Slack-compatible incoming webhook URL (`webhookUrl` in `config`). It
|
|
||||||
is built on the same HTTP core as `http` and honours `max_retries`
|
|
||||||
identically, circuit breaker included. See the Slack target section
|
|
||||||
under "Per-Webhook Event Databases" for the message format.
|
|
||||||
- **`database`** — Archive the full event as a row into a separate
|
- **`database`** — Archive the full event as a row into a separate
|
||||||
per-webhook archive database (`archive-{webhookID}.db`) for long-term
|
per-webhook archive database (`archive-{webhookID}.db`) for long-term
|
||||||
retention, with an optional creation-validated expiry (default: keep
|
retention, with an optional creation-validated expiry (default: keep
|
||||||
@@ -662,9 +621,7 @@ retries) is individually logged for full observability.
|
|||||||
|
|
||||||
#### Common Fields
|
#### Common Fields
|
||||||
|
|
||||||
Every entity except `Setting` includes these fields from `BaseModel`.
|
All entities include these fields from `BaseModel`:
|
||||||
`Setting` is a bare key-value row with no `id`, no timestamps and no
|
|
||||||
soft delete:
|
|
||||||
|
|
||||||
| Field | Type | Description |
|
| Field | Type | Description |
|
||||||
| ------------ | --------- | ----------- |
|
| ------------ | --------- | ----------- |
|
||||||
@@ -710,7 +667,7 @@ handles connection pooling, lazy opening, migrations, and cleanup.
|
|||||||
This separation provides:
|
This separation provides:
|
||||||
|
|
||||||
- **Isolation** — a high-volume webhook won't cause lock contention or
|
- **Isolation** — a high-volume webhook won't cause lock contention or
|
||||||
journal growth affecting the main application or other webhooks.
|
WAL bloat affecting the main application or other webhooks.
|
||||||
- **Independent lifecycle** — event databases can be independently
|
- **Independent lifecycle** — event databases can be independently
|
||||||
backed up, archived, rotated, or size-limited without impacting the
|
backed up, archived, rotated, or size-limited without impacting the
|
||||||
application.
|
application.
|
||||||
@@ -720,12 +677,9 @@ This separation provides:
|
|||||||
- **Per-webhook retention** — the `retention_days` field on each webhook
|
- **Per-webhook retention** — the `retention_days` field on each webhook
|
||||||
controls automatic cleanup of old events in that webhook's database
|
controls automatic cleanup of old events in that webhook's database
|
||||||
only, or disables cleanup entirely when set to `0` (retain forever).
|
only, or disables cleanup entirely when set to `0` (retain forever).
|
||||||
- **Performance** — each webhook's database has its own page cache and
|
- **Performance** — each webhook's database has its own WAL, its own
|
||||||
its own lock, so concurrent event ingestion across webhooks won't
|
page cache, and its own lock, so concurrent event ingestion across
|
||||||
contend. No write-ahead log is involved: both DSNs are
|
webhooks won't contend.
|
||||||
`file:{path}?cache=shared&mode=rwc` and no `journal_mode` pragma is
|
|
||||||
ever issued, so every database runs on SQLite's default rollback
|
|
||||||
journal.
|
|
||||||
|
|
||||||
The **database target type** builds on this architecture to provide
|
The **database target type** builds on this architecture to provide
|
||||||
long-term archiving, separate from the per-webhook event database (which
|
long-term archiving, separate from the per-webhook event database (which
|
||||||
@@ -781,9 +735,8 @@ and other compatible services). Each message includes event metadata
|
|||||||
pretty-printed in a code block. JSON payloads are automatically
|
pretty-printed in a code block. JSON payloads are automatically
|
||||||
formatted with indentation for readability; non-JSON payloads are shown
|
formatted with indentation for readability; non-JSON payloads are shown
|
||||||
as raw text. Large payloads are truncated to keep messages reasonable.
|
as raw text. Large payloads are truncated to keep messages reasonable.
|
||||||
Config stores `webhookUrl` — the Slack/Mattermost incoming webhook
|
Config stores `webhook_url` — the Slack/Mattermost incoming webhook
|
||||||
endpoint. That is the JSON key; the error text for a missing one reads
|
endpoint.
|
||||||
`webhook_url is required`, which is the message, not the key.
|
|
||||||
|
|
||||||
The database uses the
|
The database uses the
|
||||||
[modernc.org/sqlite](https://pkg.go.dev/modernc.org/sqlite) driver at
|
[modernc.org/sqlite](https://pkg.go.dev/modernc.org/sqlite) driver at
|
||||||
@@ -805,9 +758,8 @@ External Service
|
|||||||
1. Look up Entrypoint by UUID
|
1. Look up Entrypoint by UUID
|
||||||
2. Capture full request as Event
|
2. Capture full request as Event
|
||||||
3. Create Delivery records for each active Target
|
3. Create Delivery records for each active Target
|
||||||
4. Build self-contained delivery.Task structs
|
4. Build self-contained DeliveryTask structs
|
||||||
(target config + event data inline for
|
(target config + event data inline for ≤16KB)
|
||||||
bodies < 16 KiB)
|
|
||||||
5. Notify Engine via channel (no DB read needed)
|
5. Notify Engine via channel (no DB read needed)
|
||||||
│
|
│
|
||||||
▼
|
▼
|
||||||
@@ -842,7 +794,7 @@ at any time, preventing goroutine explosions regardless of queue depth.
|
|||||||
a delivery channel (new tasks from the webhook handler) and a retry
|
a delivery channel (new tasks from the webhook handler) and a retry
|
||||||
channel (tasks from backoff timers). Both are buffered to 10,000.
|
channel (tasks from backoff timers). Both are buffered to 10,000.
|
||||||
- **Fan-out via channel, not goroutines:** When an event arrives with
|
- **Fan-out via channel, not goroutines:** When an event arrives with
|
||||||
multiple targets, each `delivery.Task` is sent to the delivery channel.
|
multiple targets, each `DeliveryTask` is sent to the delivery channel.
|
||||||
Workers pick them up and process them — no goroutine-per-target.
|
Workers pick them up and process them — no goroutine-per-target.
|
||||||
- **Worker goroutines:** A fixed number of worker goroutines select from
|
- **Worker goroutines:** A fixed number of worker goroutines select from
|
||||||
both channels. Each worker processes one task at a time, then picks up
|
both channels. Each worker processes one task at a time, then picks up
|
||||||
@@ -866,12 +818,7 @@ This means:
|
|||||||
- **Independent results** — each worker records its own delivery result
|
- **Independent results** — each worker records its own delivery result
|
||||||
in the per-webhook database without coordination.
|
in the per-webhook database without coordination.
|
||||||
- **Graceful shutdown** — cancel the context, workers finish their
|
- **Graceful shutdown** — cancel the context, workers finish their
|
||||||
current task and exit. The stop hook waits for the pool via
|
current task and exit. `WaitGroup.Wait()` ensures clean shutdown.
|
||||||
`lifecycle.WaitForShutdown`, which bounds that wait by fx's stop
|
|
||||||
timeout rather than blocking forever on a wedged worker. On timeout
|
|
||||||
it logs at `ERROR` and returns an error, and the goroutines that
|
|
||||||
did not finish are still running — an unclean shutdown is reported
|
|
||||||
rather than hidden.
|
|
||||||
|
|
||||||
**Recovery paths:**
|
**Recovery paths:**
|
||||||
|
|
||||||
@@ -899,13 +846,12 @@ remains stored in the per-webhook event database, there is no way to
|
|||||||
redeliver it: manual redelivery is planned, not implemented (see
|
redeliver it: manual redelivery is planned, not implemented (see
|
||||||
[TODO.md](TODO.md)).
|
[TODO.md](TODO.md)).
|
||||||
|
|
||||||
### Circuit Breaker (HTTP and Slack Targets with Retries)
|
### Circuit Breaker (HTTP Targets with Retries)
|
||||||
|
|
||||||
`http` and `slack` targets with `max_retries` > 0 are protected by a
|
HTTP targets with `max_retries` > 0 are protected by a **per-target circuit breaker** that
|
||||||
**per-target circuit breaker** that prevents hammering a down target
|
prevents hammering a down target with repeated failed delivery attempts.
|
||||||
with repeated failed delivery attempts. The circuit breaker is
|
The circuit breaker is in-memory only and resets on restart (which is
|
||||||
in-memory only and resets on restart (which is fine — startup recovery
|
fine — startup recovery rescans the database anyway).
|
||||||
rescans the database anyway).
|
|
||||||
|
|
||||||
**States:**
|
**States:**
|
||||||
|
|
||||||
@@ -941,12 +887,10 @@ rescans the database anyway).
|
|||||||
- **Failure threshold:** 5 consecutive failures before opening
|
- **Failure threshold:** 5 consecutive failures before opening
|
||||||
- **Cooldown:** 30 seconds in open state before probing
|
- **Cooldown:** 30 seconds in open state before probing
|
||||||
|
|
||||||
**Scope:** Circuit breakers apply to **`http` and `slack` targets with
|
**Scope:** Circuit breakers only apply to **HTTP targets with
|
||||||
`max_retries` > 0**. The Slack target is built on the same HTTP core
|
`max_retries` > 0**. Fire-and-forget HTTP targets (`max_retries` == 0),
|
||||||
and hands its own `max_retries` to the same retry path, so it gets a
|
Slack targets, database targets (local operations), and log
|
||||||
breaker with the same 5-failure / 30-second defaults. Fire-and-forget
|
targets (stdout) do not use circuit breakers.
|
||||||
targets of either type (`max_retries` == 0), database targets (local
|
|
||||||
operations), and log targets (stdout) do not use circuit breakers.
|
|
||||||
|
|
||||||
When a circuit is open and a new delivery arrives, the engine marks the
|
When a circuit is open and a new delivery arrives, the engine marks the
|
||||||
delivery as `retrying` and schedules a retry timer for after the
|
delivery as `retrying` and schedules a retry timer for after the
|
||||||
@@ -962,11 +906,9 @@ unpredictable rates, and blanket limits shared with other routes would
|
|||||||
cause legitimate deliveries to be dropped.
|
cause legitimate deliveries to be dropped.
|
||||||
|
|
||||||
The receiver instead has its own dedicated abuse limit, scoped to the
|
The receiver instead has its own dedicated abuse limit, scoped to the
|
||||||
`/webhook/{uuid}` route only and keyed per client IP per request path
|
`/webhook/{uuid}` route only and keyed per client IP per entrypoint: one
|
||||||
(`httprate.KeyByEndpoint`): one misbehaving sender is throttled without
|
misbehaving sender is throttled without affecting other senders of the
|
||||||
affecting other senders of the same entrypoint or the same sender's
|
same entrypoint or the same sender's other entrypoints. The limit is
|
||||||
other entrypoints. Keying on the path rather than on the entrypoint
|
|
||||||
matters — see the aggregate limit below. The limit is
|
|
||||||
`RECEIVER_RATE_LIMIT` requests per minute (default 120, generous for
|
`RECEIVER_RATE_LIMIT` requests per minute (default 120, generous for
|
||||||
legitimate webhook senders). Requests over the limit receive HTTP 429
|
legitimate webhook senders). Requests over the limit receive HTTP 429
|
||||||
with a `Retry-After` header. A set-but-invalid `RECEIVER_RATE_LIMIT`
|
with a `Retry-After` header. A set-but-invalid `RECEIVER_RATE_LIMIT`
|
||||||
@@ -1039,7 +981,7 @@ abuse limit later; they are tracked as future work.
|
|||||||
| ------ | --------------------------- | ----------- |
|
| ------ | --------------------------- | ----------- |
|
||||||
| `GET` | `/` | Root redirect, 303 (authenticated → `/sources`, unauthenticated → `/pages/login`) |
|
| `GET` | `/` | Root redirect, 303 (authenticated → `/sources`, unauthenticated → `/pages/login`) |
|
||||||
| `GET` | `/.well-known/healthcheck` | Health check (JSON: `status`, `now`, `uptimeSeconds`, `uptimeHuman`, `version`, `appname`, `maintenanceMode`) |
|
| `GET` | `/.well-known/healthcheck` | Health check (JSON: `status`, `now`, `uptimeSeconds`, `uptimeHuman`, `version`, `appname`, `maintenanceMode`) |
|
||||||
| any | `/s/*` | Static file serving (embedded CSS, JS). Mounted for every method, not just `GET`/`HEAD`: chi's `Mount` registers all methods and `http.FileServer` special-cases only `HEAD` (by omitting the body), so a `POST` or `DELETE` to an asset is answered `200` with the file. Pinned by `TestStaticServesEveryMethod` |
|
| `GET` | `/s/*` | Static file serving (embedded CSS, JS; `GET` and `HEAD`) |
|
||||||
| `POST` | `/webhook/{uuid}` | Webhook receiver endpoint. `POST` only — every other method is answered `405 Method Not Allowed` with `Allow: POST`. Rate limited (see [Rate Limiting](#rate-limiting)) |
|
| `POST` | `/webhook/{uuid}` | Webhook receiver endpoint. `POST` only — every other method is answered `405 Method Not Allowed` with `Allow: POST`. Rate limited (see [Rate Limiting](#rate-limiting)) |
|
||||||
|
|
||||||
#### Authentication Endpoints
|
#### Authentication Endpoints
|
||||||
@@ -1121,13 +1063,12 @@ webhooker/
|
|||||||
│ │ ├── model_apikey.go # APIKey entity
|
│ │ ├── model_apikey.go # APIKey entity
|
||||||
│ │ ├── password.go # Argon2id hashing and verification
|
│ │ ├── password.go # Argon2id hashing and verification
|
||||||
│ │ ├── retention.go # Retention reaper (per-webhook event expiry)
|
│ │ ├── retention.go # Retention reaper (per-webhook event expiry)
|
||||||
│ │ ├── testing.go # NewTestDatabase: wrapper for tests, no fx lifecycle
|
|
||||||
│ │ └── webhook_db_manager.go # Per-webhook DB lifecycle manager
|
│ │ └── webhook_db_manager.go # Per-webhook DB lifecycle manager
|
||||||
│ ├── globals/
|
│ ├── globals/
|
||||||
│ │ └── globals.go # Build-time variables (appname, version, arch)
|
│ │ └── globals.go # Build-time variables (appname, version, arch)
|
||||||
│ ├── delivery/
|
│ ├── delivery/
|
||||||
│ │ ├── engine.go # Event-driven delivery engine (channel + timer based)
|
│ │ ├── engine.go # Event-driven delivery engine (channel + timer based)
|
||||||
│ │ ├── circuit_breaker.go # Per-target circuit breaker for http/slack targets with retries
|
│ │ ├── circuit_breaker.go # Per-target circuit breaker for HTTP targets with retries
|
||||||
│ │ ├── target.go # Target interface, Task, Scheduler
|
│ │ ├── target.go # Target interface, Task, Scheduler
|
||||||
│ │ ├── target_http.go # HTTP target (retries, circuit breaker)
|
│ │ ├── target_http.go # HTTP target (retries, circuit breaker)
|
||||||
│ │ ├── target_slack.go # Slack/Mattermost incoming-webhook target
|
│ │ ├── target_slack.go # Slack/Mattermost incoming-webhook target
|
||||||
@@ -1143,7 +1084,6 @@ webhooker/
|
|||||||
│ ├── handlers/
|
│ ├── handlers/
|
||||||
│ │ ├── handlers.go # Base handler struct, JSON helpers, template rendering
|
│ │ ├── handlers.go # Base handler struct, JSON helpers, template rendering
|
||||||
│ │ ├── auth.go # Login, logout handlers
|
│ │ ├── auth.go # Login, logout handlers
|
||||||
│ │ ├── event_log_view.go # Event log projection, byte-capped in SQL
|
|
||||||
│ │ ├── healthcheck.go # Health check handler
|
│ │ ├── healthcheck.go # Health check handler
|
||||||
│ │ ├── index.go # Index page handler
|
│ │ ├── index.go # Index page handler
|
||||||
│ │ ├── profile.go # User profile handler
|
│ │ ├── profile.go # User profile handler
|
||||||
@@ -1156,27 +1096,24 @@ webhooker/
|
|||||||
│ ├── middleware/
|
│ ├── middleware/
|
||||||
│ │ ├── middleware.go # Logging, CORS, Auth, Metrics, MetricsAuth, SecurityHeaders, MaxBodySize
|
│ │ ├── middleware.go # Logging, CORS, Auth, Metrics, MetricsAuth, SecurityHeaders, MaxBodySize
|
||||||
│ │ ├── csrf.go # CSRF protection middleware (gorilla/csrf)
|
│ │ ├── csrf.go # CSRF protection middleware (gorilla/csrf)
|
||||||
│ │ ├── ratelimit.go # Per-IP rate limiting middleware (go-chi/httprate)
|
│ │ └── ratelimit.go # Per-IP rate limiting middleware (go-chi/httprate)
|
||||||
│ │ └── testing.go # NewForTest: Middleware without the fx lifecycle
|
|
||||||
│ ├── server/
|
│ ├── server/
|
||||||
│ │ ├── server.go # Server struct, fx lifecycle, signal handling
|
│ │ ├── server.go # Server struct, fx lifecycle, signal handling
|
||||||
│ │ ├── http.go # HTTP server setup with timeouts
|
│ │ ├── http.go # HTTP server setup with timeouts
|
||||||
│ │ └── routes.go # All route definitions
|
│ │ └── routes.go # All route definitions
|
||||||
│ └── session/
|
│ └── session/
|
||||||
│ ├── session.go # Cookie-based session management
|
│ └── session.go # Cookie-based session management
|
||||||
│ └── testing.go # NewForTest: Session without the fx lifecycle
|
|
||||||
├── static/
|
├── static/
|
||||||
│ ├── static.go # //go:embed directive
|
│ ├── static.go # //go:embed directive
|
||||||
│ ├── css/input.css # Tailwind input, source for tailwind.css (make css)
|
│ ├── css/input.css # Tailwind input, source for tailwind.css (make css)
|
||||||
│ ├── css/tailwind.css # Generated stylesheet the pages load
|
│ ├── css/tailwind.css # Generated stylesheet the pages load
|
||||||
│ ├── css/style.css # Older hand-written stylesheet, no longer loaded
|
│ ├── css/style.css # Older hand-written stylesheet, no longer loaded
|
||||||
│ ├── js/app.js # Progressive-enhancement copy-to-clipboard
|
│ ├── js/alpine.min.js # Alpine.js, served locally (no CDN)
|
||||||
│ ├── js/alpine.min.js # Alpine.js, fetched by script/fetch-assets, not committed
|
│ └── js/app.js # Progressive-enhancement copy-to-clipboard
|
||||||
│ └── vendor.sha256 # Pinned hashes the fetched assets are verified against
|
|
||||||
├── templates/ # Go HTML templates (base, login, sources, etc.)
|
├── templates/ # Go HTML templates (base, login, sources, etc.)
|
||||||
├── script/ # Scripts to Rule Them All entrypoints
|
├── script/ # Scripts to Rule Them All entrypoints
|
||||||
├── Dockerfile # Three stages: lint, test+build, Alpine runtime
|
├── Dockerfile # Three stages: lint, test+build, Alpine runtime
|
||||||
├── Makefile # 10 of 16 targets shim script/; 6 are inline
|
├── Makefile # Thin shims over script/
|
||||||
├── go.mod / go.sum
|
├── go.mod / go.sum
|
||||||
└── .golangci.yml # Linter configuration
|
└── .golangci.yml # Linter configuration
|
||||||
```
|
```
|
||||||
@@ -1210,9 +1147,9 @@ triggers the fx lifecycle hooks in dependency order. The
|
|||||||
`delivery.Notifier` interface allows the webhook handler to send
|
`delivery.Notifier` interface allows the webhook handler to send
|
||||||
self-contained `delivery.Task` slices to the engine without a direct
|
self-contained `delivery.Task` slices to the engine without a direct
|
||||||
package dependency. Each task carries all target config and event data
|
package dependency. Each task carries all target config and event data
|
||||||
inline (for bodies under 16 KiB, `delivery.MaxInlineBodySize`), so the
|
inline (for bodies ≤16KB, `delivery.MaxInlineBodySize`), so the engine
|
||||||
engine can deliver without reading from any database — it only writes
|
can deliver without reading from any database — it only writes to
|
||||||
to record results.
|
record results.
|
||||||
|
|
||||||
### Middleware Stack
|
### Middleware Stack
|
||||||
|
|
||||||
@@ -1241,12 +1178,10 @@ would never apply. A request that declares a `Content-Length` over the
|
|||||||
limit is answered with `413 Request Entity Too Large` without its body
|
limit is answered with `413 Request Entity Too Large` without its body
|
||||||
being read and without reaching CSRF, the route group's remaining
|
being read and without reaching CSRF, the route group's remaining
|
||||||
middleware, or the handler. It is not rejected before *any* other
|
middleware, or the handler. It is not rejected before *any* other
|
||||||
middleware, though: the global entries listed above all run first, so
|
middleware, though: the eight global entries listed above all run
|
||||||
such a request is still logged and given the security headers — and
|
first, so such a request is still logged, counted in the metrics, and
|
||||||
counted in the metrics, on a deployment where `METRICS_USERNAME` is
|
given the security headers, and the rejection itself is logged at
|
||||||
set and the Metrics middleware is therefore registered at all. The
|
`WARN` with the method, path and declared length. A chunked request, or
|
||||||
rejection itself is logged at `WARN` with the method, path and
|
|
||||||
declared length. A chunked request, or
|
|
||||||
one that lies about its length, is hard-capped by
|
one that lies about its length, is hard-capped by
|
||||||
`http.MaxBytesReader` and fails downstream at form-parse time.
|
`http.MaxBytesReader` and fails downstream at form-parse time.
|
||||||
|
|
||||||
@@ -1306,8 +1241,7 @@ rather than global: **LoginRateLimit** on `/pages/login`,
|
|||||||
- Static assets embedded in binary (no filesystem access needed at
|
- Static assets embedded in binary (no filesystem access needed at
|
||||||
runtime)
|
runtime)
|
||||||
- Container runs as non-root user (UID 1000)
|
- Container runs as non-root user (UID 1000)
|
||||||
- GORM soft deletes on every entity that carries `BaseModel`, which is
|
- GORM soft deletes on all entities (data preserved for audit)
|
||||||
all of them but `Setting` (data preserved for audit)
|
|
||||||
|
|
||||||
### Docker
|
### Docker
|
||||||
|
|
||||||
@@ -1319,9 +1253,8 @@ version is fixed independently of the compiler's:
|
|||||||
installs `make`, downloads dependencies, copies the source, and runs
|
installs `make`, downloads dependencies, copies the source, and runs
|
||||||
`make fmt-check` then `make lint`.
|
`make fmt-check` then `make lint`.
|
||||||
2. **Builder stage** (`golang:1.26.1-bookworm`) — depends on the lint
|
2. **Builder stage** (`golang:1.26.1-bookworm`) — depends on the lint
|
||||||
stage passing (it copies a file from it), runs `script/fetch-assets`
|
stage passing (it copies a file from it), then runs `make test` and
|
||||||
to download and verify the third-party browser assets, then runs
|
`make build`, and finally rebuilds the binary with
|
||||||
`make test` and `make build`, and finally rebuilds the binary with
|
|
||||||
`CGO_ENABLED=1` and static linking so it runs on musl.
|
`CGO_ENABLED=1` and static linking so it runs on musl.
|
||||||
3. **Runtime stage** (`alpine:3.21`) — copies the static binary,
|
3. **Runtime stage** (`alpine:3.21`) — copies the static binary,
|
||||||
creates the `/var/lib/webhooker` directory for all SQLite databases,
|
creates the `/var/lib/webhooker` directory for all SQLite databases,
|
||||||
@@ -1335,13 +1268,9 @@ linked, which is what lets it run on the Alpine runtime image.
|
|||||||
|
|
||||||
`script/cibuild` — `docker build .` — is the CI gate: the four check
|
`script/cibuild` — `docker build .` — is the CI gate: the four check
|
||||||
targets run inside the image, so a build that succeeds is a repo that
|
targets run inside the image, so a build that succeeds is a repo that
|
||||||
is formatted, linted, tested and compiled. Only `script/cibuild` and
|
is formatted, linted, tested and compiled. Local linting goes through
|
||||||
`script/docker` involve Docker. `script/lint`, and therefore
|
the same container rather than a host golangci-lint, because a host
|
||||||
`make lint` and `make check`, run whatever `golangci-lint` is on the
|
binary can be a different version from the pinned one.
|
||||||
host, which can be a different version from the pinned one — so the
|
|
||||||
container is the authoritative lint result
|
|
||||||
([issue #109](https://git.eeqj.de/sneak/webhooker/issues/109) tracks
|
|
||||||
routing local linting through it as well).
|
|
||||||
|
|
||||||
#### CI gate honesty
|
#### CI gate honesty
|
||||||
|
|
||||||
|
|||||||
@@ -106,12 +106,6 @@ func slackConfigFields(configJSON string) []ConfigField {
|
|||||||
// and its retry settings. Header values are not shown — they
|
// and its retry settings. Header values are not shown — they
|
||||||
// routinely carry authorization tokens — only how many are
|
// routinely carry authorization tokens — only how many are
|
||||||
// configured.
|
// configured.
|
||||||
//
|
|
||||||
// The destination is masked to scheme and host by the same
|
|
||||||
// rule the Slack target uses. An HTTP target's destination is
|
|
||||||
// commonly a Slack, Discord or Teams incoming-webhook endpoint
|
|
||||||
// whose path segments are the credential, and the field takes
|
|
||||||
// an arbitrary URL, so no segment can be assumed non-secret.
|
|
||||||
func httpConfigFields(t *database.Target) []ConfigField {
|
func httpConfigFields(t *database.Target) []ConfigField {
|
||||||
cfg, err := parseHTTPConfig(t.Config)
|
cfg, err := parseHTTPConfig(t.Config)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -120,7 +114,7 @@ func httpConfigFields(t *database.Target) []ConfigField {
|
|||||||
|
|
||||||
fields := []ConfigField{{
|
fields := []ConfigField{{
|
||||||
Label: "Destination URL",
|
Label: "Destination URL",
|
||||||
Value: MaskURL(cfg.URL),
|
Value: cfg.URL,
|
||||||
}}
|
}}
|
||||||
|
|
||||||
if cfg.Timeout > 0 {
|
if cfg.Timeout > 0 {
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ const (
|
|||||||
|
|
||||||
viewExampleOrigin = "https://example.com"
|
viewExampleOrigin = "https://example.com"
|
||||||
viewExampleHook = viewExampleOrigin + "/hook"
|
viewExampleHook = viewExampleOrigin + "/hook"
|
||||||
viewMaskedOrigin = viewExampleOrigin + "/..."
|
|
||||||
viewUnavailable = "(unavailable)"
|
viewUnavailable = "(unavailable)"
|
||||||
viewExpiryNever = "never"
|
viewExpiryNever = "never"
|
||||||
)
|
)
|
||||||
@@ -163,7 +162,7 @@ func TestNewTargetViews_HTTP(t *testing.T) {
|
|||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
map[string]string{
|
map[string]string{
|
||||||
"Destination URL": viewMaskedOrigin,
|
"Destination URL": viewExampleHook,
|
||||||
"Timeout": "30s",
|
"Timeout": "30s",
|
||||||
"Headers": "1 configured",
|
"Headers": "1 configured",
|
||||||
"Max Retries": "5",
|
"Max Retries": "5",
|
||||||
@@ -189,41 +188,13 @@ func TestNewTargetViews_HTTPFireAndForget(t *testing.T) {
|
|||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
map[string]string{
|
map[string]string{
|
||||||
"Destination URL": viewMaskedOrigin,
|
"Destination URL": viewExampleHook,
|
||||||
"Max Retries": "0 (fire-and-forget)",
|
"Max Retries": "0 (fire-and-forget)",
|
||||||
},
|
},
|
||||||
fieldMap(view.Config),
|
fieldMap(view.Config),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestNewTargetViews_HTTPMasksDestinationURL proves the rule
|
|
||||||
// holds for the http target too: an http destination is
|
|
||||||
// routinely an incoming-webhook endpoint whose path segments
|
|
||||||
// are the credential, so none of them is shown.
|
|
||||||
func TestNewTargetViews_HTTPMasksDestinationURL(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
view := viewFor(t, database.Target{
|
|
||||||
Type: database.TargetTypeHTTP,
|
|
||||||
Config: `{"url":"` + slackWebhookURL + `"}`,
|
|
||||||
})
|
|
||||||
|
|
||||||
fields := fieldMap(view.Config)
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
"https://hooks.slack.com/...",
|
|
||||||
fields["Destination URL"],
|
|
||||||
)
|
|
||||||
|
|
||||||
for _, v := range fields {
|
|
||||||
assert.NotContains(t, v, slackSecretPath)
|
|
||||||
assert.NotContains(t, v, "T00000000")
|
|
||||||
assert.NotContains(t, v, "B00000000")
|
|
||||||
assert.NotContains(t, v, "XXXXXXXXXXXXXXXXXXXXXXXX")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewTargetViews_Database(t *testing.T) {
|
func TestNewTargetViews_Database(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -1,120 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"time"
|
|
||||||
"unicode/utf8"
|
|
||||||
)
|
|
||||||
|
|
||||||
// maxRenderedBodyBytes caps how many bytes of a stored event
|
|
||||||
// body reach the event log page. Bodies come from the
|
|
||||||
// unauthenticated receiver under the 1 MB ingest cap and
|
|
||||||
// renderTemplate buffers a whole page before writing it, so
|
|
||||||
// an uncapped page of paginationPerPage events is tens of
|
|
||||||
// megabytes of resident memory per concurrent viewer.
|
|
||||||
const maxRenderedBodyBytes = 8192
|
|
||||||
|
|
||||||
// eventLogColumns is the event log's projection. The casts to
|
|
||||||
// blob are load-bearing: they make substr and length count
|
|
||||||
// bytes rather than characters, so the cap bounds the page in
|
|
||||||
// bytes whatever the payload's encoding. Cutting in SQLite
|
|
||||||
// rather than in Go is the point of the projection — an
|
|
||||||
// oversized body never becomes a Go string at all.
|
|
||||||
const eventLogColumns = "id, created_at, method, content_type, " +
|
|
||||||
"substr(cast(body as blob), 1, ?) AS body, " +
|
|
||||||
"length(cast(body as blob)) AS body_bytes"
|
|
||||||
|
|
||||||
// EventLogView is the display-safe projection of an event for
|
|
||||||
// the event log page, alongside DeliveryView and TargetView.
|
|
||||||
// It carries a capped body plus the true stored size, so the
|
|
||||||
// page can mark a body as truncated without ever holding the
|
|
||||||
// whole thing.
|
|
||||||
type EventLogView struct {
|
|
||||||
ID string
|
|
||||||
CreatedAt time.Time
|
|
||||||
Method string
|
|
||||||
ContentType string
|
|
||||||
|
|
||||||
// Body holds at most maxRenderedBodyBytes bytes of the
|
|
||||||
// stored body.
|
|
||||||
Body string
|
|
||||||
|
|
||||||
// BodyBytes is the true size of the stored body.
|
|
||||||
BodyBytes int64
|
|
||||||
|
|
||||||
// BodyTruncated reports that the stored body was larger
|
|
||||||
// than the cap, so the page owes the reader a marker.
|
|
||||||
BodyTruncated bool
|
|
||||||
|
|
||||||
Deliveries []DeliveryView
|
|
||||||
}
|
|
||||||
|
|
||||||
// BodyShownBytes is how many body bytes the page is actually
|
|
||||||
// rendering, which the truncation marker reports beside the
|
|
||||||
// true size.
|
|
||||||
func (v EventLogView) BodyShownBytes() int {
|
|
||||||
return len(v.Body)
|
|
||||||
}
|
|
||||||
|
|
||||||
// eventLogRow is one row of the event log projection. Its
|
|
||||||
// body column arrives already cut to the cap by SQLite, with
|
|
||||||
// the true size beside it.
|
|
||||||
type eventLogRow struct {
|
|
||||||
ID string
|
|
||||||
CreatedAt time.Time
|
|
||||||
Method string
|
|
||||||
ContentType string
|
|
||||||
Body []byte
|
|
||||||
BodyBytes int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// view projects a loaded row for rendering.
|
|
||||||
func (r *eventLogRow) view() EventLogView {
|
|
||||||
body := r.Body
|
|
||||||
truncated := r.BodyBytes > int64(len(body))
|
|
||||||
|
|
||||||
// Only a cut body can have been left mid-sequence by
|
|
||||||
// this query. A whole body is passed through exactly as
|
|
||||||
// stored, however malformed.
|
|
||||||
if truncated {
|
|
||||||
body = trimPartialRune(body)
|
|
||||||
}
|
|
||||||
|
|
||||||
return EventLogView{
|
|
||||||
ID: r.ID,
|
|
||||||
CreatedAt: r.CreatedAt,
|
|
||||||
Method: r.Method,
|
|
||||||
ContentType: r.ContentType,
|
|
||||||
Body: string(body),
|
|
||||||
BodyBytes: r.BodyBytes,
|
|
||||||
BodyTruncated: truncated,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// trimPartialRune drops a trailing UTF-8 sequence that the
|
|
||||||
// byte-wise cut left incomplete, so a multi-byte rune severed
|
|
||||||
// at the cap does not surface as a mojibake tail.
|
|
||||||
//
|
|
||||||
// Bytes that are merely invalid UTF-8 are left exactly as
|
|
||||||
// stored: this service receives binary payloads, and rewriting
|
|
||||||
// them would misreport what was delivered. The distinction is
|
|
||||||
// utf8.FullRune's — it reports a complete sequence for an
|
|
||||||
// invalid encoding too, since that decodes to a width-1 error
|
|
||||||
// rune, so only a valid prefix still waiting for its
|
|
||||||
// continuation bytes is removed. A tail with no rune start in
|
|
||||||
// its last utf8.UTFMax bytes cannot be an incomplete sequence
|
|
||||||
// either, and is likewise left alone.
|
|
||||||
func trimPartialRune(b []byte) []byte {
|
|
||||||
for i := len(b) - 1; i >= 0 && len(b)-i <= utf8.UTFMax; i-- {
|
|
||||||
if !utf8.RuneStart(b[i]) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if utf8.FullRune(b[i:]) {
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
|
|
||||||
return b[:i]
|
|
||||||
}
|
|
||||||
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
@@ -1,258 +0,0 @@
|
|||||||
package handlers_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"unicode/utf8"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/gorm/clause"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
|
||||||
)
|
|
||||||
|
|
||||||
// bodyCap is the number of body bytes the event log page is
|
|
||||||
// allowed to render for one event.
|
|
||||||
const bodyCap = handlers.MaxRenderedBodyBytesForTest
|
|
||||||
|
|
||||||
// snowman is a three-byte rune, so a body of them straddles the
|
|
||||||
// byte-wise cut: bodyCap is not a multiple of three.
|
|
||||||
const snowman = "☃"
|
|
||||||
|
|
||||||
// seedEventWithBody records one event with the given body in the
|
|
||||||
// webhook's own database.
|
|
||||||
func seedEventWithBody(
|
|
||||||
t *testing.T,
|
|
||||||
dbMgr *database.WebhookDBManager,
|
|
||||||
webhookID string,
|
|
||||||
body string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
webhookDB, err := dbMgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
event := &database.Event{
|
|
||||||
WebhookID: webhookID,
|
|
||||||
Method: http.MethodPost,
|
|
||||||
Body: body,
|
|
||||||
ContentType: "application/octet-stream",
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, webhookDB.Omit(
|
|
||||||
clause.Associations,
|
|
||||||
).Create(event).Error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// seedAndProject stores one body and returns the projection the
|
|
||||||
// event log page would be handed for it.
|
|
||||||
func seedAndProject(
|
|
||||||
t *testing.T,
|
|
||||||
body string,
|
|
||||||
) handlers.EventLogView {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
seedEventWithBody(t, dbMgr, wh.ID, body)
|
|
||||||
|
|
||||||
views := h.LoadEventLogViewsForTest(
|
|
||||||
httptest.NewRecorder(), *wh, 1,
|
|
||||||
)
|
|
||||||
require.Len(t, views, 1)
|
|
||||||
|
|
||||||
return views[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceLogs_BoundsOversizeBody proves the rendered
|
|
||||||
// page is bounded by the cap rather than by the stored payload:
|
|
||||||
// the body here is 64 times the cap, and the ingest path would
|
|
||||||
// accept twice as much again.
|
|
||||||
func TestHandleSourceLogs_BoundsOversizeBody(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
const (
|
|
||||||
sentinel = "TAIL-SENTINEL-1f4a9c"
|
|
||||||
storedBytes = 512 * 1024
|
|
||||||
)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
seedEventWithBody(
|
|
||||||
t, dbMgr, wh.ID,
|
|
||||||
strings.Repeat("A", storedBytes-len(sentinel))+sentinel,
|
|
||||||
)
|
|
||||||
|
|
||||||
page := renderSourceLogsPage(t, h, sess, wh.ID)
|
|
||||||
|
|
||||||
// Nothing past the cap reaches the page, and the whole page
|
|
||||||
// stays far below the stored body it is reporting on.
|
|
||||||
assert.NotContains(t, page, sentinel)
|
|
||||||
assert.Less(t, len(page), 4*bodyCap)
|
|
||||||
|
|
||||||
// The marker states the true stored size, not the cut one.
|
|
||||||
assert.Contains(
|
|
||||||
t, page,
|
|
||||||
"showing "+strconv.Itoa(bodyCap)+
|
|
||||||
" of "+strconv.Itoa(storedBytes)+" bytes",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceLogs_SmallBodyRendersWhole guards the other
|
|
||||||
// side of the cap: a body under it is shown in full and carries
|
|
||||||
// no truncation marker.
|
|
||||||
func TestHandleSourceLogs_SmallBodyRendersWhole(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
seedEventWithBody(t, dbMgr, wh.ID, `{"kept":"whole"}`)
|
|
||||||
|
|
||||||
page := renderSourceLogsPage(t, h, sess, wh.ID)
|
|
||||||
|
|
||||||
assert.Contains(t, page, ""kept"")
|
|
||||||
assert.NotContains(t, page, "Body truncated for display")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventLogView_CutMidRune proves a multi-byte rune severed
|
|
||||||
// by the byte-wise cut is dropped rather than surfaced as a
|
|
||||||
// mojibake tail.
|
|
||||||
func TestEventLogView_CutMidRune(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
body := strings.Repeat(snowman, 4096)
|
|
||||||
view := seedAndProject(t, body)
|
|
||||||
|
|
||||||
// bodyCap bytes hold bodyCap/3 whole snowmen and two bytes
|
|
||||||
// of the next one; those two are dropped.
|
|
||||||
whole := bodyCap / len(snowman)
|
|
||||||
|
|
||||||
assert.True(t, view.BodyTruncated)
|
|
||||||
assert.Equal(t, int64(len(body)), view.BodyBytes)
|
|
||||||
assert.Equal(t, strings.Repeat(snowman, whole), view.Body)
|
|
||||||
assert.True(t, utf8.ValidString(view.Body))
|
|
||||||
assert.LessOrEqual(t, len(view.Body), bodyCap)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventLogView_BinaryBodyLeftAsStored proves a binary
|
|
||||||
// payload is passed through byte for byte. Its tail is invalid
|
|
||||||
// UTF-8 however the cut falls, so repairing it would misreport
|
|
||||||
// what the sender delivered.
|
|
||||||
func TestEventLogView_BinaryBodyLeftAsStored(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
raw := make([]byte, bodyCap+808)
|
|
||||||
for i := range raw {
|
|
||||||
// 0x80..0xBF: continuation bytes, never a rune start.
|
|
||||||
raw[i] = 0x80 | byte(i%0x40)
|
|
||||||
}
|
|
||||||
|
|
||||||
view := seedAndProject(t, string(raw))
|
|
||||||
|
|
||||||
assert.True(t, view.BodyTruncated)
|
|
||||||
assert.Equal(t, int64(len(raw)), view.BodyBytes)
|
|
||||||
assert.Equal(t, string(raw[:bodyCap]), view.Body)
|
|
||||||
assert.False(t, utf8.ValidString(view.Body))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTrimPartialRune covers the distinction the cut repair
|
|
||||||
// turns on: an incomplete but valid sequence is dropped, while
|
|
||||||
// bytes that are merely invalid UTF-8 are left alone.
|
|
||||||
func TestTrimPartialRune(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
in []byte
|
|
||||||
want []byte
|
|
||||||
}{{
|
|
||||||
name: "complete ascii",
|
|
||||||
in: []byte("abc"),
|
|
||||||
want: []byte("abc"),
|
|
||||||
}, {
|
|
||||||
name: "complete multibyte",
|
|
||||||
in: []byte("ab" + snowman),
|
|
||||||
want: []byte("ab" + snowman),
|
|
||||||
}, {
|
|
||||||
name: "two byte rune cut",
|
|
||||||
in: []byte{'a', 0xC3},
|
|
||||||
want: []byte{'a'},
|
|
||||||
}, {
|
|
||||||
name: "three byte rune cut after one",
|
|
||||||
in: []byte{'a', 0xE2},
|
|
||||||
want: []byte{'a'},
|
|
||||||
}, {
|
|
||||||
name: "three byte rune cut after two",
|
|
||||||
in: []byte{'a', 0xE2, 0x98},
|
|
||||||
want: []byte{'a'},
|
|
||||||
}, {
|
|
||||||
name: "four byte rune cut",
|
|
||||||
in: []byte{'a', 0xF0, 0x9F, 0x92}, // U+1F4A9 cut
|
|
||||||
want: []byte{'a'},
|
|
||||||
}, {
|
|
||||||
name: "invalid start byte kept",
|
|
||||||
in: []byte{'a', 0xFF},
|
|
||||||
want: []byte{'a', 0xFF},
|
|
||||||
}, {
|
|
||||||
name: "orphan continuation bytes kept",
|
|
||||||
in: []byte{0x80, 0x81, 0x82, 0x83, 0x84},
|
|
||||||
want: []byte{0x80, 0x81, 0x82, 0x83, 0x84},
|
|
||||||
}, {
|
|
||||||
name: "truncated sequence followed by junk kept",
|
|
||||||
in: []byte{0xE2, 0x98, 0xFF},
|
|
||||||
want: []byte{0xE2, 0x98, 0xFF},
|
|
||||||
}, {
|
|
||||||
name: "empty",
|
|
||||||
in: []byte{},
|
|
||||||
want: []byte{},
|
|
||||||
}}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t, tc.want,
|
|
||||||
handlers.TrimPartialRuneForTest(tc.in),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -3,35 +3,8 @@ package handlers
|
|||||||
import (
|
import (
|
||||||
"html/template"
|
"html/template"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// MaxRenderedBodyBytesForTest exposes the event log's body cap
|
|
||||||
// to the handlers_test package.
|
|
||||||
const MaxRenderedBodyBytesForTest = maxRenderedBodyBytes
|
|
||||||
|
|
||||||
// TrimPartialRuneForTest exposes trimPartialRune for use in the
|
|
||||||
// handlers_test package.
|
|
||||||
func TrimPartialRuneForTest(b []byte) []byte {
|
|
||||||
return trimPartialRune(b)
|
|
||||||
}
|
|
||||||
|
|
||||||
// LoadEventLogViewsForTest exposes loadEventsWithDeliveries for
|
|
||||||
// use in the handlers_test package. Assertions on the projected
|
|
||||||
// body need the bytes as loaded: html/template rewrites invalid
|
|
||||||
// UTF-8 on the way out, so the rendered page cannot show whether
|
|
||||||
// a binary body survived the projection intact.
|
|
||||||
func (s *Handlers) LoadEventLogViewsForTest(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
webhook database.Webhook,
|
|
||||||
page int,
|
|
||||||
) []EventLogView {
|
|
||||||
views, _ := s.loadEventsWithDeliveries(w, webhook, nil, page)
|
|
||||||
|
|
||||||
return views
|
|
||||||
}
|
|
||||||
|
|
||||||
// AddTemplateForTest registers a template under a page name so that
|
// AddTemplateForTest registers a template under a page name so that
|
||||||
// the handlers_test package can drive the render path with a
|
// the handlers_test package can drive the render path with a
|
||||||
// template of its own.
|
// template of its own.
|
||||||
|
|||||||
@@ -229,10 +229,8 @@ func (s *Handlers) renderTemplate(
|
|||||||
// the response only once rendering has fully succeeded. Executing
|
// the response only once rendering has fully succeeded. Executing
|
||||||
// straight into the ResponseWriter commits a partial body and a 200
|
// straight into the ResponseWriter commits a partial body and a 200
|
||||||
// status before a mid-render error can be reported, leaving no way
|
// status before a mid-render error can be reported, leaving no way
|
||||||
// to serve a 500. Buffering makes a page's rendered size resident
|
// to serve a 500. These pages are small, so holding one in memory is
|
||||||
// memory per concurrent viewer, so every page owes it a bound: the
|
// the right trade.
|
||||||
// event log caps each stored body at maxRenderedBodyBytes for exactly
|
|
||||||
// this reason.
|
|
||||||
func (s *Handlers) executeTemplate(
|
func (s *Handlers) executeTemplate(
|
||||||
w http.ResponseWriter,
|
w http.ResponseWriter,
|
||||||
tmpl *template.Template,
|
tmpl *template.Template,
|
||||||
|
|||||||
@@ -131,47 +131,6 @@ func TestHandleSourceDetail_MasksSlackWebhookURL(t *testing.T) {
|
|||||||
assert.Contains(t, body, "https://hooks.slack.com/...")
|
assert.Contains(t, body, "https://hooks.slack.com/...")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleSourceDetail_MasksHTTPDestinationURL is the
|
|
||||||
// regression test for the same leak reached through the http
|
|
||||||
// target: its destination is routinely an incoming-webhook
|
|
||||||
// endpoint whose path segments are the credential, so the
|
|
||||||
// rendered page must not contain them.
|
|
||||||
func TestHandleSourceDetail_MasksHTTPDestinationURL(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
seedConfiguredTarget(
|
|
||||||
t, db, wh.ID,
|
|
||||||
database.TargetTypeHTTP,
|
|
||||||
`{"url":"`+slackWebhookURL+`"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
body := renderSourceDetailPage(t, h, sess, wh.ID)
|
|
||||||
|
|
||||||
assert.NotContains(t, body, slackSecretPath)
|
|
||||||
assert.NotContains(t, body, "T00000000")
|
|
||||||
assert.NotContains(t, body, "B00000000")
|
|
||||||
assert.NotContains(
|
|
||||||
t, body, "XXXXXXXXXXXXXXXXXXXXXXXX",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Contains(t, body, "Destination URL")
|
|
||||||
assert.Contains(t, body, "https://hooks.slack.com/...")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceDetail_RendersNamedTargetFields proves the
|
// TestHandleSourceDetail_RendersNamedTargetFields proves the
|
||||||
// other target types render labelled fields rather than the
|
// other target types render labelled fields rather than the
|
||||||
// stored blob.
|
// stored blob.
|
||||||
@@ -213,7 +172,7 @@ func TestHandleSourceDetail_RendersNamedTargetFields(
|
|||||||
body := renderSourceDetailPage(t, h, sess, wh.ID)
|
body := renderSourceDetailPage(t, h, sess, wh.ID)
|
||||||
|
|
||||||
assert.Contains(t, body, "Destination URL")
|
assert.Contains(t, body, "Destination URL")
|
||||||
assert.Contains(t, body, "https://example.com/...")
|
assert.Contains(t, body, "https://example.com/hook")
|
||||||
assert.Contains(t, body, "Timeout")
|
assert.Contains(t, body, "Timeout")
|
||||||
assert.Contains(t, body, "1 configured")
|
assert.Contains(t, body, "1 configured")
|
||||||
assert.NotContains(t, body, "sekrit")
|
assert.NotContains(t, body, "sekrit")
|
||||||
|
|||||||
@@ -92,6 +92,13 @@ func parseRetentionDays(raw string, fallback int) (int, error) {
|
|||||||
return v, nil
|
return v, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// EventWithDeliveries holds an event and its deliveries.
|
||||||
|
type EventWithDeliveries struct {
|
||||||
|
database.Event
|
||||||
|
|
||||||
|
Deliveries []DeliveryView
|
||||||
|
}
|
||||||
|
|
||||||
// DeliveryView is the display-safe projection of a delivery
|
// DeliveryView is the display-safe projection of a delivery
|
||||||
// for the event log page. Its target is a TargetView, so the
|
// for the event log page. Its target is a TargetView, so the
|
||||||
// stored configuration blob — which holds the target's
|
// stored configuration blob — which holds the target's
|
||||||
@@ -808,18 +815,16 @@ func (h *Handlers) parsePage(r *http.Request) int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// loadEventsWithDeliveries loads paginated events and their
|
// loadEventsWithDeliveries loads paginated events and their
|
||||||
// deliveries from the per-webhook database. Events come back
|
// deliveries from the per-webhook database.
|
||||||
// as capped projections rather than database.Event rows: see
|
|
||||||
// eventLogColumns for why the cut happens in SQL.
|
|
||||||
func (h *Handlers) loadEventsWithDeliveries(
|
func (h *Handlers) loadEventsWithDeliveries(
|
||||||
w http.ResponseWriter,
|
w http.ResponseWriter,
|
||||||
webhook database.Webhook,
|
webhook database.Webhook,
|
||||||
targetMap map[string]delivery.TargetView,
|
targetMap map[string]delivery.TargetView,
|
||||||
page int,
|
page int,
|
||||||
) ([]EventLogView, int64) {
|
) ([]EventWithDeliveries, int64) {
|
||||||
var totalEvents int64
|
var totalEvents int64
|
||||||
|
|
||||||
var result []EventLogView
|
var result []EventWithDeliveries
|
||||||
|
|
||||||
if !h.dbMgr.DBExists(webhook.ID) {
|
if !h.dbMgr.DBExists(webhook.ID) {
|
||||||
return result, totalEvents
|
return result, totalEvents
|
||||||
@@ -840,25 +845,23 @@ func (h *Handlers) loadEventsWithDeliveries(
|
|||||||
|
|
||||||
offset := (page - 1) * paginationPerPage
|
offset := (page - 1) * paginationPerPage
|
||||||
|
|
||||||
var rows []eventLogRow
|
var events []database.Event
|
||||||
|
|
||||||
webhookDB.Model(&database.Event{}).Select(
|
webhookDB.Where(
|
||||||
eventLogColumns, maxRenderedBodyBytes,
|
|
||||||
).Where(
|
|
||||||
"webhook_id = ?", webhook.ID,
|
"webhook_id = ?", webhook.ID,
|
||||||
).Order("created_at DESC").Offset(offset).Limit(
|
).Order("created_at DESC").Offset(offset).Limit(
|
||||||
paginationPerPage,
|
paginationPerPage,
|
||||||
).Find(&rows)
|
).Find(&events)
|
||||||
|
|
||||||
result = make([]EventLogView, len(rows))
|
result = make([]EventWithDeliveries, len(events))
|
||||||
|
|
||||||
for i := range rows {
|
for i := range events {
|
||||||
result[i] = rows[i].view()
|
result[i].Event = events[i]
|
||||||
|
|
||||||
var deliveries []database.Delivery
|
var deliveries []database.Delivery
|
||||||
|
|
||||||
webhookDB.Where(
|
webhookDB.Where(
|
||||||
"event_id = ?", rows[i].ID,
|
"event_id = ?", events[i].ID,
|
||||||
).Find(&deliveries)
|
).Find(&deliveries)
|
||||||
|
|
||||||
result[i].Deliveries = newDeliveryViews(
|
result[i].Deliveries = newDeliveryViews(
|
||||||
|
|||||||
@@ -24,7 +24,6 @@ import (
|
|||||||
"sneak.berlin/go/webhooker/internal/middleware"
|
"sneak.berlin/go/webhooker/internal/middleware"
|
||||||
"sneak.berlin/go/webhooker/internal/server"
|
"sneak.berlin/go/webhooker/internal/server"
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
"sneak.berlin/go/webhooker/internal/session"
|
||||||
"sneak.berlin/go/webhooker/static"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// csrfCookieName is the cookie gorilla/csrf issues when it runs. Its
|
// csrfCookieName is the cookie gorilla/csrf issues when it runs. Its
|
||||||
@@ -247,56 +246,6 @@ func (e *testEnv) storedHash(t *testing.T, username string) string {
|
|||||||
return user.Password
|
return user.Password
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- /s static group ---
|
|
||||||
|
|
||||||
// TestStaticServesEveryMethod pins what the static mount actually
|
|
||||||
// answers. chi's Mount registers the handler for all methods and
|
|
||||||
// http.FileServer only special-cases HEAD (by suppressing the body),
|
|
||||||
// so a POST or a DELETE to an asset is served the file rather than
|
|
||||||
// refused. The README documents this; the test is what keeps the two
|
|
||||||
// from drifting.
|
|
||||||
func TestStaticServesEveryMethod(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := newTestEnv(t)
|
|
||||||
|
|
||||||
body, err := static.Static.ReadFile("js/app.js")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotEmpty(t, body)
|
|
||||||
|
|
||||||
for _, method := range []string{
|
|
||||||
http.MethodGet,
|
|
||||||
http.MethodHead,
|
|
||||||
http.MethodPost,
|
|
||||||
http.MethodPut,
|
|
||||||
http.MethodDelete,
|
|
||||||
} {
|
|
||||||
t.Run(method, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(
|
|
||||||
context.Background(), method,
|
|
||||||
"/s/js/app.js", nil,
|
|
||||||
)
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
env.router.ServeHTTP(w, req)
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusOK, w.Code,
|
|
||||||
"static mount answers every method")
|
|
||||||
|
|
||||||
if method == http.MethodHead {
|
|
||||||
assert.Empty(t, w.Body.Bytes(),
|
|
||||||
"HEAD must not carry a body")
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, body, w.Body.Bytes(),
|
|
||||||
"the asset itself is returned")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- /pages group ---
|
// --- /pages group ---
|
||||||
|
|
||||||
// TestPagesLogin_OversizeBody_RejectedBeforeCSRF proves the cap runs
|
// TestPagesLogin_OversizeBody_RejectedBeforeCSRF proves the cap runs
|
||||||
|
|||||||
@@ -1,50 +0,0 @@
|
|||||||
package server_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"regexp"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/templates"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestBaseTemplateScriptsAreServed walks every /s/ script the base
|
|
||||||
// template loads on each page and fetches it through the real router.
|
|
||||||
// Alpine.js is fetched at build time rather than committed, so nothing
|
|
||||||
// in the repo guarantees it is present: this is the check that the page
|
|
||||||
// still gets the JavaScript it asks for.
|
|
||||||
func TestBaseTemplateScriptsAreServed(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// scriptSrc matches the src of every <script> tag pointing at the
|
|
||||||
// /s/ static mount.
|
|
||||||
scriptSrc := regexp.MustCompile(`<script[^>]+src="(/s/[^"]+)"`)
|
|
||||||
|
|
||||||
base, err := templates.Templates.ReadFile("base.html")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
matches := scriptSrc.FindAllStringSubmatch(string(base), -1)
|
|
||||||
require.NotEmpty(t, matches, "base.html should load scripts from /s/")
|
|
||||||
|
|
||||||
env := newTestEnv(t)
|
|
||||||
|
|
||||||
for _, m := range matches {
|
|
||||||
src := m[1]
|
|
||||||
t.Run(src, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
w := env.get(src, nil)
|
|
||||||
|
|
||||||
require.Equalf(
|
|
||||||
t, http.StatusOK, w.Code,
|
|
||||||
"base.html loads %s but the server does not serve it", src,
|
|
||||||
)
|
|
||||||
assert.NotEmptyf(
|
|
||||||
t, w.Body.Bytes(), "%s is served but empty", src,
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -5,8 +5,7 @@
|
|||||||
# or apk (detected in that order); assumes NOTHING is present (not git,
|
# or apk (detected in that order); assumes NOTHING is present (not git,
|
||||||
# make, or go). golangci-lint is packaged in nix, brew, and apk; on apt
|
# make, or go). golangci-lint is packaged in nix, brew, and apk; on apt
|
||||||
# it is installed from a hash-verified GitHub release archive (never
|
# it is installed from a hash-verified GitHub release archive (never
|
||||||
# curl | sh). Finishes by running script/fetch-assets, which installs the
|
# curl | sh).
|
||||||
# hash-pinned third-party browser assets the repo does not commit.
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
@@ -116,11 +115,6 @@ main() {
|
|||||||
|
|
||||||
go mod download
|
go mod download
|
||||||
|
|
||||||
# Third-party browser assets are not committed; fetch and verify them
|
|
||||||
# so a fresh clone can build and test.
|
|
||||||
if missing curl; then pkg_install curl curl curl curl; fi
|
|
||||||
"$ROOT/script/fetch-assets"
|
|
||||||
|
|
||||||
echo "bootstrap complete"
|
echo "bootstrap complete"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,104 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# script/fetch-assets: download the third-party browser assets the web UI
|
|
||||||
# ships and install them under static/. Minified bundles are not committed
|
|
||||||
# (REPO_POLICIES.md: no build artifacts in version control), so the build
|
|
||||||
# fetches them here. Every download is verified against a hardcoded sha256
|
|
||||||
# before it is installed, and any mismatch aborts. Idempotent: an asset
|
|
||||||
# already present with its pinned hash is left alone.
|
|
||||||
set -eu
|
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
|
||||||
|
|
||||||
# The sha256 of each installed asset lives in static/vendor.sha256, in
|
|
||||||
# sha256sum(1) format, with paths relative to static/. That file is the
|
|
||||||
# single source of truth: this script verifies against it, and
|
|
||||||
# static/vendor_test.go asserts the bytes embedded into the binary match
|
|
||||||
# it, so the hash cannot rot into a value nothing checks.
|
|
||||||
MANIFEST="static/vendor.sha256"
|
|
||||||
|
|
||||||
# Alpine.js 3.14.9, 2026-08-17. Fetched from registry.npmjs.org, the
|
|
||||||
# publisher of record; the jsDelivr and unpkg copies are mirrors of this
|
|
||||||
# same tarball. dist/cdn.min.js is the browser build Alpine publishes for
|
|
||||||
# a <script> tag.
|
|
||||||
ALPINE_VERSION="3.14.9"
|
|
||||||
ALPINE_URL="https://registry.npmjs.org/alpinejs/-/alpinejs-${ALPINE_VERSION}.tgz"
|
|
||||||
# sha256 of alpinejs-3.14.9.tgz
|
|
||||||
ALPINE_TARBALL_SHA256="97dad7c0c81e659cfc8e7700055da9770f8186187cb9a8a76efb57e00d5ce52a"
|
|
||||||
ALPINE_MEMBER="package/dist/cdn.min.js"
|
|
||||||
ALPINE_DEST="js/alpine.min.js"
|
|
||||||
|
|
||||||
sha256_of() {
|
|
||||||
if command -v sha256sum >/dev/null 2>&1; then
|
|
||||||
sha256sum "$1" | cut -d' ' -f1
|
|
||||||
else
|
|
||||||
shasum -a 256 "$1" | cut -d' ' -f1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# expected_sha256 <path-relative-to-static>
|
|
||||||
expected_sha256() {
|
|
||||||
awk -v want="$1" '$2 == want { print $1; found = 1 }
|
|
||||||
END { if (!found) exit 1 }' "$ROOT/$MANIFEST"
|
|
||||||
}
|
|
||||||
|
|
||||||
# verify <file> <expected-sha256> <what>
|
|
||||||
verify() {
|
|
||||||
actual="$(sha256_of "$1")"
|
|
||||||
if [ "$actual" != "$2" ]; then
|
|
||||||
echo "fetch-assets: sha256 mismatch for $3" >&2
|
|
||||||
echo " expected: $2" >&2
|
|
||||||
echo " actual: $actual" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# up_to_date <path-relative-to-static> <expected-sha256>
|
|
||||||
up_to_date() {
|
|
||||||
[ -f "$ROOT/static/$1" ] || return 1
|
|
||||||
[ "$(sha256_of "$ROOT/static/$1")" = "$2" ]
|
|
||||||
}
|
|
||||||
|
|
||||||
fetch_alpine() {
|
|
||||||
want="$(expected_sha256 "$ALPINE_DEST")"
|
|
||||||
|
|
||||||
if up_to_date "$ALPINE_DEST" "$want"; then
|
|
||||||
echo "fetch-assets: static/$ALPINE_DEST already at $want"
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "fetch-assets: fetching Alpine.js $ALPINE_VERSION from $ALPINE_URL"
|
|
||||||
tmp="$(mktemp -d)"
|
|
||||||
trap 'rm -rf "$tmp"' EXIT INT TERM
|
|
||||||
curl -fsSL -o "$tmp/alpine.tgz" "$ALPINE_URL"
|
|
||||||
verify "$tmp/alpine.tgz" "$ALPINE_TARBALL_SHA256" "alpinejs-${ALPINE_VERSION}.tgz"
|
|
||||||
tar -xzOf "$tmp/alpine.tgz" "$ALPINE_MEMBER" >"$tmp/alpine.min.js"
|
|
||||||
verify "$tmp/alpine.min.js" "$want" "$ALPINE_MEMBER from alpinejs-${ALPINE_VERSION}.tgz"
|
|
||||||
|
|
||||||
mkdir -p "$(dirname "$ROOT/static/$ALPINE_DEST")"
|
|
||||||
cp "$tmp/alpine.min.js" "$ROOT/static/$ALPINE_DEST"
|
|
||||||
rm -rf "$tmp"
|
|
||||||
trap - EXIT INT TERM
|
|
||||||
echo "fetch-assets: installed static/$ALPINE_DEST ($want)"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Re-check every manifest entry against what is now on disk, so an entry
|
|
||||||
# no script installs fails loudly instead of passing silently.
|
|
||||||
verify_manifest() {
|
|
||||||
while read -r want path; do
|
|
||||||
case "$want" in '' | '#'*) continue ;; esac
|
|
||||||
if [ ! -f "$ROOT/static/$path" ]; then
|
|
||||||
echo "fetch-assets: $MANIFEST lists static/$path, which is missing" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
verify "$ROOT/static/$path" "$want" "static/$path"
|
|
||||||
done <"$ROOT/$MANIFEST"
|
|
||||||
}
|
|
||||||
|
|
||||||
main() {
|
|
||||||
cd "$ROOT"
|
|
||||||
fetch_alpine
|
|
||||||
verify_manifest
|
|
||||||
echo "fetch-assets: all assets in $MANIFEST verified"
|
|
||||||
}
|
|
||||||
|
|
||||||
main "$@"
|
|
||||||
5
static/js/alpine.min.js
vendored
Normal file
5
static/js/alpine.min.js
vendored
Normal file
File diff suppressed because one or more lines are too long
@@ -1 +0,0 @@
|
|||||||
3ed1eed252488921df65e363d6715deb04d7f92aaedb9e52199fdf73cb1e0ad3 js/alpine.min.js
|
|
||||||
@@ -1,92 +0,0 @@
|
|||||||
package static_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/static"
|
|
||||||
)
|
|
||||||
|
|
||||||
const manifestPath = "vendor.sha256"
|
|
||||||
|
|
||||||
// fetchHint is appended to every failure here: the assets the manifest
|
|
||||||
// covers are fetched by the build, not committed, so a fresh clone that
|
|
||||||
// has not run script/fetch-assets fails this test and should be told why.
|
|
||||||
const fetchHint = "run `script/fetch-assets` (or `make assets`) to install " +
|
|
||||||
"the pinned third-party assets"
|
|
||||||
|
|
||||||
// TestVendoredAssetsMatchManifest asserts that every asset listed in
|
|
||||||
// static/vendor.sha256 is embedded in the binary with exactly the pinned
|
|
||||||
// bytes. script/fetch-assets verifies the same hashes at download time;
|
|
||||||
// this test verifies them again on what actually ships, so a build that
|
|
||||||
// skipped, cached, or subverted the fetch cannot produce a binary serving
|
|
||||||
// unpinned third-party JavaScript.
|
|
||||||
func TestVendoredAssetsMatchManifest(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
entries := readManifest(t)
|
|
||||||
require.NotEmpty(t, entries, "%s lists no assets", manifestPath)
|
|
||||||
|
|
||||||
for path, want := range entries {
|
|
||||||
t.Run(path, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
data, err := static.Static.ReadFile(path)
|
|
||||||
require.NoErrorf(
|
|
||||||
t, err,
|
|
||||||
"%s is listed in %s but is not embedded; %s",
|
|
||||||
path, manifestPath, fetchHint,
|
|
||||||
)
|
|
||||||
|
|
||||||
sum := sha256.Sum256(data)
|
|
||||||
got := hex.EncodeToString(sum[:])
|
|
||||||
require.Equalf(
|
|
||||||
t, want, got,
|
|
||||||
"embedded %s does not match its pinned sha256 in %s; %s",
|
|
||||||
path, manifestPath, fetchHint,
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// readManifest parses static/vendor.sha256, which is in sha256sum(1)
|
|
||||||
// format with paths relative to static/.
|
|
||||||
func readManifest(t *testing.T) map[string]string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
f, err := os.Open(manifestPath)
|
|
||||||
require.NoError(t, err, "opening %s", manifestPath)
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, f.Close()) }()
|
|
||||||
|
|
||||||
entries := make(map[string]string)
|
|
||||||
scanner := bufio.NewScanner(f)
|
|
||||||
|
|
||||||
for scanner.Scan() {
|
|
||||||
line := strings.TrimSpace(scanner.Text())
|
|
||||||
if line == "" || strings.HasPrefix(line, "#") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
fields := strings.Fields(line)
|
|
||||||
require.Lenf(
|
|
||||||
t, fields, 2,
|
|
||||||
"%s: malformed entry %q, want \"<sha256> <path>\"",
|
|
||||||
manifestPath, line,
|
|
||||||
)
|
|
||||||
|
|
||||||
sum, path := fields[0], fields[1]
|
|
||||||
require.Lenf(t, sum, 64, "%s: %q is not a sha256", manifestPath, sum)
|
|
||||||
entries[path] = sum
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, scanner.Err(), "reading %s", manifestPath)
|
|
||||||
|
|
||||||
return entries
|
|
||||||
}
|
|
||||||
@@ -37,9 +37,6 @@
|
|||||||
|
|
||||||
<div x-show="open" x-cloak class="mt-3 p-3 bg-gray-50 rounded-md">
|
<div x-show="open" x-cloak class="mt-3 p-3 bg-gray-50 rounded-md">
|
||||||
<pre class="text-xs text-gray-700 overflow-x-auto whitespace-pre-wrap break-all">{{.Body}}</pre>
|
<pre class="text-xs text-gray-700 overflow-x-auto whitespace-pre-wrap break-all">{{.Body}}</pre>
|
||||||
{{if .BodyTruncated}}
|
|
||||||
<p class="mt-2 text-xs text-gray-500">Body truncated for display: showing {{.BodyShownBytes}} of {{.BodyBytes}} bytes. The stored body is unchanged.</p>
|
|
||||||
{{end}}
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{{else}}
|
{{else}}
|
||||||
|
|||||||
Reference in New Issue
Block a user