Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
df2d9acd4d | ||
|
|
c0f7cc29fd | ||
|
|
f14fe02011 | ||
|
|
573f7f986a | ||
|
|
373a2f828d | ||
|
|
267d5b12f2 | ||
|
|
e73beb20df |
+15
-9
@@ -214,16 +214,22 @@ RUN version="$(make version VERSION="$VERSION")"; \
|
||||
# (Alpine.js, extracted from its tarball in 3p/) first.
|
||||
RUN make build VERSION="$VERSION"
|
||||
|
||||
# Rebuild with static linking for Alpine runtime.
|
||||
# make build already verified compilation.
|
||||
# The CGO binary from `make build` is dynamically linked against glibc,
|
||||
# which doesn't exist on Alpine (musl). Rebuild with static linking so
|
||||
# the binary runs on Alpine without glibc.
|
||||
# Rebuild with static linking for the Alpine runtime. The cgo binary the
|
||||
# step above builds is dynamically linked against glibc, which Alpine
|
||||
# (musl) does not have.
|
||||
#
|
||||
# The static flags go in through GO_LDFLAGS rather than a -ldflags of
|
||||
# their own: the build target composes them with the -X that stamps the
|
||||
# version, so this relink cannot silently drop the stamp.
|
||||
RUN CGO_ENABLED=1 make build VERSION="$VERSION" GO_LDFLAGS='-extldflags "-static"'
|
||||
# A statically linked glibc still loads shared libraries at run time for
|
||||
# getaddrinfo and dlopen, and Alpine has none to load, so the build tags
|
||||
# keep both out of the binary: netgo and osusergo select Go's own DNS
|
||||
# resolver and user lookup instead of glibc's, and
|
||||
# sqlite_omit_load_extension leaves out SQLite extension loading, which
|
||||
# webhooker never uses.
|
||||
#
|
||||
# The static flags and tags go in through GO_LDFLAGS and GO_TAGS rather
|
||||
# than a -ldflags of their own: the build target composes them with the
|
||||
# -X that stamps the version, so this relink cannot silently drop the
|
||||
# stamp.
|
||||
RUN CGO_ENABLED=1 make build VERSION="$VERSION" GO_TAGS=netgo,osusergo,sqlite_omit_load_extension GO_LDFLAGS='-extldflags "-static"'
|
||||
|
||||
# Runtime stage
|
||||
# alpine:3.21, 2026-03-17
|
||||
|
||||
@@ -16,14 +16,16 @@ VERSION ?= $(shell script/version)
|
||||
# exists to correct.
|
||||
override VERSION := $(or $(strip $(VERSION)),$(shell script/version))
|
||||
|
||||
# Extra linker flags for the build target. The static relink in the
|
||||
# Dockerfile adds -extldflags here rather than passing its own -ldflags,
|
||||
# so composing flags cannot drop the version stamp.
|
||||
# Extra linker flags and build tags for the build target. The static
|
||||
# relink in the Dockerfile passes -extldflags and its build tags through
|
||||
# these rather than a -ldflags of its own, so composing flags cannot drop
|
||||
# the version stamp.
|
||||
#
|
||||
# The build target itself always passes -trimpath and -s -w, as the Go
|
||||
# Dockerfile in REPO_POLICIES.md does: no build paths, symbol table or
|
||||
# debug information in the binary.
|
||||
GO_LDFLAGS ?=
|
||||
GO_TAGS ?=
|
||||
|
||||
bootstrap:
|
||||
@script/bootstrap
|
||||
@@ -53,7 +55,7 @@ check:
|
||||
@script/check
|
||||
|
||||
build: assets
|
||||
go build -trimpath -ldflags '$(strip -s -w -X main.version=$(VERSION) $(GO_LDFLAGS))' -o bin/webhooker ./cmd/webhooker
|
||||
go build -trimpath -tags '$(GO_TAGS)' -ldflags '$(strip -s -w -X main.version=$(VERSION) $(GO_LDFLAGS))' -o bin/webhooker ./cmd/webhooker
|
||||
|
||||
run: build
|
||||
./bin/webhooker
|
||||
|
||||
@@ -465,10 +465,7 @@ idle timestamp is rewritten at most once per tenth of the idle window rather
|
||||
than on every request, which means a session may expire up to 10% early relative
|
||||
to the user's true last request, but never late.
|
||||
|
||||
Both clocks are anchored by timestamps stored in the session cookie. Sessions
|
||||
issued before this feature existed carry neither, so they are treated as
|
||||
expired: upgrading to a build that has it logs every existing session out once,
|
||||
and those users sign in again.
|
||||
Both clocks are anchored by timestamps stored in the session cookie.
|
||||
|
||||
#### Invalid values abort startup
|
||||
|
||||
@@ -702,11 +699,11 @@ app as that user.
|
||||
`webhooker.db` holds target configuration in plaintext — bearer tokens, API
|
||||
keys, Slack webhook URLs — along with the session encryption key, so webhooker
|
||||
creates every SQLite file it owns `0600`: each database and both of its `-wal`
|
||||
and `-shm` sidecars, across all three tiers. Files an earlier build left `0644`
|
||||
are tightened when they are opened. The directory's `0750` is defence in depth —
|
||||
it stops other local users listing the directory and learning your webhook UUIDs
|
||||
from the `events-{uuid}.db` filenames — not the barrier protecting the
|
||||
credentials.
|
||||
and `-shm` sidecars, across all three tiers. Any of them found at another mode
|
||||
is set to `0600` when the database is opened. The directory's `0750` is defence
|
||||
in depth — it stops other local users listing the directory and learning your
|
||||
webhook UUIDs from the `events-{uuid}.db` filenames — not the barrier protecting
|
||||
the credentials.
|
||||
|
||||
### Running under upaas
|
||||
|
||||
@@ -805,7 +802,11 @@ comes up, serves pages, and is broken in a way nothing reports.
|
||||
proxy's log names it regardless. nginx's default `combined` format already
|
||||
logs `$remote_addr`; do not replace it with one that drops the client
|
||||
address, and retain those logs as long as you would want to answer a question
|
||||
about traffic.
|
||||
about traffic. That log also records the full request path, so it holds the
|
||||
entrypoint UUID of every event a sender posts. webhooker's own log never
|
||||
does. Protect the proxy's log as you would the entrypoint URLs themselves:
|
||||
anyone who can read it can post events to every entrypoint in it (see
|
||||
[The entrypoint URL is the authentication secret](#the-entrypoint-url-is-the-authentication-secret)).
|
||||
|
||||
### nginx
|
||||
|
||||
@@ -831,7 +832,9 @@ server {
|
||||
client_max_body_size 1m;
|
||||
|
||||
# $remote_addr is the client. webhooker's own log names it, as
|
||||
# clientIP, only while TRUSTED_PROXIES covers this proxy.
|
||||
# clientIP, only while TRUSTED_PROXIES covers this proxy. The
|
||||
# request path in this log holds the entrypoint UUID, so protect
|
||||
# the file like the entrypoint URL itself.
|
||||
access_log /var/log/nginx/webhooker.access.log combined;
|
||||
|
||||
location / {
|
||||
@@ -918,7 +921,7 @@ replaces the `max-age` it stored, so to end the policy sooner, set a short
|
||||
`HSTS_MAX_AGE` such as `1s` and keep it until returning browsers have picked it
|
||||
up, or have a reverse proxy send `max-age=0`.
|
||||
|
||||
## Backup, Restore, and Upgrades
|
||||
## Backup and Restore
|
||||
|
||||
### What to back up
|
||||
|
||||
@@ -1071,60 +1074,14 @@ beside it, or wait until there are none.
|
||||
|
||||
4. Start the service. The container gives the directory and the restored files
|
||||
to the `webhooker` user before the app starts, whoever restored them (see
|
||||
[Running with Docker](#running-with-docker)). Each restored database gets any
|
||||
of its SQL migrations it has not had as it is opened (see
|
||||
[Upgrades](#upgrades)).
|
||||
|
||||
### Upgrades
|
||||
|
||||
**Back up before every upgrade.** Each kind of database file has its schema in
|
||||
SQL files built into the binary, one directory per kind under
|
||||
`internal/db/migrations/`. In each, `000_migration.sql` creates the
|
||||
`schema_migrations` table, which records the number of every file the database
|
||||
has had, and `001_schema.sql` holds the whole schema. Opening a database runs,
|
||||
in one transaction, every file in its directory that `schema_migrations` does
|
||||
not list:
|
||||
|
||||
- `main/` for the main database, on connect — settings, users, API keys,
|
||||
webhooks, entrypoints and targets
|
||||
- `events/` for each event database, when it is lazily opened — events,
|
||||
deliveries, delivery results and their running totals
|
||||
- `archive/` for each archive database, on every open and reopen
|
||||
|
||||
There are no down migrations.
|
||||
|
||||
Upgrade procedure:
|
||||
|
||||
1. Stop the service.
|
||||
2. Back up `DATA_DIR` using one of the procedures above.
|
||||
3. Pull the new image and start it.
|
||||
4. Confirm `database migrations completed` in the logs before putting traffic
|
||||
back on it.
|
||||
5. Confirm the new build is the one running:
|
||||
`curl -s http://host:8080/.well-known/healthcheck` reports the version it was
|
||||
stamped with (see [Version stamping](#version-stamping)).
|
||||
|
||||
**Upgrading past the introduction of `BIND_ADDRESS`:** earlier versions always
|
||||
bound every interface. **Container deployments are unaffected** — the image
|
||||
ships `ENV BIND_ADDRESS=0.0.0.0`, so a `docker run` or Compose service that
|
||||
worked before still works with nothing changed.
|
||||
|
||||
A **bare binary** is the case that changes: the listener now binds `127.0.0.1`
|
||||
unless `BIND_ADDRESS` says otherwise, so a deployment that relied on reaching it
|
||||
from another host becomes unreachable until it sets the address the proxy
|
||||
connects to. Check the `bindAddress` field of the startup log to see what a
|
||||
running process bound. See [Bind address](#bind-address).
|
||||
|
||||
**Downgrade is unsupported.** Once a newer binary has migrated the files there
|
||||
is no way to move them back. The only supported way back to an older version is
|
||||
restoring the pre-upgrade backup, which discards everything received since that
|
||||
backup was taken.
|
||||
[Running with Docker](#running-with-docker)).
|
||||
|
||||
### Version stamping
|
||||
|
||||
The binary reports its version at `/.well-known/healthcheck` (the `version`
|
||||
field), in the UI footer, and in the startup log line (`msg=starting`,
|
||||
`version=...`). It is also the Sentry release name, as `webhooker-{version}`.
|
||||
`curl -s http://host:8080/.well-known/healthcheck` shows which build is running.
|
||||
The value is stamped in at build time by the linker; it is not read from a file
|
||||
at runtime, so it identifies the build itself.
|
||||
|
||||
@@ -1209,7 +1166,12 @@ What that means for an operator:
|
||||
|
||||
- **The URL is a capability, so treat it as a secret.** Keep it out of logs,
|
||||
ticket bodies, chat messages and screenshots. Anyone who reads it anywhere can
|
||||
post events as that sender.
|
||||
post events as that sender. webhooker's own log lines never carry the UUID:
|
||||
they name a request to the receiver by the route pattern `/h/{uuid}`, and the
|
||||
`webhook request received` line names the entrypoint by its `entrypoint_id`.
|
||||
The reverse proxy's access log does carry the UUID, in the request path, so it
|
||||
must be protected like the URL itself (see
|
||||
[Deployment behind a reverse proxy](#deployment-behind-a-reverse-proxy)).
|
||||
- **Rotating means minting a new entrypoint, not changing a key.** There is no
|
||||
way to rotate the UUID in place. To retire one, delete the entrypoint (or
|
||||
deactivate it, which answers `410`) and create a new one, then point the
|
||||
@@ -1366,20 +1328,22 @@ webhooker solves this by acting as a durable intermediary:
|
||||
nothing is lost if downstream targets are unavailable.
|
||||
|
||||
2. **Guaranteed delivery** — Events are queued for delivery to each configured
|
||||
target. Failed deliveries are retried with configurable backoff. Every
|
||||
delivery attempt is logged with status codes, response bodies, and timing.
|
||||
target. Failed deliveries are retried with configurable backoff. Delivery
|
||||
attempts are logged with status codes, response bodies, and timing.
|
||||
|
||||
**That guarantee is at-least-once, not exactly-once.** When a send reaches
|
||||
its target but the write recording that outcome fails, the delivery is
|
||||
deliberately left in a recoverable state rather than marked done — losing a
|
||||
delivery is the worse failure — so the pending sweep picks it up about
|
||||
fifteen minutes later, or the next restart does, and the target receives a
|
||||
payload it already got. webhooker adds no delivery identifier of its own to
|
||||
an outbound request, so **make your receiver idempotent** against whatever
|
||||
the payload itself carries.
|
||||
payload it already got. A send still in flight when the service stops is
|
||||
left the same way: its attempt is not recorded, and the next start sends it
|
||||
again (see Shutdown). webhooker adds no delivery identifier of its own to an
|
||||
outbound request, so **make your receiver idempotent** against whatever the
|
||||
payload itself carries.
|
||||
|
||||
3. **Observability** — Full request/response logging for every webhook received
|
||||
and every delivery attempted. Prometheus metrics expose volume, latency, and
|
||||
and for delivery attempts. Prometheus metrics expose volume, latency, and
|
||||
error rates. The web UI provides real-time visibility into event flow.
|
||||
|
||||
4. **Fan-out** — A single incoming webhook can be delivered to multiple targets
|
||||
@@ -1583,8 +1547,7 @@ The cap is not arbitrary: the reaper computes its cutoff as a `time.Duration`,
|
||||
an int64 nanosecond count, and a longer period overflows it. An overflowed
|
||||
cutoff lands in the future, where it matches every row, so the sweep would
|
||||
delete every event the webhook has instead of none. The reaper also clamps the
|
||||
value it is given, so a row written by an older version cannot trigger that
|
||||
either.
|
||||
value it reads to the cap, so no stored value can trigger that either.
|
||||
|
||||
#### Entrypoint
|
||||
|
||||
@@ -1818,7 +1781,7 @@ is what bounds it.
|
||||
#### DeliveryResult
|
||||
|
||||
The result of a single delivery attempt. Every attempt (including retries) is
|
||||
individually logged for full observability.
|
||||
individually logged, except one cut off by a stop (see Shutdown).
|
||||
|
||||
| Field | Type | Description |
|
||||
| --------------- | ------- | ----------------------------------------- |
|
||||
@@ -1967,7 +1930,7 @@ and application state:
|
||||
- **APIKeys** — programmatic access credentials
|
||||
|
||||
On first startup the main database is created from its SQL files (see
|
||||
[Upgrades](#upgrades)), a session encryption key is generated and stored, and an
|
||||
[Schema](#schema)), a session encryption key is generated and stored, and an
|
||||
`admin` user is created.
|
||||
|
||||
**Per-Webhook Event Databases** (`{DATA_DIR}/events-{webhook_uuid}.db`) — each
|
||||
@@ -2164,6 +2127,21 @@ The database uses the
|
||||
though CGO is required at build time due to the transitive `mattn/go-sqlite3`
|
||||
dependency from `gorm.io/driver/sqlite`.
|
||||
|
||||
#### Schema
|
||||
|
||||
Each kind of database file has its schema in SQL files built into the binary,
|
||||
one directory per kind under `internal/db/migrations/`. In each,
|
||||
`000_migration.sql` creates the `schema_migrations` table, which records the
|
||||
number of every file the database has had, and `001_schema.sql` holds the whole
|
||||
schema. Opening a database runs, in one transaction, every file in its directory
|
||||
that `schema_migrations` does not list:
|
||||
|
||||
- `main/` for the main database, on connect — settings, users, API keys,
|
||||
webhooks, entrypoints and targets
|
||||
- `events/` for each event database, when it is lazily opened — events,
|
||||
deliveries, delivery results and their running totals
|
||||
- `archive/` for each archive database, on every open and reopen
|
||||
|
||||
### Request Flow
|
||||
|
||||
```
|
||||
@@ -2241,12 +2219,16 @@ This means:
|
||||
buffer rather than spawning more goroutines.
|
||||
- **Independent results** — each worker records its own delivery result in the
|
||||
per-webhook database without coordination.
|
||||
- **Graceful shutdown** — cancel the context, workers finish their current task
|
||||
and exit. The stop hook waits for the pool via `lifecycle.WaitForShutdown`,
|
||||
which bounds that wait by fx's stop timeout rather than blocking forever on a
|
||||
wedged worker. On timeout it logs at `ERROR` and returns an error, and the
|
||||
goroutines that did not finish are still running — an unclean shutdown is
|
||||
reported rather than hidden.
|
||||
- **Graceful shutdown** — the stop hook cancels the workers' context. A worker
|
||||
may still take tasks already queued before it exits. An `http` or `slack`
|
||||
request in flight is cut off, and one started after the cancel fails at once.
|
||||
Neither is recorded or counted in the metrics, and the next start sends the
|
||||
delivery again (see Shutdown). A `database` or `log` task runs as normal. The
|
||||
stop hook waits for the pool via `lifecycle.WaitForShutdown`, which bounds
|
||||
that wait by fx's stop timeout rather than blocking forever on a wedged
|
||||
worker. On timeout it logs at `ERROR` and returns an error, and the goroutines
|
||||
that did not finish are still running — an unclean shutdown is reported rather
|
||||
than hidden.
|
||||
|
||||
**Recovery paths:**
|
||||
|
||||
@@ -2352,7 +2334,7 @@ getting anywhere.
|
||||
| Metric | Type | Meaning |
|
||||
| -------------------------------------- | --------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `webhooker_events_received_total` | counter | Events received and durably stored. Compare against the delivery counters on one dashboard |
|
||||
| `webhooker_delivery_attempts_total` | counter | Delivery attempts actually dispatched to a target. A delivery a circuit breaker refused is not one: it is counted as a retry instead, but only when the refusal moves it into `retrying` |
|
||||
| `webhooker_delivery_attempts_total` | counter | Delivery attempts actually dispatched to a target. A delivery a circuit breaker refused is not one: it is counted as a retry instead, but only when the refusal moves it into `retrying`. An attempt cut off by a stop is not counted (see Shutdown) |
|
||||
| `webhooker_deliveries_succeeded_total` | counter | Deliveries that reached `delivered` |
|
||||
| `webhooker_deliveries_failed_total` | counter | Deliveries that failed terminally and will not be retried |
|
||||
| `webhooker_delivery_retries_total` | counter | Deliveries put back into `retrying` |
|
||||
@@ -2451,27 +2433,32 @@ log volume it caps rather than eliminates. A path that names no entrypoint is
|
||||
recorded by the handler at `DEBUG`, and the aggregate limiter logs its own
|
||||
rejections at `DEBUG` and without the path, so neither appears at all under the
|
||||
default level. The per-entrypoint limiter is the loud one: it logs every
|
||||
rejection at `WARN` with the request path, which on this route is
|
||||
attacker-controlled text. A client hammering a single invented path is served
|
||||
rejection at `WARN`, naming the route pattern `/h/{uuid}` rather than the
|
||||
request path. A client hammering a single invented path is served
|
||||
`RECEIVER_RATE_LIMIT` requests and has the rest of its aggregate budget rejected
|
||||
there, so the aggregate limit is what bounds the _number_ of those `WARN` lines
|
||||
— to under ten times `RECEIVER_RATE_LIMIT` per minute per client IP, 1080 at the
|
||||
defaults, where before it there was no bound at all. Their _width_ is bounded by
|
||||
the field budgets below, the same ones the access log spends. The access log is
|
||||
bounded by neither limit: every request is recorded once at `INFO`, served or
|
||||
rejected alike.
|
||||
defaults, where before it there was no bound at all. The access log is bounded
|
||||
by neither limit: every request is recorded once at `INFO`, served or rejected
|
||||
alike.
|
||||
|
||||
What the access log does bound is the _content_ of those lines. A 3xx or 4xx
|
||||
response logs the chi route pattern — `/h/{uuid}`, `/user/{username}//`, or the
|
||||
literal `(unmatched)` when the request hit no route at all — in place of the
|
||||
concrete URL. Those are the outcomes an unauthenticated client can drive for
|
||||
free: 404 and 429 on any invented receiver path, a login redirect on any
|
||||
invented profile path. Logging the URL there would let a flood write text of its
|
||||
own choosing, at a length of its own choosing, into the log. 2xx and 5xx
|
||||
responses keep the concrete path — a success resolved against a static route or
|
||||
against the operator's own data (on the receiver, against a stored entrypoint
|
||||
UUID), and a 5xx is a bug in this service, where the exact path is the evidence
|
||||
and no client can provoke one at will.
|
||||
What the access log does bound is the _content_ of those lines. On the receiver
|
||||
route it logs the route pattern `/h/{uuid}` whatever the status, because the
|
||||
concrete path carries the entrypoint UUID, which is the entrypoint's credential
|
||||
(see
|
||||
[The entrypoint URL is the authentication secret](#the-entrypoint-url-is-the-authentication-secret)).
|
||||
A request chi never routed, because it matched no route or because a middleware
|
||||
answered it first, logs the literal `(unmatched)` whatever the status. In the
|
||||
`dev` environment the CORS middleware answers a preflight that way, so a
|
||||
preflight to the receiver does not log its UUID either. On every other route, a
|
||||
3xx or 4xx response logs the chi route pattern, such as `/user/{username}//`, in
|
||||
place of the concrete URL. Those are the outcomes an unauthenticated client can
|
||||
drive for free: a 404 on any invented path, a login redirect on any invented
|
||||
profile path. Logging the URL there would let a flood write text of its own
|
||||
choosing, at a length of its own choosing, into the log. 2xx and 5xx responses
|
||||
keep the concrete path — a success resolved against a static route or against
|
||||
the operator's own data, and a 5xx is a bug in this service, where the exact
|
||||
path is the evidence and no client can provoke one at will.
|
||||
|
||||
The query string is never logged; it is replaced by the fixed marker
|
||||
`?(redacted)`. It is client-chosen on every route, and
|
||||
@@ -2497,9 +2484,10 @@ The same hook rewrites the request URL. The SDK builds it as
|
||||
`scheme://host/path` from the concrete path, which on the receiver route is
|
||||
`/h/<uuid>` in full — and that UUID is a write capability, not an identifier:
|
||||
anyone holding it can post events this service accepts and its targets then
|
||||
deliver. A tracker has its own retention, access control and deletion policy, so
|
||||
the rule the access log follows above does not carry across that boundary. What
|
||||
is sent is the chi route pattern instead: `http://host/h/{uuid}`.
|
||||
deliver. What is sent is the chi route pattern instead: `http://host/h/{uuid}`.
|
||||
A tracker has its own retention, access control and deletion policy, so the
|
||||
pattern is sent on every route, including the 2xx and 5xx responses outside the
|
||||
receiver for which the access log keeps the concrete path.
|
||||
|
||||
The scheme and the host are kept, and everything else in the URL is discarded
|
||||
rather than edited, so a future SDK version that starts appending a query string
|
||||
@@ -2628,9 +2616,7 @@ than it:
|
||||
| ------------------------------------------ | ------- | ------------------- | ------------------------------------------ |
|
||||
| `request body exceeds limit` (413) | `WARN` | path, method | yes — `MaxBodySize` precedes `RequireAuth` |
|
||||
| `csrf: token validation failed` (403) | `WARN` | path, method | yes — `CSRF` precedes `RequireAuth` |
|
||||
| `... rate limit exceeded` (429) | `WARN` | path | yes, on the receiver |
|
||||
| `auth middleware: unauthenticated request` | `DEBUG` | path, method | yes, by definition |
|
||||
| `entrypoint not found` | `DEBUG` | entrypoint UUID | yes, on the receiver |
|
||||
| `user not found` / `invalid password` | `DEBUG` | username | yes, on the sign-in form |
|
||||
| `login failure limit exceeded` (429) | `WARN` | path | yes, on the sign-in form |
|
||||
| `password verification capacity exhausted` | `WARN` | path | yes, on the sign-in form |
|
||||
@@ -2658,13 +2644,12 @@ wider of the two handlers spends more on than the client spent sending them.
|
||||
Every case holds each line to the 2,560-byte ceiling. That per-line ceiling is
|
||||
what the figure above states, and every row establishes it.
|
||||
|
||||
Three of the sites go further and bound the whole flood's output — the total
|
||||
bytes a run of distinct invented values wrote, which is the shape an operator
|
||||
sizing storage cares about. They are `request body exceeds limit`
|
||||
(`TestMaxBodySize_FloodOfOversizePathsDoesNotGrowTheLog`),
|
||||
`entrypoint not found` and `user not found` (the last two through
|
||||
`assertBoundedFlood`). The other rows carry no aggregate assertion; the per-line
|
||||
ceiling is what they establish.
|
||||
Two of the sites go further and bound the whole flood's output — the total bytes
|
||||
a run of distinct invented values wrote, which is the shape an operator sizing
|
||||
storage cares about. They are `request body exceeds limit`
|
||||
(`TestMaxBodySize_FloodOfOversizePathsDoesNotGrowTheLog`) and `user not found`
|
||||
(through `assertBoundedFlood`). The other rows carry no aggregate assertion; the
|
||||
per-line ceiling is what they establish.
|
||||
|
||||
`internal/logfield/logfield_test.go` measures the per-rune charge against what
|
||||
the handlers really emit, over roughly 3,000 code points on each, so an
|
||||
@@ -2682,7 +2667,7 @@ everything else, so its lines take the level the operator set and the handler
|
||||
`internal/logger` selected, and every value it emits is spent through the same
|
||||
`internal/logfield` budget. A record-not-found is not logged as an error: it is
|
||||
the expected outcome on both of those paths, and each handler already records
|
||||
its own miss at `DEBUG` — bounded, per the table above — without the SQL. Slow
|
||||
its own miss at `DEBUG`, without the SQL and within the ceiling above. Slow
|
||||
statements are kept, at `WARN`, above the same 200 ms threshold GORM used and
|
||||
with the statement bounded, because that report is the one thing GORM's logger
|
||||
gave an operator that nothing else here does. The adapter orders its cases
|
||||
@@ -3319,7 +3304,8 @@ stop-hook log:
|
||||
3. `server` — the HTTP drain, bounded by `server.ShutdownTimeout` (**3
|
||||
seconds**) and by what the hooks before it left, then a Sentry flush if
|
||||
`SENTRY_DSN` is set
|
||||
4. `delivery.Engine` — waits for its workers, then closes the archive databases
|
||||
4. `delivery.Engine` — cuts off any `http` or `slack` request in flight, waits
|
||||
for its workers, then closes the archive databases
|
||||
5. `healthcheck`
|
||||
6. `WebhookDBManager`
|
||||
7. the database close
|
||||
@@ -3330,6 +3316,12 @@ by the stop context, so a wedge there consumes the 5 seconds before the HTTP
|
||||
server hook is ever entered. The hooks after the server are microsecond-scale in
|
||||
normal operation.
|
||||
|
||||
The delivery engine cancels a delivery request in flight. That attempt is not
|
||||
recorded or counted in the metrics, and the delivery keeps its `pending` or
|
||||
`retrying` status, so startup recovery sends it again on the next start (see
|
||||
Recovery paths under Bounded Worker Pool). If the receiver already had it, it
|
||||
receives it twice: the at-least-once duplicate described under Rationale.
|
||||
|
||||
The HTTP drain budget is deliberately **shorter** than the sequence budget. Were
|
||||
the two equal, a drain that used its whole budget would exhaust the sequence
|
||||
budget at the instant it finished, and every later hook — the delivery engine,
|
||||
@@ -3454,12 +3446,14 @@ fixed independently of the compiler's:
|
||||
phases and the `css-check` and `markdown-check` stages passing (it copies a
|
||||
file from each), runs `make build` (which extracts Alpine.js from `3p/`
|
||||
first), and then rebuilds the binary with `CGO_ENABLED=1` and static linking
|
||||
so it runs on musl. Both builds go through `make build`, the relink adding
|
||||
its `-extldflags` via `GO_LDFLAGS`, so neither can drop the `-X` that stamps
|
||||
the version. The version is the `VERSION` build arg if one is given,
|
||||
otherwise derived from the `.git` in the context, and the stage fails if a
|
||||
context with `.git` would stamp an empty version, `dev` or `unknown` (see
|
||||
[Version stamping](#version-stamping)).
|
||||
so it runs on musl, with the build tags `netgo`, `osusergo` and
|
||||
`sqlite_omit_load_extension` so it needs none of glibc's shared libraries at
|
||||
run time. Both builds go through `make build`, the relink adding its
|
||||
`-extldflags` via `GO_LDFLAGS` and its tags via `GO_TAGS`, so neither can
|
||||
drop the `-X` that stamps the version. The version is the `VERSION` build arg
|
||||
if one is given, otherwise derived from the `.git` in the context, and the
|
||||
stage fails if a context with `.git` would stamp an empty version, `dev` or
|
||||
`unknown` (see [Version stamping](#version-stamping)).
|
||||
7. **Runtime stage** (`alpine:3.21`) — copies the static binary and
|
||||
`deploy/docker-entrypoint.sh`, creates the `/var/lib/webhooker` directory for
|
||||
all SQLite databases, exposes port 8080, and includes a health check against
|
||||
|
||||
@@ -45,8 +45,7 @@ type Event struct {
|
||||
|
||||
// ResubmittedFromID names the event this one was copied from by
|
||||
// an operator resubmit. It is nil for an event that arrived on
|
||||
// the receiver, which is every event created before the column
|
||||
// existed. It is not a foreign key: the source event can be
|
||||
// the receiver. It is not a foreign key: the source event can be
|
||||
// reaped by retention while its copies remain, and the id is
|
||||
// kept as the record of where the copy came from either way.
|
||||
ResubmittedFromID *string `gorm:"type:uuid;index:idx_events_resubmitted_from_id,priority:1;index:idx_events_entrypoint_id,priority:3" json:"resubmittedFromId,omitempty"`
|
||||
|
||||
@@ -82,7 +82,7 @@ type Webhook struct {
|
||||
// value while building the insert statement, which runs after
|
||||
// BeforeSave; rewriting any later than this loses that race and the
|
||||
// row lands at 30 days. Living on the model also means a future call
|
||||
// site — a REST API, a fixture, a migration — cannot bypass it.
|
||||
// site, such as a REST API or a fixture, cannot bypass it.
|
||||
func (w *Webhook) BeforeSave(_ *gorm.DB) error {
|
||||
if w.RetentionDays <= 0 {
|
||||
w.RetentionDays = RetentionForeverDays
|
||||
@@ -98,8 +98,8 @@ func (w *Webhook) BeforeSave(_ *gorm.DB) error {
|
||||
// are exempt from reaping.
|
||||
//
|
||||
// It accepts the RetentionForeverDays sentinel written by BeforeSave
|
||||
// and, defensively, the non-positive values that rows written before
|
||||
// the sentinel existed may still carry.
|
||||
// and, defensively, any non-positive value, which BeforeSave treats
|
||||
// as a request to retain forever.
|
||||
func retainsForever(retentionDays int) bool {
|
||||
return retentionDays <= 0 ||
|
||||
retentionDays >= RetentionForeverDays
|
||||
|
||||
@@ -253,7 +253,7 @@ func (r *RetentionReaper) reapWebhook(
|
||||
// row in the database: the sweep then deletes every event, delivery,
|
||||
// and delivery result, including ones created seconds ago. Rejecting
|
||||
// out-of-range input at the form is the primary guard; saturating here
|
||||
// means an old row, a migration, or a future call site cannot turn a
|
||||
// means a value that reaches the row by any other path cannot turn a
|
||||
// too-large retention into total data loss.
|
||||
func retentionCutoff(
|
||||
now time.Time,
|
||||
|
||||
@@ -118,9 +118,8 @@ const SQLiteFilePerm fs.FileMode = 0o600
|
||||
// SQLite still reports the absence rather than this function
|
||||
// materializing an empty database the caller asked not to create.
|
||||
//
|
||||
// Chmod of a file that already exists is what tightens a data
|
||||
// directory an earlier build left at 0644 — including a developer's
|
||||
// own scratch directory — without any migration machinery.
|
||||
// The mode is set on every open, whoever created the file, so a
|
||||
// database or sidecar found at a wider mode is tightened too.
|
||||
func reserveSQLiteFile(path string, create bool) error {
|
||||
if create {
|
||||
// gosec G304: the path is the database file the caller asked
|
||||
@@ -242,7 +241,19 @@ func SQLiteDSN(path, mode string) string {
|
||||
)
|
||||
q.Add("_pragma", "journal_mode(WAL)")
|
||||
|
||||
return "file:" + path + "?" + q.Encode()
|
||||
// SQLite reads a file: URI's path only up to the first ? or #, and
|
||||
// decodes %XX escapes in it, so url.URL escapes the path. OmitHost
|
||||
// keeps a relative path as "file:data/x.db"; written
|
||||
// "file://data/x.db", SQLite would reject "data" as the URI's
|
||||
// authority.
|
||||
dsn := url.URL{
|
||||
Scheme: "file",
|
||||
Path: path,
|
||||
OmitHost: true,
|
||||
RawQuery: q.Encode(),
|
||||
}
|
||||
|
||||
return dsn.String()
|
||||
}
|
||||
|
||||
// OpenSQLite opens the SQLite file at path with the service's
|
||||
|
||||
@@ -2,6 +2,8 @@ package database_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -112,6 +114,79 @@ func TestMainDBAppliesPragmasOnALiveHandle(t *testing.T) {
|
||||
assert.Equal(t, "10000", busy)
|
||||
}
|
||||
|
||||
// TestOpenSQLiteOpensThePathAskedFor uses a data directory whose name
|
||||
// holds ?, # and %. SQLite reads a file: URI's path only up to a ? or
|
||||
// #, and decodes %41 to A, so an unescaped path sent every database in
|
||||
// that directory to one file outside it.
|
||||
func TestOpenSQLiteOpensThePathAskedFor(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := filepath.Join(t.TempDir(), "hooks#1?x%41")
|
||||
require.NoError(t, os.Mkdir(dir, 0o700))
|
||||
|
||||
for _, name := range []string{
|
||||
database.MainDBFileName,
|
||||
"events-" + uuid.New().String() + ".db",
|
||||
} {
|
||||
path := filepath.Join(dir, name)
|
||||
|
||||
sqlDB, err := database.OpenSQLite(
|
||||
path, database.SQLiteModeCreate,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Cleanup(func() { assert.NoError(t, sqlDB.Close()) })
|
||||
|
||||
assertOpenedFile(t, sqlDB, path)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOpenSQLiteOpensARelativePath opens a database at a path relative
|
||||
// to the working directory, which is what a relative DATA_DIR such as
|
||||
// the README's DATA_DIR=../webhooker-data gives OpenSQLite.
|
||||
//
|
||||
//nolint:paralleltest // t.Chdir moves the whole process.
|
||||
func TestOpenSQLiteOpensARelativePath(t *testing.T) {
|
||||
t.Chdir(t.TempDir())
|
||||
require.NoError(t, os.Mkdir("data", 0o700))
|
||||
|
||||
path := filepath.Join("data", database.MainDBFileName)
|
||||
|
||||
sqlDB, err := database.OpenSQLite(path, database.SQLiteModeCreate)
|
||||
require.NoError(t, err)
|
||||
|
||||
defer func() { require.NoError(t, sqlDB.Close()) }()
|
||||
|
||||
assertOpenedFile(t, sqlDB, path)
|
||||
}
|
||||
|
||||
// assertOpenedFile checks that sqlDB is the SQLite file at path.
|
||||
// database_list names the file SQLite actually opened, as an absolute
|
||||
// path with symlinks resolved, so the files are compared, not the names.
|
||||
func assertOpenedFile(t *testing.T, sqlDB *sql.DB, path string) {
|
||||
t.Helper()
|
||||
|
||||
var (
|
||||
seq int
|
||||
schema, file string
|
||||
)
|
||||
|
||||
require.NoError(t, sqlDB.
|
||||
QueryRowContext(t.Context(), "pragma database_list").
|
||||
Scan(&seq, &schema, &file))
|
||||
|
||||
asked, err := os.Stat(path)
|
||||
require.NoError(t, err)
|
||||
|
||||
opened, err := os.Stat(file)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.True(
|
||||
t, os.SameFile(asked, opened),
|
||||
"asked for %s, SQLite opened %s", path, file,
|
||||
)
|
||||
}
|
||||
|
||||
// TestConcurrentReaderDoesNotBlockWrites is the unit-scale form of the
|
||||
// reproduction in
|
||||
// https://git.eeqj.de/sneak/webhooker/issues/256: an operator's
|
||||
|
||||
@@ -201,8 +201,7 @@ func (s *ArchiveSweeper) sweep(ctx context.Context) {
|
||||
|
||||
// sweepTarget prunes the archive of a single database target.
|
||||
// A missing, empty, or "never" expiry parses as a zero duration
|
||||
// and is skipped entirely, so those archives keep exactly the
|
||||
// behaviour they had before the sweep existed.
|
||||
// and is skipped entirely, so those archives are never pruned.
|
||||
func (s *ArchiveSweeper) sweepTarget(target *database.Target) {
|
||||
expiry, err := parseArchiveExpiry(target.Config)
|
||||
if err != nil {
|
||||
|
||||
@@ -32,8 +32,9 @@ func targetTotals(
|
||||
|
||||
// TestUpdateDeliveryStatus_FinishTimeAndTargetTotals pins what a status
|
||||
// write records for the webhook page's statistics: the time a delivery
|
||||
// finished, set only when it becomes delivered or failed, and one more
|
||||
// on its target's delivered or failed total.
|
||||
// finished, set only when it becomes delivered or failed and stored in
|
||||
// the same form as created_at, and one more on its target's delivered
|
||||
// or failed total.
|
||||
func TestUpdateDeliveryStatus_FinishTimeAndTargetTotals(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -75,6 +76,16 @@ func TestUpdateDeliveryStatus_FinishTimeAndTargetTotals(t *testing.T) {
|
||||
if tt.finished {
|
||||
require.NotNil(t, stored.FinishedAt)
|
||||
assert.False(t, stored.FinishedAt.Before(before))
|
||||
|
||||
var createdAt, finishedAt string
|
||||
|
||||
require.NoError(t, db.Raw(
|
||||
`SELECT CAST(created_at AS TEXT),
|
||||
CAST(finished_at AS TEXT)
|
||||
FROM deliveries WHERE id = ?`, d.ID,
|
||||
).Row().Scan(&createdAt, &finishedAt))
|
||||
assertStoredTime(t, createdAt)
|
||||
assertStoredTime(t, finishedAt)
|
||||
} else {
|
||||
assert.Nil(t, stored.FinishedAt)
|
||||
}
|
||||
|
||||
@@ -1701,8 +1701,11 @@ func writeDeliveryStatus(
|
||||
database.DeliveryStatusFailed,
|
||||
}).
|
||||
Updates(map[string]any{
|
||||
"status": status,
|
||||
"finished_at": time.Now(),
|
||||
"status": status,
|
||||
// GORM's clock, the one that stamps created_at. A bare
|
||||
// time.Now() carries a monotonic clock reading, which the
|
||||
// driver would store as an " m=+…" suffix.
|
||||
"finished_at": tx.NowFunc(),
|
||||
})
|
||||
if res.Error != nil || res.RowsAffected == 0 {
|
||||
return res.Error
|
||||
@@ -1906,12 +1909,13 @@ func (e *Engine) takeForRedispatch(
|
||||
return false
|
||||
}
|
||||
|
||||
// GORM's clock, as writeDeliveryStatus explains.
|
||||
res := webhookDB.
|
||||
Model(&database.Delivery{}).
|
||||
Where(
|
||||
"id = ? AND status = ?", deliveryID, eligible,
|
||||
).
|
||||
UpdateColumn("updated_at", time.Now())
|
||||
UpdateColumn("updated_at", webhookDB.NowFunc())
|
||||
|
||||
if res.Error != nil {
|
||||
e.log.Error(
|
||||
|
||||
@@ -2,11 +2,16 @@ package delivery_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/fx"
|
||||
"gorm.io/gorm/clause"
|
||||
@@ -272,6 +277,154 @@ func TestEngine_StopHookHonoursStopTimeout(t *testing.T) {
|
||||
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
||||
}
|
||||
|
||||
// seedHeldDelivery gives the setup's webhook an http target with
|
||||
// maxRetries, pointed at a receiver that keeps every request open until
|
||||
// the client gives up on it, and seeds one delivery to that target at
|
||||
// status. It returns the task that sends attempt attemptNum of the
|
||||
// delivery, and a channel that is closed when a request reaches the
|
||||
// receiver.
|
||||
func seedHeldDelivery(
|
||||
t *testing.T,
|
||||
s iSetup,
|
||||
maxRetries int,
|
||||
status database.DeliveryStatus,
|
||||
attemptNum int,
|
||||
) (delivery.Task, <-chan struct{}) {
|
||||
t.Helper()
|
||||
|
||||
arrived := make(chan struct{})
|
||||
|
||||
var once sync.Once
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(
|
||||
func(_ http.ResponseWriter, r *http.Request) {
|
||||
// The server notices the client hanging up only once the
|
||||
// body has been read; until then r.Context() never ends.
|
||||
_, _ = io.Copy(io.Discard, r.Body)
|
||||
|
||||
once.Do(func() { close(arrived) })
|
||||
|
||||
<-r.Context().Done()
|
||||
},
|
||||
))
|
||||
t.Cleanup(ts.Close)
|
||||
|
||||
cfg := iHTTPConfig(ts.URL)
|
||||
targetID := uuid.New().String()
|
||||
|
||||
iCreateWebhook(t, s.MainDB, s.WebhookID, "hook")
|
||||
iCreateTarget(
|
||||
t, s.MainDB, targetID, s.WebhookID, "held",
|
||||
database.TargetTypeHTTP, cfg, maxRetries,
|
||||
)
|
||||
|
||||
event := iSeedEvent(t, s.WebhookDB, s.WebhookID, `{"held":true}`)
|
||||
d := iSeedDelivery(t, s.WebhookDB, event.ID, targetID, status)
|
||||
|
||||
bodyStr := event.Body
|
||||
|
||||
return iTask(
|
||||
d, event, s.WebhookID, targetID, "held", cfg,
|
||||
maxRetries, attemptNum, &bodyStr,
|
||||
), arrived
|
||||
}
|
||||
|
||||
// stopDuringAttempt waits until the receiver holds an attempt open, then
|
||||
// stops the engine through its real stop hook.
|
||||
func stopDuringAttempt(
|
||||
t *testing.T, lc *recordingLifecycle, arrived <-chan struct{},
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
select {
|
||||
case <-arrived:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the delivery attempt never reached the receiver")
|
||||
}
|
||||
|
||||
require.NoError(t, lc.hooks[0].OnStop(context.Background()))
|
||||
}
|
||||
|
||||
// requireRequeued runs restart recovery on the stopped engine and
|
||||
// requires it to queue task's attempt again on ch.
|
||||
func requireRequeued(
|
||||
t *testing.T, s iSetup, ch <-chan delivery.Task, task delivery.Task,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
s.Engine.ExportRecoverWebhookDeliveries(
|
||||
context.Background(), s.WebhookID,
|
||||
)
|
||||
|
||||
select {
|
||||
case got := <-ch:
|
||||
assert.Equal(t, task.DeliveryID, got.DeliveryID)
|
||||
assert.Equal(t, task.AttemptNum, got.AttemptNum)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("restart recovery did not queue the delivery again")
|
||||
}
|
||||
}
|
||||
|
||||
// TestEngine_StopLeavesFireAndForgetAttemptPending is the regression
|
||||
// test for a stop that turned an attempt in flight into a terminal
|
||||
// failure. Stopping the engine cancels the request; for a target with
|
||||
// no retries that was the delivery's only attempt, recorded as failed
|
||||
// with "context canceled", and nothing ever sent it again. The
|
||||
// delivery must instead stay pending, with no result and no count in
|
||||
// the attempt metrics, for restart recovery to send.
|
||||
func TestEngine_StopLeavesFireAndForgetAttemptPending(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
s := newISetup(t)
|
||||
reg := mIsolate(t, s)
|
||||
lc := startEngineViaHook(t, s.Engine)
|
||||
|
||||
task, arrived := seedHeldDelivery(
|
||||
t, s, 0, database.DeliveryStatusPending, 1,
|
||||
)
|
||||
|
||||
s.Engine.Notify([]delivery.Task{task})
|
||||
stopDuringAttempt(t, lc, arrived)
|
||||
|
||||
iAssertStatus(
|
||||
t, s.WebhookDB, task.DeliveryID, database.DeliveryStatusPending,
|
||||
)
|
||||
assert.Empty(t, iResults(t, s.WebhookDB, task.DeliveryID))
|
||||
assert.Zero(t, mCounter(t, reg, mAttempts, mTypeHTTP))
|
||||
assert.Zero(t, mHTTPDurations(t, reg))
|
||||
|
||||
requireRequeued(t, s, s.Engine.ExportDeliveryCh(), task)
|
||||
}
|
||||
|
||||
// TestEngine_StopLeavesLastRetryAttemptRetrying is the same regression
|
||||
// for a target with retries, on its last attempt: the cancelled attempt
|
||||
// used to exhaust the retries and end the delivery failed. It must stay
|
||||
// retrying with only its earlier attempt recorded, and restart recovery
|
||||
// must send that last attempt again.
|
||||
func TestEngine_StopLeavesLastRetryAttemptRetrying(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const maxRetries = 2
|
||||
|
||||
s := newISetup(t)
|
||||
lc := startEngineViaHook(t, s.Engine)
|
||||
|
||||
task, arrived := seedHeldDelivery(
|
||||
t, s, maxRetries, database.DeliveryStatusRetrying, maxRetries,
|
||||
)
|
||||
iSeedFailedResult(t, s.WebhookDB, task.DeliveryID)
|
||||
|
||||
s.Engine.ExportScheduleRetry(task, 0)
|
||||
stopDuringAttempt(t, lc, arrived)
|
||||
|
||||
iAssertStatus(
|
||||
t, s.WebhookDB, task.DeliveryID, database.DeliveryStatusRetrying,
|
||||
)
|
||||
assert.Len(t, iResults(t, s.WebhookDB, task.DeliveryID), 1)
|
||||
|
||||
requireRequeued(t, s, s.Engine.ExportRetryCh(), task)
|
||||
}
|
||||
|
||||
// deliverToArchive gives the setup's webhook a database target,
|
||||
// runs one delivery to it through the running engine, and returns
|
||||
// the target's ID and archive file path. The archive writer holds
|
||||
|
||||
@@ -117,6 +117,18 @@ func seedDelivery(
|
||||
return d
|
||||
}
|
||||
|
||||
// assertStoredTime asserts that text, a timestamp as SQLite holds it,
|
||||
// is in the form GORM stores created_at in: the layout of
|
||||
// time.Time.String, with no " m=+…" monotonic clock reading after it.
|
||||
func assertStoredTime(t *testing.T, text string) {
|
||||
t.Helper()
|
||||
|
||||
_, err := time.Parse(
|
||||
"2006-01-02 15:04:05.999999999 -0700 MST", text,
|
||||
)
|
||||
assert.NoError(t, err, "stored as %q", text)
|
||||
}
|
||||
|
||||
// httpDeliveryFixture holds the task and delivery needed
|
||||
// for HTTP delivery tests.
|
||||
type httpDeliveryFixture struct {
|
||||
|
||||
@@ -462,7 +462,8 @@ func TestScheduledRetryIsNotSweptDuringBackoff(t *testing.T) {
|
||||
|
||||
// TestRedispatchStampsTheRow pins the cadence control: a stranded
|
||||
// delivery that has just been handed out is not selected again by the
|
||||
// next tick a minute later.
|
||||
// next tick a minute later. The stamp is stored in the same form as
|
||||
// created_at.
|
||||
func TestRedispatchStampsTheRow(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -493,4 +494,13 @@ func TestRedispatchStampsTheRow(t *testing.T) {
|
||||
"a re-dispatched delivery must be stamped so the "+
|
||||
"next tick does not select it again",
|
||||
)
|
||||
|
||||
var createdAt, updatedAt string
|
||||
|
||||
require.NoError(t, s.WebhookDB.Raw(
|
||||
`SELECT CAST(created_at AS TEXT), CAST(updated_at AS TEXT)
|
||||
FROM deliveries WHERE id = ?`, d.ID,
|
||||
).Row().Scan(&createdAt, &updatedAt))
|
||||
assertStoredTime(t, createdAt)
|
||||
assertStoredTime(t, updatedAt)
|
||||
}
|
||||
|
||||
@@ -32,7 +32,8 @@ type Scheduler interface {
|
||||
// Engine.observeAttempt, alongside the DeliveryResult it records for
|
||||
// it. Deliver is also entered for attempts that never happen — an
|
||||
// open circuit breaker refuses one — so the count cannot be taken
|
||||
// from around this call.
|
||||
// from around this call. An attempt cut off by a stop is neither
|
||||
// counted nor recorded.
|
||||
type Target interface {
|
||||
Deliver(
|
||||
ctx context.Context,
|
||||
|
||||
@@ -277,7 +277,11 @@ func (w *archiveWriter) write(
|
||||
}
|
||||
}
|
||||
|
||||
row.ArchivedAt = time.Now()
|
||||
// GORM's clock, the one that stamps created_at in the other
|
||||
// databases, so this is stored in the same form. A bare time.Now()
|
||||
// carries a monotonic clock reading, which the driver would store
|
||||
// as an " m=+…" suffix.
|
||||
row.ArchivedAt = w.db.NowFunc()
|
||||
|
||||
err := w.db.Create(&row).Error
|
||||
if err != nil {
|
||||
|
||||
@@ -150,6 +150,13 @@ func TestArchiveWriter_WritesRow(t *testing.T) {
|
||||
assert.Equal(t, row.ContentType, got[0].ContentType)
|
||||
assert.JSONEq(t, `{"hello":"world"}`, got[0].Body)
|
||||
assert.False(t, got[0].ArchivedAt.IsZero())
|
||||
|
||||
var archivedAt string
|
||||
|
||||
require.NoError(t, w.DB().Raw(
|
||||
"SELECT CAST(archived_at AS TEXT) FROM archived_events",
|
||||
).Row().Scan(&archivedAt))
|
||||
assertStoredTime(t, archivedAt)
|
||||
}
|
||||
|
||||
func TestArchiveWriter_RecreatesAfterRemoval(
|
||||
|
||||
@@ -82,8 +82,8 @@ func isReservedTargetHeader(name string) bool {
|
||||
// one pair silently overwriting the other.
|
||||
//
|
||||
// An input with no pairs yields an empty map, which omitempty drops
|
||||
// from the stored config: a target configured with no headers keeps
|
||||
// the same config JSON it had before this field existed.
|
||||
// from the stored config, so a target configured with no headers
|
||||
// stores no headers key at all.
|
||||
func ParseTargetHeaders(raw string) (map[string]string, error) {
|
||||
headers := make(map[string]string)
|
||||
|
||||
|
||||
@@ -57,7 +57,11 @@ type httpCore struct {
|
||||
// recorded and no circuit breaker is consulted. A positive
|
||||
// maxRetries gates the attempt on the circuit breaker and
|
||||
// schedules a backed-off retry on failure.
|
||||
//
|
||||
// ctx is the context attempt sends its request with. Stopping the
|
||||
// engine cancels it; see interruptedByStop.
|
||||
func (c *httpCore) deliver(
|
||||
ctx context.Context,
|
||||
webhookDB *gorm.DB,
|
||||
d *database.Delivery,
|
||||
task *Task,
|
||||
@@ -66,21 +70,55 @@ func (c *httpCore) deliver(
|
||||
attempt func() attemptResult,
|
||||
) {
|
||||
if maxRetries == 0 {
|
||||
c.fireAndForget(webhookDB, d, attempt())
|
||||
c.fireAndForget(ctx, webhookDB, d, attempt())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
c.withRetry(
|
||||
webhookDB, d, task, sched, maxRetries, attempt,
|
||||
ctx, webhookDB, d, task, sched, maxRetries, attempt,
|
||||
)
|
||||
}
|
||||
|
||||
// interruptedByStop reports whether an attempt failed because the
|
||||
// engine is stopping, which cancels the context its request is made
|
||||
// with. Such an attempt is not recorded: no DeliveryResult, no circuit
|
||||
// breaker outcome, no status change, no count in the attempt metrics.
|
||||
// The delivery stays pending or retrying, and startup recovery sends
|
||||
// it again on the next start.
|
||||
//
|
||||
// A target's own timeout leaves that context alone, so its attempt is
|
||||
// still recorded as the failure it is.
|
||||
func (c *httpCore) interruptedByStop(
|
||||
ctx context.Context,
|
||||
d *database.Delivery,
|
||||
res attemptResult,
|
||||
) bool {
|
||||
if res.success || ctx.Err() == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
c.eng.log.Warn(
|
||||
"delivery attempt interrupted by shutdown; not recorded, "+
|
||||
"it is sent again on the next start",
|
||||
"delivery_id", d.ID,
|
||||
"event_id", d.EventID,
|
||||
"target_id", d.TargetID,
|
||||
)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *httpCore) fireAndForget(
|
||||
ctx context.Context,
|
||||
webhookDB *gorm.DB,
|
||||
d *database.Delivery,
|
||||
res attemptResult,
|
||||
) {
|
||||
if c.interruptedByStop(ctx, d, res) {
|
||||
return
|
||||
}
|
||||
|
||||
c.eng.observeAttempt(d.Target.Type, res.elapsed())
|
||||
|
||||
err := c.eng.recordResult(
|
||||
@@ -110,6 +148,7 @@ func (c *httpCore) fireAndForget(
|
||||
}
|
||||
|
||||
func (c *httpCore) withRetry(
|
||||
ctx context.Context,
|
||||
webhookDB *gorm.DB,
|
||||
d *database.Delivery,
|
||||
task *Task,
|
||||
@@ -131,6 +170,10 @@ func (c *httpCore) withRetry(
|
||||
|
||||
res := attempt()
|
||||
|
||||
if c.interruptedByStop(ctx, d, res) {
|
||||
return
|
||||
}
|
||||
|
||||
c.eng.observeAttempt(d.Target.Type, res.elapsed())
|
||||
|
||||
err := c.eng.recordResult(
|
||||
@@ -395,7 +438,7 @@ func (t *httpTarget) Deliver(
|
||||
}
|
||||
|
||||
t.deliver(
|
||||
webhookDB, d, task, sched,
|
||||
ctx, webhookDB, d, task, sched,
|
||||
d.Target.MaxRetries, attempt,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -13,9 +13,9 @@ import (
|
||||
const RedactionMarker = "(redacted)"
|
||||
|
||||
// Redactor removes one target's own credential material from
|
||||
// text that target's remote peer chose: a delivery response
|
||||
// body, or a delivery error stored before the delivery path
|
||||
// learned to mask the URLs it embeds.
|
||||
// a delivery response body, whose text the target's remote
|
||||
// peer chose, and from a stored delivery error as a second
|
||||
// line behind the delivery path's own URL masking.
|
||||
//
|
||||
// It removes byte-identical echoes of strings taken from the
|
||||
// target's stored configuration, and nothing else. Anything
|
||||
|
||||
@@ -82,7 +82,7 @@ func (t *slackTarget) Deliver(
|
||||
}
|
||||
|
||||
t.deliver(
|
||||
webhookDB, d, task, sched,
|
||||
ctx, webhookDB, d, task, sched,
|
||||
d.Target.MaxRetries, attempt,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -42,9 +42,6 @@ func TestDeliveryAttempts_ReadInTheTargetTypesOwnTerms(t *testing.T) {
|
||||
{database.TargetTypeLog, true, 0, "", "written to the log", ""},
|
||||
}
|
||||
|
||||
// Named for type and outcome, so that no two cases share a name:
|
||||
// Go adds "#01" to a repeated one, and a "#" in the temporary
|
||||
// directory's path cuts the path SQLiteDSN gives SQLite short.
|
||||
for _, tc := range cases {
|
||||
t.Run(string(tc.targetType)+" "+tc.outcome, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -12,13 +12,12 @@ import (
|
||||
//
|
||||
// The bound is enforced here and in SQL, because this page's
|
||||
// memory profile must not depend on a constant in another
|
||||
// package staying where it is, and because rows predating the
|
||||
// delivery engine's own cap or restored from an archive are
|
||||
// not covered by it at all.
|
||||
// package staying where it is, or on every stored row having
|
||||
// been written by the delivery engine under its own cap.
|
||||
//
|
||||
// It happens to equal that engine cap, so a row written by the
|
||||
// current engine reaches this one exactly and is never cut
|
||||
// twice. Nothing here may assume the two differ: see view.
|
||||
// engine reaches this one exactly and is never cut twice.
|
||||
// Nothing here may assume the two differ: see view.
|
||||
const maxRenderedResponseBytes = 4096
|
||||
|
||||
// deliveryResultColumns is the delivery attempt projection.
|
||||
@@ -40,9 +39,8 @@ const deliveryResultColumns = "delivery_id, attempt_num, created_at, " +
|
||||
//
|
||||
// Both Error and ResponseBody have been through the target's
|
||||
// Redactor. The engine already masks the URL out of the
|
||||
// errors it stores, so for errors this is a second line
|
||||
// covering rows written before it did; for response bodies it
|
||||
// is the only line, and its reach is what
|
||||
// errors it stores, so for errors this is a second line; for
|
||||
// response bodies it is the only line, and its reach is what
|
||||
// delivery.Redactor documents.
|
||||
type DeliveryResultView struct {
|
||||
AttemptNum int
|
||||
|
||||
@@ -1,19 +1,15 @@
|
||||
package handlers_test
|
||||
|
||||
// The handler-side half of the log-field audit. Two slog calls in
|
||||
// this package reach a value an UNAUTHENTICATED client picks outright
|
||||
// and of a length it picks outright:
|
||||
// The handler-side half of the log-field audit. The slog calls in
|
||||
// this package that reach a value an UNAUTHENTICATED client picks
|
||||
// outright, and of a length it picks outright, are the failed-login
|
||||
// DEBUG lines, whose username is a form field.
|
||||
//
|
||||
// - the unknown-entrypoint DEBUG line on /h/{uuid}, whose
|
||||
// path segment matched no stored entrypoint and so is bounded by
|
||||
// nothing;
|
||||
// - the failed-login DEBUG lines, whose username is a form field.
|
||||
//
|
||||
// Both are at DEBUG, which is off in production by default. That is
|
||||
// They are at DEBUG, which is off in production by default. That is
|
||||
// not a bound: an operator turning DEBUG on to diagnose a flood must
|
||||
// not thereby hand the flood an unbounded write. Both spend the same
|
||||
// internal/logfield budget as the access log, and both are held here
|
||||
// to middleware.MaxAccessLogLineBytes.
|
||||
// not thereby hand the flood an unbounded write. They spend the same
|
||||
// internal/logfield budget as the access log, and are held here to
|
||||
// middleware.MaxAccessLogLineBytes.
|
||||
//
|
||||
// The two login lines past the username lookup — "invalid password"
|
||||
// and "user logged in" — carry the same cap without needing it, since
|
||||
@@ -260,43 +256,6 @@ func postLoginWithPassword(
|
||||
return w.Code
|
||||
}
|
||||
|
||||
// TestUnknownEntrypoint_LogLineDoesNotTrackPathSize drives 8 KB of
|
||||
// client-chosen path at the unauthenticated receiver's
|
||||
// unknown-entrypoint DEBUG line and holds it to the same ceiling the
|
||||
// access log states.
|
||||
func TestUnknownEntrypoint_LogLineDoesNotTrackPathSize(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for handlerName, newHandler := range logHandlers() {
|
||||
for fillName, fill := range escapeFills() {
|
||||
t.Run(handlerName+"/"+fillName, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
h, buf := capturingHandlers(t, newHandler)
|
||||
router := receiverRouter(h)
|
||||
|
||||
for i := range floodRequests {
|
||||
assert.Equal(
|
||||
t,
|
||||
http.StatusNotFound,
|
||||
postReceiver(
|
||||
t, router,
|
||||
oversizedFill(fill)+
|
||||
strings.Repeat("y", i),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
lines := logLines(t, buf)
|
||||
require.Len(t, lines, floodRequests)
|
||||
assertNoClientText(t, buf)
|
||||
|
||||
assertBoundedFlood(t, buf.Len())
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestFailedLogin_LogLineDoesNotTrackUsernameSize drives 8 KB of
|
||||
// client-chosen username at the unauthenticated login endpoint's
|
||||
// DEBUG line and holds it to the same ceiling.
|
||||
|
||||
@@ -49,13 +49,6 @@ var (
|
||||
// working retry configuration over a typo. A default answers a
|
||||
// question that was not asked; it never answers one that was asked
|
||||
// badly.
|
||||
//
|
||||
// A target stored with a count above the ceiling before this
|
||||
// validation existed keeps rendering and keeps delivering — nothing
|
||||
// clamps the row. Re-saving it from the edit form does have to bring
|
||||
// it into range, because the form submits the pre-filled value back
|
||||
// and accepting it would be the ceiling not applying to the edit
|
||||
// path. The 400 names the ceiling, so the fix is one field.
|
||||
func parseMaxRetries(raw string, fallback int) (int, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
|
||||
@@ -10,7 +10,6 @@ import (
|
||||
"gorm.io/gorm"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/delivery"
|
||||
"sneak.berlin/go/webhooker/internal/logfield"
|
||||
"sneak.berlin/go/webhooker/internal/middleware"
|
||||
)
|
||||
|
||||
@@ -49,14 +48,13 @@ func (h *Handlers) HandleWebhook() http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
// Logged only once the UUID is known to name a real
|
||||
// entrypoint. The UUID comes straight out of the path on
|
||||
// the one unauthenticated endpoint, so logging it before
|
||||
// the lookup let a client write an INFO line per invented
|
||||
// path; the request itself is already in the access log
|
||||
// and a miss is already logged at DEBUG.
|
||||
// Logged only once the lookup has found the entrypoint, so
|
||||
// an invented path writes no INFO line; a miss is logged at
|
||||
// DEBUG. The entrypoint is named by its ID: the UUID is its
|
||||
// credential and is never logged.
|
||||
h.log.Info("webhook request received",
|
||||
"entrypoint_uuid", entrypointUUID,
|
||||
"entrypoint_id", entrypoint.ID,
|
||||
"webhook_id", entrypoint.WebhookID,
|
||||
"method", r.Method,
|
||||
"remoteIP", middleware.RemoteIP(r),
|
||||
"clientIP", middleware.ClientIP(r),
|
||||
@@ -133,17 +131,10 @@ func (h *Handlers) lookupEntrypoint(
|
||||
"path = ?", entrypointUUID,
|
||||
).First(&entrypoint)
|
||||
if result.Error != nil {
|
||||
// The receiver is unauthenticated and /h/{uuid}
|
||||
// matches any single segment, so this value is entirely
|
||||
// client-chosen on exactly the branch where the lookup
|
||||
// failed. DEBUG is off by default; the cap is what keeps
|
||||
// turning it on from restoring an unbounded write.
|
||||
h.log.Debug(
|
||||
"entrypoint not found",
|
||||
"path", logfield.Truncate(
|
||||
entrypointUUID, logfield.MaxBytes,
|
||||
),
|
||||
)
|
||||
// Not even an unknown UUID is logged. The receiver's log
|
||||
// lines never carry the path segment, which is a credential
|
||||
// whenever it names an entrypoint.
|
||||
h.log.Debug("entrypoint not found")
|
||||
http.NotFound(w, r)
|
||||
|
||||
return entrypoint, false
|
||||
|
||||
@@ -12,12 +12,15 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi"
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm/clause"
|
||||
"sneak.berlin/go/webhooker/internal/config"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/handlers"
|
||||
"sneak.berlin/go/webhooker/internal/middleware"
|
||||
"sneak.berlin/go/webhooker/internal/middleware/middlewaretest"
|
||||
)
|
||||
|
||||
// TestHandleWebhook_LogsClientNextToThePeer checks that the
|
||||
@@ -99,6 +102,87 @@ func TestHandleWebhook_LogsClientNextToThePeer(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleWebhook_LogsNoEntrypointUUID sends an accepted event, a
|
||||
// request over the receiver rate limit and a request for an unknown
|
||||
// UUID through the receiver as routes.go mounts it, and checks that
|
||||
// nothing logged, at any level, carries either UUID.
|
||||
func TestHandleWebhook_LogsNoEntrypointUUID(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
db *database.Database
|
||||
)
|
||||
|
||||
cfg := &config.Config{DataDir: t.TempDir(), ReceiverRateLimit: 1}
|
||||
|
||||
app := newTestAppWithConfig(t, cfg, &h, &db)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
log := slog.New(slog.NewJSONHandler(
|
||||
buf, &slog.HandlerOptions{Level: slog.LevelDebug},
|
||||
))
|
||||
h.SetLogForTest(log)
|
||||
mw := middlewaretest.New(t, log, cfg, nil)
|
||||
|
||||
webhook := seedWebhook(t, db)
|
||||
entrypoint := &database.Entrypoint{
|
||||
WebhookID: webhook.ID,
|
||||
Path: uuid.New().String(),
|
||||
Active: true,
|
||||
}
|
||||
require.NoError(t, db.DB().Omit(clause.Associations).
|
||||
Create(entrypoint).Error)
|
||||
|
||||
unknownUUID := uuid.New().String()
|
||||
|
||||
router := chi.NewRouter()
|
||||
router.Use(mw.Logging())
|
||||
router.With(mw.ReceiverRateLimit()).HandleFunc(
|
||||
middleware.ReceiverRoute, h.HandleWebhook(),
|
||||
)
|
||||
|
||||
// In order: the limit is one request per minute per path, so the
|
||||
// second request for the known UUID is the rejected one.
|
||||
for _, step := range []struct {
|
||||
entrypointUUID string
|
||||
wantStatus int
|
||||
}{
|
||||
{entrypoint.Path, http.StatusOK},
|
||||
{entrypoint.Path, http.StatusTooManyRequests},
|
||||
{unknownUUID, http.StatusNotFound},
|
||||
} {
|
||||
req := httptest.NewRequestWithContext(
|
||||
context.Background(), http.MethodPost,
|
||||
"/h/"+step.entrypointUUID, strings.NewReader("{}"),
|
||||
)
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
require.Equal(t, step.wantStatus, w.Code)
|
||||
}
|
||||
|
||||
// The lines this test is about were all written, so the checks
|
||||
// below are not passing on an empty log.
|
||||
for _, msg := range []string{
|
||||
"webhook request received",
|
||||
"webhook receiver rate limit exceeded",
|
||||
"entrypoint not found",
|
||||
} {
|
||||
assert.Contains(t, buf.String(), `"msg":"`+msg+`"`)
|
||||
}
|
||||
|
||||
assert.NotContains(t, buf.String(), entrypoint.Path)
|
||||
assert.NotContains(t, buf.String(), unknownUUID)
|
||||
|
||||
line := receivedLine(t, buf)
|
||||
assert.Equal(t, entrypoint.ID, line["entrypoint_id"])
|
||||
assert.Equal(t, webhook.ID, line["webhook_id"])
|
||||
}
|
||||
|
||||
// receivedLine returns the one "webhook request received" line in the
|
||||
// captured JSON log.
|
||||
func receivedLine(t *testing.T, buf *bytes.Buffer) map[string]any {
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -27,6 +28,13 @@ const floodRequests = 64
|
||||
// line for a redirected or rejected request may contain it.
|
||||
const attackerMarker = "QQATTACKERTEXTQQ"
|
||||
|
||||
// knownEntrypointUUID and failingEntrypointUUID are the UUIDs the
|
||||
// stand-in receiver in accessLogRouter answers 200 and 500 to.
|
||||
const (
|
||||
knownEntrypointUUID = "6f1e2a9c-4b7d-4e3a-9c2f-1d8b5a7e3c60"
|
||||
failingEntrypointUUID = "0b7c3e5a-2d9f-4a61-8e4b-7c1d6f2a9e35"
|
||||
)
|
||||
|
||||
// maxLineBytes bounds a single access log line whose client-supplied
|
||||
// fields are of ordinary size. Well above what the fixed fields need,
|
||||
// well below the length of the oversized input the amplification tests
|
||||
@@ -132,17 +140,21 @@ func accessLogRouter(m *middleware.Middleware) *chi.Mux {
|
||||
)
|
||||
|
||||
router.HandleFunc(
|
||||
"/h/{uuid}",
|
||||
middleware.ReceiverRoute,
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
// Stands in for the real handler: an unknown entrypoint
|
||||
// UUID 404s, a known one succeeds.
|
||||
if chi.URLParam(r, "uuid") != "known" {
|
||||
// Stands in for the real handler: a known entrypoint
|
||||
// UUID succeeds, a failing one is a server error, and
|
||||
// any other 404s.
|
||||
switch chi.URLParam(r, "uuid") {
|
||||
case knownEntrypointUUID:
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case failingEntrypointUUID:
|
||||
http.Error(
|
||||
w, "boom", http.StatusInternalServerError,
|
||||
)
|
||||
default:
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
},
|
||||
)
|
||||
|
||||
@@ -649,17 +661,84 @@ func TestAccessLog_SuccessKeepsConcretePathAndRedactsQuery(
|
||||
router := accessLogRouter(m)
|
||||
|
||||
assert.Equal(
|
||||
t, http.StatusOK, get(t, router, "/h/known?src=ci"),
|
||||
t, http.StatusOK,
|
||||
get(t, router, "/.well-known/healthcheck?src=ci"),
|
||||
)
|
||||
|
||||
// The path resolved against a stored entrypoint, so it stays. The
|
||||
// query never does: see TestAccessLog_UnauthenticatedSuccess...
|
||||
entries := accessLogEntries(t, buf)
|
||||
require.Len(t, entries, 1)
|
||||
assert.Equal(t, "/h/known?(redacted)", entries[0]["url"])
|
||||
assert.Equal(
|
||||
t, "/.well-known/healthcheck?(redacted)", entries[0]["url"],
|
||||
)
|
||||
assert.NotContains(t, buf.String(), "src=ci")
|
||||
}
|
||||
|
||||
// TestAccessLog_ReceiverNeverLogsTheUUID covers the two statuses that
|
||||
// keep the concrete path on every other route. On the receiver that
|
||||
// path carries the entrypoint UUID, so the line names the route
|
||||
// pattern instead.
|
||||
func TestAccessLog_ReceiverNeverLogsTheUUID(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for entrypointUUID, wantStatus := range map[string]int{
|
||||
knownEntrypointUUID: http.StatusOK,
|
||||
failingEntrypointUUID: http.StatusInternalServerError,
|
||||
} {
|
||||
t.Run(strconv.Itoa(wantStatus), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
m, buf := capturingMiddleware(t)
|
||||
router := accessLogRouter(m)
|
||||
|
||||
assert.Equal(
|
||||
t, wantStatus,
|
||||
get(t, router, "/h/"+entrypointUUID+"?src=ci"),
|
||||
)
|
||||
|
||||
entries := accessLogEntries(t, buf)
|
||||
require.Len(t, entries, 1)
|
||||
assert.Equal(t, middleware.ReceiverRoute, entries[0]["url"])
|
||||
assert.NotContains(t, buf.String(), entrypointUUID)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestAccessLog_ReceiverPreflightNeverLogsTheUUID sends a CORS
|
||||
// preflight to the receiver in dev, in the production middleware
|
||||
// order. CORS answers it with 200 before chi routes it, so the line
|
||||
// names no route and carries no UUID.
|
||||
func TestAccessLog_ReceiverPreflightNeverLogsTheUUID(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
m, buf := capturingMiddleware(t)
|
||||
|
||||
router := chi.NewRouter()
|
||||
router.Use(m.Logging())
|
||||
router.Use(m.CORS())
|
||||
router.HandleFunc(
|
||||
middleware.ReceiverRoute,
|
||||
func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
},
|
||||
)
|
||||
|
||||
req := httptest.NewRequestWithContext(
|
||||
context.Background(), http.MethodOptions,
|
||||
"/h/"+knownEntrypointUUID, nil,
|
||||
)
|
||||
req.Header.Set("Origin", "https://sender.example")
|
||||
req.Header.Set("Access-Control-Request-Method", http.MethodPost)
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, req)
|
||||
assert.Equal(t, http.StatusOK, w.Code)
|
||||
|
||||
entries := accessLogEntries(t, buf)
|
||||
require.Len(t, entries, 1)
|
||||
assert.Equal(t, unmatchedRouteLiteral, entries[0]["url"])
|
||||
assert.NotContains(t, buf.String(), knownEntrypointUUID)
|
||||
}
|
||||
|
||||
func TestAccessLog_ServerErrorKeepsConcreteURL(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -10,8 +10,10 @@ package middleware_test
|
||||
// the client's choosing.
|
||||
// - CSRF's 403 rejection, at WARN, also registered ahead of
|
||||
// RequireAuth.
|
||||
// - The rate limiters' 429 rejection, at WARN, on the
|
||||
// unauthenticated receiver among others.
|
||||
// - The rate limiters' 429 rejection, at WARN. On the receiver
|
||||
// route it names the route pattern rather than the path, and
|
||||
// the limiters that log a path sit behind RequireAuth, so its cap
|
||||
// is defensive.
|
||||
// - RequireAuth's own unauthenticated-request line, at DEBUG.
|
||||
// - RecordLoginFailure's throttle rejection, at WARN. Its cap is
|
||||
// defensive rather than load-bearing today: chi pins the one
|
||||
@@ -268,10 +270,14 @@ func logSites() map[string]logSite {
|
||||
send: postNoToken,
|
||||
wantStatus: http.StatusForbidden,
|
||||
},
|
||||
// The per-entrypoint receiver limiter, unauthenticated. Its
|
||||
// bucket is keyed on the path, so the first request through a
|
||||
// fresh path is served and only the ones after it are
|
||||
// rejected; sendUntilLimited absorbs that.
|
||||
// The 429 line every limiter but the aggregate one shares.
|
||||
// Mounted outside any route, as here, it logs the path, as
|
||||
// it does for the limiters behind RequireAuth; on
|
||||
// ReceiverRoute it names the pattern instead. The
|
||||
// per-entrypoint receiver limiter's bucket is keyed on the
|
||||
// path, so the first request through a fresh path is served
|
||||
// and only the ones after it are rejected; sendUntilLimited
|
||||
// absorbs that.
|
||||
"receiver rate limit 429": {
|
||||
build: func(
|
||||
t *testing.T, m *middleware.Middleware,
|
||||
|
||||
@@ -41,7 +41,8 @@ const unmatchedMethod = unmatchedRoute
|
||||
// after /h/ is client-chosen, so labelling by path lets any
|
||||
// unauthenticated client mint permanent series at will, and publishes
|
||||
// the entrypoint UUID (the receiver's only credential) in the scrape
|
||||
// while doing it.
|
||||
// while doing it. The access log and the 429 line read it for the
|
||||
// same reason: see ReceiverRoute.
|
||||
//
|
||||
// chi populates the route context during routeHTTP, so this is only
|
||||
// valid once routing has run. Every caller below is on the recording
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"time"
|
||||
|
||||
basicauth "github.com/99designs/basicauth-go"
|
||||
"github.com/go-chi/chi"
|
||||
"github.com/go-chi/chi/middleware"
|
||||
"github.com/go-chi/cors"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
@@ -96,14 +95,14 @@ const (
|
||||
// It is also the ceiling on every OTHER line this service writes
|
||||
// THROUGH SLOG that carries text an UNAUTHENTICATED client
|
||||
// supplies. Those lines — the MaxBodySize rejection, the CSRF
|
||||
// rejection, the rate-limit rejection, the unauthenticated-request
|
||||
// and unknown-entrypoint DEBUG lines, the failed-login DEBUG
|
||||
// lines, and the two login-throttle WARN lines ("login failure
|
||||
// limit exceeded" in loginguard.go and "password verification
|
||||
// capacity exhausted" in internal/handlers/auth.go) — spend the
|
||||
// same per-field budgets, and each carries strictly fewer
|
||||
// client-supplied fields than the access log does,
|
||||
// so none of them can reach a width the access log cannot. That is
|
||||
// rejection, the unauthenticated-request DEBUG line, the
|
||||
// failed-login DEBUG lines, and the two login-throttle WARN
|
||||
// lines ("login failure limit exceeded" in loginguard.go and
|
||||
// "password verification capacity exhausted" in
|
||||
// internal/handlers/auth.go) — spend the same per-field budgets,
|
||||
// and each carries strictly fewer client-supplied fields than the
|
||||
// access log does, so none of them can reach a width the access
|
||||
// log cannot. That is
|
||||
// asserted directly, per line and under both handlers, rather than
|
||||
// left to the reasoning: see logbound_test.go in this package and
|
||||
// in internal/handlers.
|
||||
@@ -151,6 +150,12 @@ const (
|
||||
MaxAccessLogLineBytes = 2560
|
||||
)
|
||||
|
||||
// ReceiverRoute is the route pattern the webhook receiver is served
|
||||
// at. Its path carries the entrypoint UUID, which is the receiver's
|
||||
// only credential, so the log lines this package writes name a
|
||||
// request on this route by the pattern, never by its path.
|
||||
const ReceiverRoute = "/h/{uuid}"
|
||||
|
||||
//nolint:revive // MiddlewareParams is a standard fx naming convention.
|
||||
type MiddlewareParams struct {
|
||||
fx.In
|
||||
@@ -268,14 +273,14 @@ func (lrw *loggingResponseWriter) Unwrap() http.ResponseWriter {
|
||||
// marker.
|
||||
//
|
||||
// The path on those branches is bounded by the service's routes or by
|
||||
// the operator's data — a 2xx on the receiver means the UUID named a
|
||||
// stored entrypoint, a 2xx under /s means the file is in the embedded
|
||||
// tree. The query is not bounded by anything: /.well-known/healthcheck
|
||||
// and /s/* take no authentication and sit behind no rate limiter, and
|
||||
// /pages/login behind only the login limiter, so any of them will
|
||||
// answer 200 to a URL carrying an arbitrary number of arbitrary bytes
|
||||
// after the '?'. Keeping the path and dropping the query is what makes
|
||||
// this branch as bounded as the pattern branches below.
|
||||
// the operator's data — a 2xx under /s means the file is in the
|
||||
// embedded tree. The query is not bounded by anything:
|
||||
// /.well-known/healthcheck and /s/* take no authentication and sit
|
||||
// behind no rate limiter, and /pages/login behind only the login
|
||||
// limiter, so any of them will answer 200 to a URL carrying an
|
||||
// arbitrary number of arbitrary bytes after the '?'. Keeping the path
|
||||
// and dropping the query is what makes this branch as bounded as the
|
||||
// pattern branches below.
|
||||
//
|
||||
// Nothing debuggable is lost. The only query parameters the service
|
||||
// reads are the login page's `next`, the page to return to,
|
||||
@@ -297,36 +302,40 @@ func concreteLogURL(r *http.Request) string {
|
||||
|
||||
// accessLogURL returns the value for the access log's url field.
|
||||
//
|
||||
// 2xx and 5xx responses get the concrete path (see concreteLogURL). A
|
||||
// success resolved against a static route or against the operator's
|
||||
// own data — on the receiver, a 2xx means the UUID named a stored
|
||||
// entrypoint — and a server error is our own bug, where the exact URL
|
||||
// is the primary evidence and which no client can provoke at will.
|
||||
// On ReceiverRoute it is the route pattern whatever the status, so
|
||||
// the entrypoint UUID in the path is never logged. A request chi never
|
||||
// routed gets unmatchedRoute whatever the status, for the same reason:
|
||||
// in dev, CORS answers a preflight to the receiver before routing.
|
||||
//
|
||||
// On every other route, 2xx and 5xx responses get the concrete path
|
||||
// (see concreteLogURL). A success resolved against a static route or
|
||||
// against the operator's own data, and a server error is our own bug,
|
||||
// where the exact URL is the primary evidence and which no client can
|
||||
// provoke at will.
|
||||
//
|
||||
// 3xx and 4xx responses get the chi route pattern instead. Those are
|
||||
// the outcomes an unauthenticated client drives for free: 404 or 429
|
||||
// on any invented /h/ path, 303 to the login page on any
|
||||
// invented /user/ path. Logging the concrete URL there lets a flood
|
||||
// write attacker-chosen text, of attacker-chosen length, into the
|
||||
// operator's log at one line per request. The pattern comes from the
|
||||
// router's own table, so it is bounded by the service's routes while
|
||||
// still naming which class of request was rejected.
|
||||
// the outcomes an unauthenticated client drives for free, such as a
|
||||
// 303 to the login page on any invented /user/ path. Logging the
|
||||
// concrete URL there lets a flood write attacker-chosen text, of
|
||||
// attacker-chosen length, into the operator's log at one line per
|
||||
// request. The pattern comes from the router's own table, so it is
|
||||
// bounded by the service's routes while still naming which class of
|
||||
// request was rejected.
|
||||
//
|
||||
// The pattern is only populated once routing has run, so this must be
|
||||
// called after the handler returns, not before.
|
||||
func accessLogURL(r *http.Request, status int) string {
|
||||
pattern := routePatternID(r.Context())
|
||||
if pattern == ReceiverRoute || pattern == unmatchedRoute {
|
||||
return pattern
|
||||
}
|
||||
|
||||
if status < http.StatusMultipleChoices ||
|
||||
status >= http.StatusInternalServerError {
|
||||
return concreteLogURL(r)
|
||||
}
|
||||
|
||||
if rc := chi.RouteContext(r.Context()); rc != nil {
|
||||
if pattern := rc.RoutePattern(); pattern != "" {
|
||||
return pattern
|
||||
}
|
||||
}
|
||||
|
||||
return unmatchedRoute
|
||||
return pattern
|
||||
}
|
||||
|
||||
// Logging returns middleware that logs each HTTP request with
|
||||
|
||||
@@ -259,26 +259,28 @@ func (m *Middleware) clientAddr(r *http.Request) (netip.Addr, bool) {
|
||||
return peer, true
|
||||
}
|
||||
|
||||
// tooManyRequests returns the 429 handler used by the
|
||||
// password-change and per-entrypoint receiver limiters: it logs the
|
||||
// tooManyRequests returns the 429 handler used by the postRateLimit
|
||||
// limiters and the per-entrypoint receiver limiter: it logs the
|
||||
// rejection with logMessage and answers with responseMessage.
|
||||
// httprate adds the Retry-After header (RFC 6585). The aggregate
|
||||
// receiver limiter uses floodTooManyRequests instead.
|
||||
//
|
||||
// The path is capped against the same budget as the access log's url
|
||||
// field. The per-entrypoint receiver limiter is unauthenticated and
|
||||
// its path is a client-chosen segment of client-chosen length, so at
|
||||
// WARN an uncapped path would let a sender pick the size of the line
|
||||
// it writes — the same defect the access log capping closed.
|
||||
// On ReceiverRoute the line names the route pattern, so the
|
||||
// entrypoint UUID in the path is never logged. Elsewhere it names the
|
||||
// path, capped against the same budget as the access log's url field;
|
||||
// the limiters there sit behind RequireAuth, so the cap is defensive.
|
||||
func (m *Middleware) tooManyRequests(
|
||||
logMessage, responseMessage string,
|
||||
) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
path := r.URL.Path
|
||||
if routePatternID(r.Context()) == ReceiverRoute {
|
||||
path = ReceiverRoute
|
||||
}
|
||||
|
||||
m.log.Warn(
|
||||
logMessage,
|
||||
"path", logfield.Truncate(
|
||||
r.URL.Path, logfield.MaxBytes,
|
||||
),
|
||||
"path", logfield.Truncate(path, logfield.MaxBytes),
|
||||
"remoteIP", RemoteIP(r),
|
||||
"clientIP", ClientIP(r),
|
||||
)
|
||||
|
||||
@@ -6,7 +6,8 @@ import (
|
||||
|
||||
sentryhttp "github.com/getsentry/sentry-go/http"
|
||||
"github.com/go-chi/chi"
|
||||
"github.com/go-chi/chi/middleware"
|
||||
chimw "github.com/go-chi/chi/middleware"
|
||||
"sneak.berlin/go/webhooker/internal/middleware"
|
||||
"sneak.berlin/go/webhooker/static"
|
||||
)
|
||||
|
||||
@@ -60,7 +61,7 @@ func (s *Server) SetupRoutes() {
|
||||
}
|
||||
|
||||
func (s *Server) setupGlobalMiddleware() {
|
||||
s.router.Use(middleware.RequestID)
|
||||
s.router.Use(chimw.RequestID)
|
||||
s.router.Use(s.mw.SecurityHeaders())
|
||||
s.router.Use(s.mw.Logging())
|
||||
|
||||
@@ -345,7 +346,7 @@ func (s *Server) setupWebhookRoutes() {
|
||||
// unauthenticated endpoint's body; TestReceiver_OversizeBodyRefused
|
||||
// pins it.
|
||||
s.router.With(s.mw.ReceiverRateLimit()).HandleFunc(
|
||||
"/h/{uuid}",
|
||||
middleware.ReceiverRoute,
|
||||
s.h.HandleWebhook(),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -441,10 +441,9 @@ func (s *Session) Regenerate(
|
||||
//
|
||||
// Whichever comes first ends the session.
|
||||
//
|
||||
// A session that claims to be authenticated but carries no
|
||||
// timestamps predates this check; it is treated as expired so the
|
||||
// user re-authenticates rather than being granted an unbounded
|
||||
// session.
|
||||
// A session that claims to be authenticated but lacks the timestamp
|
||||
// a deadline is measured from is treated as expired, so the user
|
||||
// re-authenticates rather than being granted an unbounded session.
|
||||
func (s *Session) expired(sess *sessions.Session) bool {
|
||||
now := s.now()
|
||||
|
||||
|
||||
@@ -156,7 +156,8 @@ func TestVersion_IsStableAcrossInvocations(t *testing.T) {
|
||||
// The build target is the only place composing linker flags. If a
|
||||
// future edit drops either half, the binary silently reports "dev"
|
||||
// again (the defect this package exists for) or fails to link on
|
||||
// Alpine.
|
||||
// Alpine. If it stops passing GO_TAGS, the image's static binary links
|
||||
// glibc's getaddrinfo and dlopen again.
|
||||
func TestMakefile_BuildComposesVersionAndExtraFlags(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -164,6 +165,7 @@ func TestMakefile_BuildComposesVersionAndExtraFlags(t *testing.T) {
|
||||
|
||||
require.Contains(t, makefile, "-X main.version=$(VERSION)")
|
||||
require.Contains(t, makefile, "$(GO_LDFLAGS)")
|
||||
require.Contains(t, makefile, "-tags '$(GO_TAGS)'")
|
||||
require.Contains(t, makefile, "VERSION ?= $(shell script/version)")
|
||||
}
|
||||
|
||||
@@ -233,7 +235,9 @@ func TestDockerfile_BuildsThroughTheMakeTarget(t *testing.T) {
|
||||
"a raw go build bypasses the Makefile's -X flag")
|
||||
require.Contains(t, dockerfile, "ARG VERSION")
|
||||
require.Contains(t, dockerfile,
|
||||
`make build VERSION="$VERSION" GO_LDFLAGS='-extldflags "-static"'`)
|
||||
`make build VERSION="$VERSION"`+
|
||||
` GO_TAGS=netgo,osusergo,sqlite_omit_load_extension`+
|
||||
` GO_LDFLAGS='-extldflags "-static"'`)
|
||||
}
|
||||
|
||||
func read(t *testing.T, path string) string {
|
||||
|
||||
Reference in New Issue
Block a user