Compare commits
14
Commits
9b39b865ca
..
next
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8c05ec4378 | ||
|
|
872e4eb4a1 | ||
|
|
dfe8f3138f | ||
|
|
0d38aea12c | ||
|
|
c0f7cc29fd | ||
|
|
f14fe02011 | ||
|
|
573f7f986a | ||
|
|
373a2f828d | ||
|
|
267d5b12f2 | ||
|
|
e73beb20df | ||
|
|
c8efbd41d4 | ||
|
|
15377f792b | ||
|
|
f86ab6321d | ||
|
|
957de7f99f |
+14
-18
@@ -76,8 +76,6 @@ RUN --network=none node_modules/.bin/prettier --check '**/*.md'
|
|||||||
# depends on it.
|
# depends on it.
|
||||||
#
|
#
|
||||||
# golangci/golangci-lint:v2.14.0 (Debian-based), 2026-09-24
|
# golangci/golangci-lint:v2.14.0 (Debian-based), 2026-09-24
|
||||||
# Using Debian-based image because mattn/go-sqlite3 (CGO) does not
|
|
||||||
# compile on Alpine musl (off64_t is a glibc type).
|
|
||||||
FROM golangci/golangci-lint:v2.14.0@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
FROM golangci/golangci-lint:v2.14.0@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
||||||
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
@@ -167,9 +165,7 @@ RUN --mount=type=tmpfs,target=/tmp/tests,size=512m \
|
|||||||
|
|
||||||
# Build stage
|
# Build stage
|
||||||
# golang:1.26.1-bookworm (Debian-based), 2026-03-17
|
# golang:1.26.1-bookworm (Debian-based), 2026-03-17
|
||||||
# Using Debian-based image because gorm.io/driver/sqlite pulls in
|
# The image ships git and make, which the version step below uses.
|
||||||
# mattn/go-sqlite3 (CGO), which does not compile on Alpine musl. The image
|
|
||||||
# ships git and make, which the version step below uses.
|
|
||||||
FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a3492282a6c820bf4755fd64a4 AS builder
|
FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a3492282a6c820bf4755fd64a4 AS builder
|
||||||
|
|
||||||
# Nothing is wanted from the lint and test phases or from the stylesheet and
|
# Nothing is wanted from the lint and test phases or from the stylesheet and
|
||||||
@@ -211,20 +207,11 @@ RUN version="$(make version VERSION="$VERSION")"; \
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
# Builds through the Makefile's build target, which runs script/assets
|
# Builds through the Makefile's build target, which runs script/assets
|
||||||
# (Alpine.js, extracted from its tarball in 3p/) first.
|
# (Alpine.js, extracted from its tarball in 3p/) first. The target builds
|
||||||
|
# with cgo off, so the binary is statically linked and runs on the Alpine
|
||||||
|
# runtime image below as it is.
|
||||||
RUN make build VERSION="$VERSION"
|
RUN make build VERSION="$VERSION"
|
||||||
|
|
||||||
# Rebuild with static linking for Alpine runtime.
|
|
||||||
# make build already verified compilation.
|
|
||||||
# The CGO binary from `make build` is dynamically linked against glibc,
|
|
||||||
# which doesn't exist on Alpine (musl). Rebuild with static linking so
|
|
||||||
# the binary runs on Alpine without glibc.
|
|
||||||
#
|
|
||||||
# The static flags go in through GO_LDFLAGS rather than a -ldflags of
|
|
||||||
# their own: the build target composes them with the -X that stamps the
|
|
||||||
# version, so this relink cannot silently drop the stamp.
|
|
||||||
RUN CGO_ENABLED=1 make build VERSION="$VERSION" GO_LDFLAGS='-extldflags "-static"'
|
|
||||||
|
|
||||||
# Runtime stage
|
# Runtime stage
|
||||||
# alpine:3.21, 2026-03-17
|
# alpine:3.21, 2026-03-17
|
||||||
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
|
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
|
||||||
@@ -268,8 +255,17 @@ EXPOSE 8080
|
|||||||
# port.
|
# port.
|
||||||
ENV BIND_ADDRESS=0.0.0.0
|
ENV BIND_ADDRESS=0.0.0.0
|
||||||
|
|
||||||
|
# The probe requests the container's own address, not loopback, so a
|
||||||
|
# listener that answers only on loopback (an empty or loopback
|
||||||
|
# BIND_ADDRESS) fails it. On a network with IPv4, hostname -i prints the
|
||||||
|
# container's IPv4 address, which 0.0.0.0 and :: both answer. On an
|
||||||
|
# IPv6-only network it prints none, and the probe requests the IPv6
|
||||||
|
# address /etc/hosts lists under the container's hostname instead, which
|
||||||
|
# only :: answers.
|
||||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
||||||
CMD wget --no-verbose --tries=1 --spider http://localhost:8080/.well-known/healthcheck || exit 1
|
CMD addr="$(hostname -i 2>/dev/null | cut -d ' ' -f 1)"; \
|
||||||
|
[ -n "$addr" ] || addr="[$(getent hosts "$(hostname)" | cut -d ' ' -f 1)]"; \
|
||||||
|
wget --no-verbose --tries=1 --spider "http://$addr:8080/.well-known/healthcheck" || exit 1
|
||||||
|
|
||||||
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
|
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
|
||||||
CMD ["/app/webhooker"]
|
CMD ["/app/webhooker"]
|
||||||
|
|||||||
@@ -16,15 +16,6 @@ VERSION ?= $(shell script/version)
|
|||||||
# exists to correct.
|
# exists to correct.
|
||||||
override VERSION := $(or $(strip $(VERSION)),$(shell script/version))
|
override VERSION := $(or $(strip $(VERSION)),$(shell script/version))
|
||||||
|
|
||||||
# Extra linker flags for the build target. The static relink in the
|
|
||||||
# Dockerfile adds -extldflags here rather than passing its own -ldflags,
|
|
||||||
# so composing flags cannot drop the version stamp.
|
|
||||||
#
|
|
||||||
# The build target itself always passes -trimpath and -s -w, as the Go
|
|
||||||
# Dockerfile in REPO_POLICIES.md does: no build paths, symbol table or
|
|
||||||
# debug information in the binary.
|
|
||||||
GO_LDFLAGS ?=
|
|
||||||
|
|
||||||
bootstrap:
|
bootstrap:
|
||||||
@script/bootstrap
|
@script/bootstrap
|
||||||
|
|
||||||
@@ -52,8 +43,12 @@ fmt-check:
|
|||||||
check:
|
check:
|
||||||
@script/check
|
@script/check
|
||||||
|
|
||||||
|
# Built with cgo off, -trimpath and -s -w, as the Go Dockerfile in
|
||||||
|
# REPO_POLICIES.md does: the binary carries no build paths, symbol table or
|
||||||
|
# debug information, and on Linux, the image's included, it is statically
|
||||||
|
# linked.
|
||||||
build: assets
|
build: assets
|
||||||
go build -trimpath -ldflags '$(strip -s -w -X main.version=$(VERSION) $(GO_LDFLAGS))' -o bin/webhooker ./cmd/webhooker
|
CGO_ENABLED=0 go build -trimpath -ldflags '-s -w -X main.version=$(VERSION)' -o bin/webhooker ./cmd/webhooker
|
||||||
|
|
||||||
run: build
|
run: build
|
||||||
./bin/webhooker
|
./bin/webhooker
|
||||||
|
|||||||
@@ -310,16 +310,17 @@ the namespace, whatever the process bound. Exposure is decided at the publish
|
|||||||
flag instead — `-p 127.0.0.1:8080:8080` rather than `-p 8080:8080` — which is
|
flag instead — `-p 127.0.0.1:8080:8080` rather than `-p 8080:8080` — which is
|
||||||
the operator's to choose and is what [Running with Docker](#running-with-docker)
|
the operator's to choose and is what [Running with Docker](#running-with-docker)
|
||||||
shows. A loopback bind inside a container buys nothing and makes the process
|
shows. A loopback bind inside a container buys nothing and makes the process
|
||||||
unreachable through its own published port.
|
unreachable through its own published port. A container on an IPv6-only Docker
|
||||||
|
network has no IPv4 address for `0.0.0.0` to answer on, so there it needs
|
||||||
|
`BIND_ADDRESS=::`.
|
||||||
|
|
||||||
The value must be an IP address literal:
|
The value must be an IP address literal:
|
||||||
|
|
||||||
- `127.0.0.1` — loopback only (the binary's default). Use this with a reverse
|
- `127.0.0.1` — loopback only (the binary's default). Use this with a reverse
|
||||||
proxy on the same host.
|
proxy on the same host.
|
||||||
- `0.0.0.0` — every IPv4 address. The image's default; on a bare host, only
|
- `0.0.0.0` — every IPv4 address and no IPv6 address. The image's default; on a
|
||||||
behind a firewall on the port.
|
bare host, only behind a firewall on the port.
|
||||||
- `::` — every address, IPv6 and (on Linux, with the default
|
- `::` — every IPv6 address and every IPv4 address.
|
||||||
`net.ipv6.bindv6only=0`) IPv4 as well.
|
|
||||||
- A specific address such as `10.0.0.5` — that interface only.
|
- A specific address such as `10.0.0.5` — that interface only.
|
||||||
|
|
||||||
An **empty** value is treated as unset, as everywhere else here, and takes the
|
An **empty** value is treated as unset, as everywhere else here, and takes the
|
||||||
@@ -340,7 +341,8 @@ machine that has no such interface — is a valid literal, so it reaches the
|
|||||||
listener and fails there. The process logs the bind error and exits non-zero
|
listener and fails there. The process logs the bind error and exits non-zero
|
||||||
rather than staying up with nothing listening. The effective value is in the
|
rather than staying up with nothing listening. The effective value is in the
|
||||||
`bindAddress` field of the startup log line, which is the way to check what a
|
`bindAddress` field of the startup log line, which is the way to check what a
|
||||||
running deployment actually bound.
|
running deployment actually bound. On a bare host, `ss -ltn` lists the socket
|
||||||
|
itself as `0.0.0.0:8080` for a `0.0.0.0` bind and as `*:8080` for `::`.
|
||||||
|
|
||||||
#### Metrics credentials
|
#### Metrics credentials
|
||||||
|
|
||||||
@@ -465,10 +467,7 @@ idle timestamp is rewritten at most once per tenth of the idle window rather
|
|||||||
than on every request, which means a session may expire up to 10% early relative
|
than on every request, which means a session may expire up to 10% early relative
|
||||||
to the user's true last request, but never late.
|
to the user's true last request, but never late.
|
||||||
|
|
||||||
Both clocks are anchored by timestamps stored in the session cookie. Sessions
|
Both clocks are anchored by timestamps stored in the session cookie.
|
||||||
issued before this feature existed carry neither, so they are treated as
|
|
||||||
expired: upgrading to a build that has it logs every existing session out once,
|
|
||||||
and those users sign in again.
|
|
||||||
|
|
||||||
#### Invalid values abort startup
|
#### Invalid values abort startup
|
||||||
|
|
||||||
@@ -671,9 +670,9 @@ Either set a literal or leave the variable out entirely.
|
|||||||
Overriding `BIND_ADDRESS` to a loopback address in a container — by that route
|
Overriding `BIND_ADDRESS` to a loopback address in a container — by that route
|
||||||
or deliberately — makes the container unreachable from outside its namespace
|
or deliberately — makes the container unreachable from outside its namespace
|
||||||
even with `-p`. The published port answers nothing, and the health check fails
|
even with `-p`. The published port answers nothing, and the health check fails
|
||||||
too: it requests `http://localhost:8080`, `localhost` resolves to `::1` first,
|
too: it requests the container's own address (its IPv4 address, or its IPv6
|
||||||
and a `127.0.0.1` bind is not listening there. The container then goes
|
address on an IPv6-only network), where a loopback bind is not listening. The
|
||||||
`unhealthy` about **65 seconds** after start — from
|
container then goes `unhealthy` about **65 seconds** after start — from
|
||||||
`HEALTHCHECK --start-period=5s --interval=30s --retries=3`, so failing probes at
|
`HEALTHCHECK --start-period=5s --interval=30s --retries=3`, so failing probes at
|
||||||
5s, 35s and 65s, and `unhealthy` on the third. (Docker's probe cadence during
|
5s, 35s and 65s, and `unhealthy` on the third. (Docker's probe cadence during
|
||||||
the start period has changed between versions; re-derive from those three values
|
the start period has changed between versions; re-derive from those three values
|
||||||
@@ -702,11 +701,11 @@ app as that user.
|
|||||||
`webhooker.db` holds target configuration in plaintext — bearer tokens, API
|
`webhooker.db` holds target configuration in plaintext — bearer tokens, API
|
||||||
keys, Slack webhook URLs — along with the session encryption key, so webhooker
|
keys, Slack webhook URLs — along with the session encryption key, so webhooker
|
||||||
creates every SQLite file it owns `0600`: each database and both of its `-wal`
|
creates every SQLite file it owns `0600`: each database and both of its `-wal`
|
||||||
and `-shm` sidecars, across all three tiers. Files an earlier build left `0644`
|
and `-shm` sidecars, across all three tiers. Any of them found at another mode
|
||||||
are tightened when they are opened. The directory's `0750` is defence in depth —
|
is set to `0600` when the database is opened. The directory's `0750` is defence
|
||||||
it stops other local users listing the directory and learning your webhook UUIDs
|
in depth — it stops other local users listing the directory and learning your
|
||||||
from the `events-{uuid}.db` filenames — not the barrier protecting the
|
webhook UUIDs from the `events-{uuid}.db` filenames — not the barrier protecting
|
||||||
credentials.
|
the credentials.
|
||||||
|
|
||||||
### Running under upaas
|
### Running under upaas
|
||||||
|
|
||||||
@@ -729,8 +728,8 @@ credentials.
|
|||||||
address (directly, or through anything that can rewrite source addresses,
|
address (directly, or through anything that can rewrite source addresses,
|
||||||
such as NAT or a published container port). The `remoteIP` field of the
|
such as NAT or a published container port). The `remoteIP` field of the
|
||||||
`http request` log line for a request that came through the proxy shows
|
`http request` log line for a request that came through the proxy shows
|
||||||
that address; the health check's own lines show `::1`. See
|
that address; the health check's own lines show the container's own
|
||||||
[Trusted proxies](#trusted-proxies).
|
address. See [Trusted proxies](#trusted-proxies).
|
||||||
- Leave `BIND_ADDRESS` and `DATA_DIR` unset: the image sets `BIND_ADDRESS`
|
- Leave `BIND_ADDRESS` and `DATA_DIR` unset: the image sets `BIND_ADDRESS`
|
||||||
to `0.0.0.0`, and `DATA_DIR` defaults to `/var/lib/webhooker`.
|
to `0.0.0.0`, and `DATA_DIR` defaults to `/var/lib/webhooker`.
|
||||||
- Everything else is optional; see [Configuration](#configuration).
|
- Everything else is optional; see [Configuration](#configuration).
|
||||||
@@ -805,7 +804,12 @@ comes up, serves pages, and is broken in a way nothing reports.
|
|||||||
proxy's log names it regardless. nginx's default `combined` format already
|
proxy's log names it regardless. nginx's default `combined` format already
|
||||||
logs `$remote_addr`; do not replace it with one that drops the client
|
logs `$remote_addr`; do not replace it with one that drops the client
|
||||||
address, and retain those logs as long as you would want to answer a question
|
address, and retain those logs as long as you would want to answer a question
|
||||||
about traffic.
|
about traffic. That log also records the full request path, so it holds the
|
||||||
|
entrypoint UUID of every event a sender posts. webhooker's own log does not,
|
||||||
|
apart from the line a `log` target writes. Protect the proxy's log as you
|
||||||
|
would the entrypoint URLs themselves: anyone who can read it can post events
|
||||||
|
to every entrypoint in it (see
|
||||||
|
[The entrypoint URL is the authentication secret](#the-entrypoint-url-is-the-authentication-secret)).
|
||||||
|
|
||||||
### nginx
|
### nginx
|
||||||
|
|
||||||
@@ -831,7 +835,9 @@ server {
|
|||||||
client_max_body_size 1m;
|
client_max_body_size 1m;
|
||||||
|
|
||||||
# $remote_addr is the client. webhooker's own log names it, as
|
# $remote_addr is the client. webhooker's own log names it, as
|
||||||
# clientIP, only while TRUSTED_PROXIES covers this proxy.
|
# clientIP, only while TRUSTED_PROXIES covers this proxy. The
|
||||||
|
# request path in this log holds the entrypoint UUID, so protect
|
||||||
|
# the file like the entrypoint URL itself.
|
||||||
access_log /var/log/nginx/webhooker.access.log combined;
|
access_log /var/log/nginx/webhooker.access.log combined;
|
||||||
|
|
||||||
location / {
|
location / {
|
||||||
@@ -911,13 +917,14 @@ as `example.com`, not `hooks.example.com`. A listed domain is HTTPS-only in
|
|||||||
every browser that ships the list, even for users who have never visited it, and
|
every browser that ships the list, even for users who have never visited it, and
|
||||||
removal takes months to reach those browsers, so treat a listing as permanent.
|
removal takes months to reach those browsers, so treat a listing as permanent.
|
||||||
|
|
||||||
Setting `HSTS_MAX_AGE=0` later stops the header being sent, but a browser that
|
Setting `HSTS_MAX_AGE=0` stops the header being sent, but a browser that already
|
||||||
already received it over HTTPS does not forget it. That browser keeps refusing
|
received it over HTTPS does not forget it: it keeps refusing plaintext HTTP
|
||||||
plaintext HTTP until the `max-age` it saw runs out. Only a `max-age=0` header
|
until the `max-age` it saw runs out. Each header a browser receives over HTTPS
|
||||||
received over HTTPS clears it sooner. webhooker never sends one; a reverse proxy
|
replaces the `max-age` it stored, so to end the policy sooner, set a short
|
||||||
in front of it can.
|
`HSTS_MAX_AGE` such as `1s` and keep it until returning browsers have picked it
|
||||||
|
up, or have a reverse proxy send `max-age=0`.
|
||||||
|
|
||||||
## Backup, Restore, and Upgrades
|
## Backup and Restore
|
||||||
|
|
||||||
### What to back up
|
### What to back up
|
||||||
|
|
||||||
@@ -1070,59 +1077,14 @@ beside it, or wait until there are none.
|
|||||||
|
|
||||||
4. Start the service. The container gives the directory and the restored files
|
4. Start the service. The container gives the directory and the restored files
|
||||||
to the `webhooker` user before the app starts, whoever restored them (see
|
to the `webhooker` user before the app starts, whoever restored them (see
|
||||||
[Running with Docker](#running-with-docker)). `AutoMigrate` runs against each
|
[Running with Docker](#running-with-docker)).
|
||||||
restored database as it is opened.
|
|
||||||
|
|
||||||
### Upgrades
|
|
||||||
|
|
||||||
**Back up before every upgrade.** Every start runs GORM `AutoMigrate`
|
|
||||||
unconditionally against whatever files it finds:
|
|
||||||
|
|
||||||
- the main database on connect — `Setting`, `User`, `APIKey`, `Webhook`,
|
|
||||||
`Entrypoint`, `Target`
|
|
||||||
- each event database when it is lazily opened — `Event`, `Delivery`,
|
|
||||||
`DeliveryResult`, `EventTotals`, `TargetTotals`, `EntrypointTotals`
|
|
||||||
- each archive database on every open and reopen
|
|
||||||
|
|
||||||
There is no schema version table, no migration ledger, and no down migrations.
|
|
||||||
Nothing in the files records which version wrote them, and no code path undoes a
|
|
||||||
migration.
|
|
||||||
|
|
||||||
Upgrade procedure:
|
|
||||||
|
|
||||||
1. Stop the service.
|
|
||||||
2. Back up `DATA_DIR` using one of the procedures above.
|
|
||||||
3. Pull the new image and start it.
|
|
||||||
4. Confirm `database migrations completed` in the logs before putting traffic
|
|
||||||
back on it.
|
|
||||||
5. Confirm the new build is the one running:
|
|
||||||
`curl -s http://host:8080/.well-known/healthcheck` reports the version it was
|
|
||||||
stamped with (see [Version stamping](#version-stamping)).
|
|
||||||
|
|
||||||
**Upgrading past the introduction of `BIND_ADDRESS`:** earlier versions always
|
|
||||||
bound every interface. **Container deployments are unaffected** — the image
|
|
||||||
ships `ENV BIND_ADDRESS=0.0.0.0`, so a `docker run` or Compose service that
|
|
||||||
worked before still works with nothing changed.
|
|
||||||
|
|
||||||
A **bare binary** is the case that changes: the listener now binds `127.0.0.1`
|
|
||||||
unless `BIND_ADDRESS` says otherwise, so a deployment that relied on reaching it
|
|
||||||
from another host becomes unreachable until it sets the address the proxy
|
|
||||||
connects to. Check the `bindAddress` field of the startup log to see what a
|
|
||||||
running process bound. See [Bind address](#bind-address).
|
|
||||||
|
|
||||||
**Downgrade is unsupported.** Once a newer binary has migrated the files there
|
|
||||||
is no way to move them back. `AutoMigrate` is additive — it adds tables, columns
|
|
||||||
and indexes and never drops or rewrites them — so an older binary will generally
|
|
||||||
open migrated files and appear to work while writing against a schema it does
|
|
||||||
not know about. The failure mode is silent divergence, not a startup error. The
|
|
||||||
only supported way back to an older version is restoring the pre-upgrade backup,
|
|
||||||
which discards everything received since that backup was taken.
|
|
||||||
|
|
||||||
### Version stamping
|
### Version stamping
|
||||||
|
|
||||||
The binary reports its version at `/.well-known/healthcheck` (the `version`
|
The binary reports its version at `/.well-known/healthcheck` (the `version`
|
||||||
field), in the UI footer, and in the startup log line (`msg=starting`,
|
field), in the UI footer, and in the startup log line (`msg=starting`,
|
||||||
`version=...`). It is also the Sentry release name, as `webhooker-{version}`.
|
`version=...`). It is also the Sentry release name, as `webhooker-{version}`.
|
||||||
|
`curl -s http://host:8080/.well-known/healthcheck` shows which build is running.
|
||||||
The value is stamped in at build time by the linker; it is not read from a file
|
The value is stamped in at build time by the linker; it is not read from a file
|
||||||
at runtime, so it identifies the build itself.
|
at runtime, so it identifies the build itself.
|
||||||
|
|
||||||
@@ -1181,26 +1143,6 @@ at rest and restrict who can read them.
|
|||||||
- `events-{uuid}.db` and `archive-….db` hold the **full payload body and
|
- `events-{uuid}.db` and `archive-….db` hold the **full payload body and
|
||||||
headers** of every event as received, including whatever the sending service
|
headers** of every event as received, including whatever the sending service
|
||||||
put in them — tokens, signatures, personal data.
|
put in them — tokens, signatures, personal data.
|
||||||
- Event databases written before
|
|
||||||
[issue #206](https://git.eeqj.de/sneak/webhooker/issues/206) was fixed **also
|
|
||||||
contain target credentials**: a GORM association upsert on the delivery and
|
|
||||||
retry write path copied `targets` rows, `config` included, into the
|
|
||||||
per-webhook database. For a Slack target the `webhookUrl` _is_ the bearer
|
|
||||||
credential, and an `http` target's URL can embed userinfo. This version never
|
|
||||||
writes those rows; the first time it opens such a file it deletes them and
|
|
||||||
vacuums the file, which removes the credential bytes rather than only
|
|
||||||
unlinking the rows. Deleting alone would not: the bytes stay readable in the
|
|
||||||
file's free pages until it is rewritten. The sweep is recorded in the file's
|
|
||||||
`user_version` only once the vacuum returns, so a sweep that fails or is
|
|
||||||
interrupted fails the open and is retried on the next one, and a file this
|
|
||||||
version has opened without error holds no leaked rows and no recoverable bytes
|
|
||||||
from them. On upgrade this rewrites each existing `events-{uuid}.db` once, on
|
|
||||||
its first open. Two cases still hand over live delivery destinations: a backup
|
|
||||||
taken from an older build, and a backup of a file this version has not yet
|
|
||||||
opened successfully. Copies already made stay affected — the sweep only
|
|
||||||
rewrites the file it opens, and freed blocks may persist in filesystem
|
|
||||||
snapshots and on the underlying storage. Rotate any target credential that was
|
|
||||||
in a backup you cannot account for.
|
|
||||||
- `webhooker.db` stores target config **unencrypted**, tracked at
|
- `webhooker.db` stores target config **unencrypted**, tracked at
|
||||||
[issue #212](https://git.eeqj.de/sneak/webhooker/issues/212), next to the
|
[issue #212](https://git.eeqj.de/sneak/webhooker/issues/212), next to the
|
||||||
session encryption key and the Argon2id password hashes.
|
session encryption key and the Argon2id password hashes.
|
||||||
@@ -1227,7 +1169,16 @@ What that means for an operator:
|
|||||||
|
|
||||||
- **The URL is a capability, so treat it as a secret.** Keep it out of logs,
|
- **The URL is a capability, so treat it as a secret.** Keep it out of logs,
|
||||||
ticket bodies, chat messages and screenshots. Anyone who reads it anywhere can
|
ticket bodies, chat messages and screenshots. Anyone who reads it anywhere can
|
||||||
post events as that sender.
|
post events as that sender. Apart from a `log` target's line, webhooker's own
|
||||||
|
log lines never carry the UUID: they name a request to the receiver by the
|
||||||
|
route pattern `/h/{uuid}`, the `webhook request received` line names the
|
||||||
|
entrypoint by its `entrypoint_id`, and the access log keeps only the scheme
|
||||||
|
and host of a `Referer`. A `log` target writes the request's headers as
|
||||||
|
received, so a `Referer` naming the entrypoint URL appears in its line, as it
|
||||||
|
does when a sender has followed a redirect from `http://` to `https://`. The
|
||||||
|
reverse proxy's access log does carry the UUID, in the request path, so it
|
||||||
|
must be protected like the URL itself (see
|
||||||
|
[Deployment behind a reverse proxy](#deployment-behind-a-reverse-proxy)).
|
||||||
- **Rotating means minting a new entrypoint, not changing a key.** There is no
|
- **Rotating means minting a new entrypoint, not changing a key.** There is no
|
||||||
way to rotate the UUID in place. To retire one, delete the entrypoint (or
|
way to rotate the UUID in place. To retire one, delete the entrypoint (or
|
||||||
deactivate it, which answers `410`) and create a new one, then point the
|
deactivate it, which answers `410`) and create a new one, then point the
|
||||||
@@ -1302,34 +1253,37 @@ markup. The CSP build runs no expressions, so every Alpine directive in
|
|||||||
`static/js/app.js`: `x-data="collapsible"` and `@click="toggle"`, never
|
`static/js/app.js`: `x-data="collapsible"` and `@click="toggle"`, never
|
||||||
`x-data="{ open: false }"` or `@click="open = !open"`.
|
`x-data="{ open: false }"` or `@click="open = !open"`.
|
||||||
|
|
||||||
A browser test in `internal/server` loads the webhook page, its edit pages and
|
A browser test in `internal/server` loads the sign-in page, the webhook page,
|
||||||
the event log under the real policy and checks that: the add entrypoint form
|
its edit pages, the event log, the webhook list, the settings page, the profile
|
||||||
stays hidden until Add is clicked; for every target type, the targets section's
|
page and the `404` error page shown for a path that does not exist, under the
|
||||||
Add shows only a choice of type with Next and Cancel, Next shows only that
|
real policy and checks that: the add entrypoint form stays hidden until Add is
|
||||||
type's fields (no url field for `database` or `log`), Cancel at either step
|
clicked; for every target type, the targets section's Add shows only a choice of
|
||||||
closes the form, and saving adds the target; a refused target comes back with
|
type with Next and Cancel, Next shows only that type's fields (no url field for
|
||||||
its form open, the values entered and the reason, and after Cancel the next Add
|
`database` or `log`), Cancel at either step closes the form, and saving adds the
|
||||||
starts with an empty form and no reason; a refused save on the target edit page
|
target; a refused target comes back with its form open, the values entered and
|
||||||
and on the webhook edit page comes back with the reason and every value entered;
|
the reason, and after Cancel the next Add starts with an empty form and no
|
||||||
the Copy button beside an entrypoint URL reads "Copied" once clicked; an
|
reason; a refused save on the target edit page and on the webhook edit page
|
||||||
entrypoint's Edit button shows its edit form in place of its description and
|
comes back with the reason and every value entered; the Copy button beside an
|
||||||
hides until the form closes, Cancel hides the form and drops what was typed, as
|
entrypoint URL reads "Copied" once clicked; an entrypoint's Edit button shows
|
||||||
does leaving the page and going back to it, and Save changes the description; of
|
its edit form in place of its description and hides until the form closes,
|
||||||
the recent events on the webhook page only the newest starts expanded, each
|
Cancel hides the form and drops what was typed, as does leaving the page and
|
||||||
expands and collapses, and Open leads to the event's own page; of the events in
|
going back to it, and Save changes the description; of the recent events on the
|
||||||
the event log only the newest starts expanded, and an event there expands and
|
webhook page only the newest starts expanded, each expands and collapses, and
|
||||||
collapses when its row's caret or its ID is clicked, and from the keyboard, but
|
Open leads to the event's own page; of the events in the event log only the
|
||||||
not when its ID is selected with the mouse, and a delivery's attempts inside it
|
newest starts expanded, and an event there expands and collapses when its row's
|
||||||
expand and collapse; and at phone width the menu button opens and closes the
|
caret or its ID is clicked, and from the keyboard, but not when its ID is
|
||||||
mobile menu, and neither the webhook page nor the event log, with a delivery's
|
selected with the mouse, and a delivery's attempts inside it expand and
|
||||||
attempts open, scrolls sideways or cuts anything off at the page's or a card's
|
collapse; at phone width the menu button opens and closes the mobile menu, and
|
||||||
edge. It also fails if the browser reports a console warning or error, an
|
neither the webhook page nor the event log, with a delivery's attempts open,
|
||||||
uncaught exception, or anything the policy refused. `make check` and the image
|
scrolls sideways or cuts anything off at the page's or a card's edge; and a
|
||||||
build lint it but do not run it, and `make test` leaves it out (its file is
|
target's Delete asks first, naming the target, sends nothing when the answer is
|
||||||
built only with the `browser` build tag). Run it with `make test-browser` after
|
no and deletes the target when it is yes. It also fails if the browser reports a
|
||||||
changing `templates/` or `static/js/`: that builds `Dockerfile.browser`, which
|
console warning or error, an uncaught exception, or anything the policy refused.
|
||||||
runs the test in a digest-pinned headless browser image, so the host needs no
|
`make check` and the image build lint it but do not run it, and `make test`
|
||||||
browser.
|
leaves it out (its file is built only with the `browser` build tag). Run it with
|
||||||
|
`make test-browser` after changing `templates/` or `static/js/`: that builds
|
||||||
|
`Dockerfile.browser`, which runs the test in a digest-pinned headless browser
|
||||||
|
image, so the host needs no browser.
|
||||||
|
|
||||||
The package's tarball is committed as `3p/alpinejs-csp-3.14.9.tgz`, byte for
|
The package's tarball is committed as `3p/alpinejs-csp-3.14.9.tgz`, byte for
|
||||||
byte as the npm registry publishes it. It is a dependency, not this repo's build
|
byte as the npm registry publishes it. It is a dependency, not this repo's build
|
||||||
@@ -1362,7 +1316,9 @@ and commit the regenerated file with the change. Tailwind takes classes only
|
|||||||
from the files that `input.css` names in its `@source` lines; a class written in
|
from the files that `input.css` names in its `@source` lines; a class written in
|
||||||
any other file is not generated until that file is named there too. `make check`
|
any other file is not generated until that file is named there too. `make check`
|
||||||
and the image build fail when the committed file differs from what `make css`
|
and the image build fail when the committed file differs from what `make css`
|
||||||
generates. `static/css/style.css` is hand-written and is not generated.
|
generates. `static/css/style.css` is hand-written and is not generated. The
|
||||||
|
pages' Content-Security-Policy refuses inline styles, so a template styles an
|
||||||
|
element with a class, never a `style` attribute.
|
||||||
|
|
||||||
`make css` runs the Tailwind standalone CLI in Docker, at the version and sha256
|
`make css` runs the Tailwind standalone CLI in Docker, at the version and sha256
|
||||||
pinned in the Dockerfile's stylesheet stages; it is never installed on the host.
|
pinned in the Dockerfile's stylesheet stages; it is never installed on the host.
|
||||||
@@ -1384,20 +1340,22 @@ webhooker solves this by acting as a durable intermediary:
|
|||||||
nothing is lost if downstream targets are unavailable.
|
nothing is lost if downstream targets are unavailable.
|
||||||
|
|
||||||
2. **Guaranteed delivery** — Events are queued for delivery to each configured
|
2. **Guaranteed delivery** — Events are queued for delivery to each configured
|
||||||
target. Failed deliveries are retried with configurable backoff. Every
|
target. Failed deliveries are retried with configurable backoff. Delivery
|
||||||
delivery attempt is logged with status codes, response bodies, and timing.
|
attempts are logged with status codes, response bodies, and timing.
|
||||||
|
|
||||||
**That guarantee is at-least-once, not exactly-once.** When a send reaches
|
**That guarantee is at-least-once, not exactly-once.** When a send reaches
|
||||||
its target but the write recording that outcome fails, the delivery is
|
its target but the write recording that outcome fails, the delivery is
|
||||||
deliberately left in a recoverable state rather than marked done — losing a
|
deliberately left in a recoverable state rather than marked done — losing a
|
||||||
delivery is the worse failure — so the pending sweep picks it up about
|
delivery is the worse failure — so the pending sweep picks it up about
|
||||||
fifteen minutes later, or the next restart does, and the target receives a
|
fifteen minutes later, or the next restart does, and the target receives a
|
||||||
payload it already got. webhooker adds no delivery identifier of its own to
|
payload it already got. A send still in flight when the service stops is
|
||||||
an outbound request, so **make your receiver idempotent** against whatever
|
left the same way: its attempt is not recorded, and the next start sends it
|
||||||
the payload itself carries.
|
again (see Shutdown). webhooker adds no delivery identifier of its own to an
|
||||||
|
outbound request, so **make your receiver idempotent** against whatever the
|
||||||
|
payload itself carries.
|
||||||
|
|
||||||
3. **Observability** — Full request/response logging for every webhook received
|
3. **Observability** — Full request/response logging for every webhook received
|
||||||
and every delivery attempted. Prometheus metrics expose volume, latency, and
|
and for delivery attempts. Prometheus metrics expose volume, latency, and
|
||||||
error rates. The web UI provides real-time visibility into event flow.
|
error rates. The web UI provides real-time visibility into event flow.
|
||||||
|
|
||||||
4. **Fan-out** — A single incoming webhook can be delivered to multiple targets
|
4. **Fan-out** — A single incoming webhook can be delivered to multiple targets
|
||||||
@@ -1435,10 +1393,10 @@ It uses:
|
|||||||
management
|
management
|
||||||
- **[go-chi](https://github.com/go-chi/chi)** for HTTP routing
|
- **[go-chi](https://github.com/go-chi/chi)** for HTTP routing
|
||||||
- **[GORM](https://gorm.io)** for database access with
|
- **[GORM](https://gorm.io)** for database access with
|
||||||
**[modernc.org/sqlite](https://pkg.go.dev/modernc.org/sqlite)** as the runtime
|
**[modernc.org/sqlite](https://pkg.go.dev/modernc.org/sqlite)**, which is pure
|
||||||
SQLite driver. Note: `gorm.io/driver/sqlite` transitively depends on
|
Go, as the SQLite driver. `gorm.io/driver/sqlite` also imports
|
||||||
`mattn/go-sqlite3`, which requires CGO at build time (see [Docker](#docker)
|
`mattn/go-sqlite3`, whose stub for builds without cgo is linked into the
|
||||||
section)
|
binary but never used
|
||||||
- **[slog](https://pkg.go.dev/log/slog)** (stdlib) for structured logging with
|
- **[slog](https://pkg.go.dev/log/slog)** (stdlib) for structured logging with
|
||||||
TTY detection (text for dev, JSON for prod)
|
TTY detection (text for dev, JSON for prod)
|
||||||
- **[gorilla/sessions](https://github.com/gorilla/sessions)** for encrypted
|
- **[gorilla/sessions](https://github.com/gorilla/sessions)** for encrypted
|
||||||
@@ -1601,8 +1559,7 @@ The cap is not arbitrary: the reaper computes its cutoff as a `time.Duration`,
|
|||||||
an int64 nanosecond count, and a longer period overflows it. An overflowed
|
an int64 nanosecond count, and a longer period overflows it. An overflowed
|
||||||
cutoff lands in the future, where it matches every row, so the sweep would
|
cutoff lands in the future, where it matches every row, so the sweep would
|
||||||
delete every event the webhook has instead of none. The reaper also clamps the
|
delete every event the webhook has instead of none. The reaper also clamps the
|
||||||
value it is given, so a row written by an older version cannot trigger that
|
value it reads to the cap, so no stored value can trigger that either.
|
||||||
either.
|
|
||||||
|
|
||||||
#### Entrypoint
|
#### Entrypoint
|
||||||
|
|
||||||
@@ -1836,7 +1793,7 @@ is what bounds it.
|
|||||||
#### DeliveryResult
|
#### DeliveryResult
|
||||||
|
|
||||||
The result of a single delivery attempt. Every attempt (including retries) is
|
The result of a single delivery attempt. Every attempt (including retries) is
|
||||||
individually logged for full observability.
|
individually logged, except one cut off by a stop (see Shutdown).
|
||||||
|
|
||||||
| Field | Type | Description |
|
| Field | Type | Description |
|
||||||
| --------------- | ------- | ----------------------------------------- |
|
| --------------- | ------- | ----------------------------------------- |
|
||||||
@@ -1918,8 +1875,9 @@ Deliveries still `pending` or `retrying` count in neither.
|
|||||||
|
|
||||||
#### Event-tier indexes
|
#### Event-tier indexes
|
||||||
|
|
||||||
These indexes on the per-webhook event databases are declared in the model tags,
|
These indexes on the per-webhook event databases are created by
|
||||||
so `AutoMigrate` creates them on a fresh database:
|
`internal/db/migrations/events/001_schema.sql`, and declared in the model tags
|
||||||
|
to match:
|
||||||
|
|
||||||
| Table | Columns | Serves |
|
| Table | Columns | Serves |
|
||||||
| ------------------ | ------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
| ------------------ | ------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
@@ -1983,8 +1941,9 @@ and application state:
|
|||||||
- **Targets** — delivery destination configurations
|
- **Targets** — delivery destination configurations
|
||||||
- **APIKeys** — programmatic access credentials
|
- **APIKeys** — programmatic access credentials
|
||||||
|
|
||||||
On first startup the main database is auto-migrated, a session encryption key is
|
On first startup the main database is created from its SQL files (see
|
||||||
generated and stored, and an `admin` user is created.
|
[Schema](#schema)), a session encryption key is generated and stored, and an
|
||||||
|
`admin` user is created.
|
||||||
|
|
||||||
**Per-Webhook Event Databases** (`{DATA_DIR}/events-{webhook_uuid}.db`) — each
|
**Per-Webhook Event Databases** (`{DATA_DIR}/events-{webhook_uuid}.db`) — each
|
||||||
webhook gets its own dedicated SQLite file containing:
|
webhook gets its own dedicated SQLite file containing:
|
||||||
@@ -1996,9 +1955,10 @@ webhook gets its own dedicated SQLite file containing:
|
|||||||
the above, the deliveries per target, and each entrypoint's last event, kept
|
the above, the deliveries per target, and each entrypoint's last event, kept
|
||||||
through retention
|
through retention
|
||||||
|
|
||||||
Per-webhook databases are created automatically when a webhook is created. They
|
Per-webhook databases are created automatically when a webhook is created, from
|
||||||
are managed by the `WebhookDBManager` component, which handles connection
|
the SQL files in `internal/db/migrations/events/`. They hold no copy of the main
|
||||||
pooling, lazy opening, migrations, and cleanup.
|
database's tables. They are managed by the `WebhookDBManager` component, which
|
||||||
|
handles connection pooling, lazy opening, migrations, and cleanup.
|
||||||
|
|
||||||
A per-webhook database that is missing or zero-length later means its webhook's
|
A per-webhook database that is missing or zero-length later means its webhook's
|
||||||
events and pending deliveries are gone. The next time it is opened, an empty one
|
events and pending deliveries are gone. The next time it is opened, an empty one
|
||||||
@@ -2082,14 +2042,15 @@ the service again.
|
|||||||
After each write the archive handle is closed and reopened, debounced to at most
|
After each write the archive handle is closed and reopened, debounced to at most
|
||||||
once per second, so an operator can move the archive file away for offline
|
once per second, so an operator can move the archive file away for offline
|
||||||
archiving without stopping the service; a moved or removed archive file is
|
archiving without stopping the service; a moved or removed archive file is
|
||||||
recreated automatically on the next write. A zero-length archive file is written
|
recreated automatically on the next write, from the SQL files in
|
||||||
to as a new archive: SQLite opens it as an empty database, so it holds nothing
|
`internal/db/migrations/archive/`. A zero-length archive file is written to as a
|
||||||
to lose. An optional `expiry` in the target's config JSON (e.g.
|
new archive: SQLite opens it as an empty database, so it holds nothing to lose.
|
||||||
`{"expiry":"720h"}`) is validated when the target is created — the default
|
An optional `expiry` in the target's config JSON (e.g. `{"expiry":"720h"}`) is
|
||||||
(unset or the literal `never`) keeps rows forever — and rows older than the
|
validated when the target is created — the default (unset or the literal
|
||||||
expiry are pruned each time the archive is (re)opened. An archive write failure
|
`never`) keeps rows forever — and rows older than the expiry are pruned each
|
||||||
is never silent success: the delivery records a failed attempt with the error
|
time the archive is (re)opened. An archive write failure is never silent
|
||||||
and is marked failed.
|
success: the delivery records a failed attempt with the error and is marked
|
||||||
|
failed.
|
||||||
|
|
||||||
Because reopens only happen on writes, an archive whose target has stopped
|
Because reopens only happen on writes, an archive whose target has stopped
|
||||||
receiving events would never be pruned. A background **archive sweeper** closes
|
receiving events would never be pruned. A background **archive sweeper** closes
|
||||||
@@ -2174,9 +2135,22 @@ endpoint. That is the JSON key; the error text for a missing one reads
|
|||||||
`webhook_url is required`, which is the message, not the key.
|
`webhook_url is required`, which is the message, not the key.
|
||||||
|
|
||||||
The database uses the
|
The database uses the
|
||||||
[modernc.org/sqlite](https://pkg.go.dev/modernc.org/sqlite) driver at runtime,
|
[modernc.org/sqlite](https://pkg.go.dev/modernc.org/sqlite) driver.
|
||||||
though CGO is required at build time due to the transitive `mattn/go-sqlite3`
|
|
||||||
dependency from `gorm.io/driver/sqlite`.
|
#### Schema
|
||||||
|
|
||||||
|
Each kind of database file has its schema in SQL files built into the binary,
|
||||||
|
one directory per kind under `internal/db/migrations/`. In each,
|
||||||
|
`000_migration.sql` creates the `schema_migrations` table, which records the
|
||||||
|
number of every file the database has had, and `001_schema.sql` holds the whole
|
||||||
|
schema. Opening a database runs, in one transaction, every file in its directory
|
||||||
|
that `schema_migrations` does not list:
|
||||||
|
|
||||||
|
- `main/` for the main database, on connect — settings, users, API keys,
|
||||||
|
webhooks, entrypoints and targets
|
||||||
|
- `events/` for each event database, when it is lazily opened — events,
|
||||||
|
deliveries, delivery results and their running totals
|
||||||
|
- `archive/` for each archive database, on every open and reopen
|
||||||
|
|
||||||
### Request Flow
|
### Request Flow
|
||||||
|
|
||||||
@@ -2255,12 +2229,16 @@ This means:
|
|||||||
buffer rather than spawning more goroutines.
|
buffer rather than spawning more goroutines.
|
||||||
- **Independent results** — each worker records its own delivery result in the
|
- **Independent results** — each worker records its own delivery result in the
|
||||||
per-webhook database without coordination.
|
per-webhook database without coordination.
|
||||||
- **Graceful shutdown** — cancel the context, workers finish their current task
|
- **Graceful shutdown** — the stop hook cancels the workers' context. A worker
|
||||||
and exit. The stop hook waits for the pool via `lifecycle.WaitForShutdown`,
|
may still take tasks already queued before it exits. An `http` or `slack`
|
||||||
which bounds that wait by fx's stop timeout rather than blocking forever on a
|
request in flight is cut off, and one started after the cancel fails at once.
|
||||||
wedged worker. On timeout it logs at `ERROR` and returns an error, and the
|
Neither is recorded or counted in the metrics, and the next start sends the
|
||||||
goroutines that did not finish are still running — an unclean shutdown is
|
delivery again (see Shutdown). A `database` or `log` task runs as normal. The
|
||||||
reported rather than hidden.
|
stop hook waits for the pool via `lifecycle.WaitForShutdown`, which bounds
|
||||||
|
that wait by fx's stop timeout rather than blocking forever on a wedged
|
||||||
|
worker. On timeout it logs at `ERROR` and returns an error, and the goroutines
|
||||||
|
that did not finish are still running — an unclean shutdown is reported rather
|
||||||
|
than hidden.
|
||||||
|
|
||||||
**Recovery paths:**
|
**Recovery paths:**
|
||||||
|
|
||||||
@@ -2366,7 +2344,7 @@ getting anywhere.
|
|||||||
| Metric | Type | Meaning |
|
| Metric | Type | Meaning |
|
||||||
| -------------------------------------- | --------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
| -------------------------------------- | --------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
| `webhooker_events_received_total` | counter | Events received and durably stored. Compare against the delivery counters on one dashboard |
|
| `webhooker_events_received_total` | counter | Events received and durably stored. Compare against the delivery counters on one dashboard |
|
||||||
| `webhooker_delivery_attempts_total` | counter | Delivery attempts actually dispatched to a target. A delivery a circuit breaker refused is not one: it is counted as a retry instead, but only when the refusal moves it into `retrying` |
|
| `webhooker_delivery_attempts_total` | counter | Delivery attempts actually dispatched to a target. A delivery a circuit breaker refused is not one: it is counted as a retry instead, but only when the refusal moves it into `retrying`. An attempt cut off by a stop is not counted (see Shutdown) |
|
||||||
| `webhooker_deliveries_succeeded_total` | counter | Deliveries that reached `delivered` |
|
| `webhooker_deliveries_succeeded_total` | counter | Deliveries that reached `delivered` |
|
||||||
| `webhooker_deliveries_failed_total` | counter | Deliveries that failed terminally and will not be retried |
|
| `webhooker_deliveries_failed_total` | counter | Deliveries that failed terminally and will not be retried |
|
||||||
| `webhooker_delivery_retries_total` | counter | Deliveries put back into `retrying` |
|
| `webhooker_delivery_retries_total` | counter | Deliveries put back into `retrying` |
|
||||||
@@ -2465,27 +2443,32 @@ log volume it caps rather than eliminates. A path that names no entrypoint is
|
|||||||
recorded by the handler at `DEBUG`, and the aggregate limiter logs its own
|
recorded by the handler at `DEBUG`, and the aggregate limiter logs its own
|
||||||
rejections at `DEBUG` and without the path, so neither appears at all under the
|
rejections at `DEBUG` and without the path, so neither appears at all under the
|
||||||
default level. The per-entrypoint limiter is the loud one: it logs every
|
default level. The per-entrypoint limiter is the loud one: it logs every
|
||||||
rejection at `WARN` with the request path, which on this route is
|
rejection at `WARN`, naming the route pattern `/h/{uuid}` rather than the
|
||||||
attacker-controlled text. A client hammering a single invented path is served
|
request path. A client hammering a single invented path is served
|
||||||
`RECEIVER_RATE_LIMIT` requests and has the rest of its aggregate budget rejected
|
`RECEIVER_RATE_LIMIT` requests and has the rest of its aggregate budget rejected
|
||||||
there, so the aggregate limit is what bounds the _number_ of those `WARN` lines
|
there, so the aggregate limit is what bounds the _number_ of those `WARN` lines
|
||||||
— to under ten times `RECEIVER_RATE_LIMIT` per minute per client IP, 1080 at the
|
— to under ten times `RECEIVER_RATE_LIMIT` per minute per client IP, 1080 at the
|
||||||
defaults, where before it there was no bound at all. Their _width_ is bounded by
|
defaults, where before it there was no bound at all. The access log is bounded
|
||||||
the field budgets below, the same ones the access log spends. The access log is
|
by neither limit: every request is recorded once at `INFO`, served or rejected
|
||||||
bounded by neither limit: every request is recorded once at `INFO`, served or
|
alike.
|
||||||
rejected alike.
|
|
||||||
|
|
||||||
What the access log does bound is the _content_ of those lines. A 3xx or 4xx
|
What the access log does bound is the _content_ of those lines. On the receiver
|
||||||
response logs the chi route pattern — `/h/{uuid}`, `/user/{username}//`, or the
|
route it logs the route pattern `/h/{uuid}` whatever the status, because the
|
||||||
literal `(unmatched)` when the request hit no route at all — in place of the
|
concrete path carries the entrypoint UUID, which is the entrypoint's credential
|
||||||
concrete URL. Those are the outcomes an unauthenticated client can drive for
|
(see
|
||||||
free: 404 and 429 on any invented receiver path, a login redirect on any
|
[The entrypoint URL is the authentication secret](#the-entrypoint-url-is-the-authentication-secret)).
|
||||||
invented profile path. Logging the URL there would let a flood write text of its
|
A request chi never routed, because it matched no route or because a middleware
|
||||||
own choosing, at a length of its own choosing, into the log. 2xx and 5xx
|
answered it first, logs the literal `(unmatched)` whatever the status. In the
|
||||||
responses keep the concrete path — a success resolved against a static route or
|
`dev` environment the CORS middleware answers a preflight that way, so a
|
||||||
against the operator's own data (on the receiver, against a stored entrypoint
|
preflight to the receiver does not log its UUID either. On every other route, a
|
||||||
UUID), and a 5xx is a bug in this service, where the exact path is the evidence
|
3xx or 4xx response logs the chi route pattern, such as `/user/{username}//`, in
|
||||||
and no client can provoke one at will.
|
place of the concrete URL. Those are the outcomes an unauthenticated client can
|
||||||
|
drive for free: a 404 on any invented path, a login redirect on any invented
|
||||||
|
profile path. Logging the URL there would let a flood write text of its own
|
||||||
|
choosing, at a length of its own choosing, into the log. 2xx and 5xx responses
|
||||||
|
keep the concrete path — a success resolved against a static route or against
|
||||||
|
the operator's own data, and a 5xx is a bug in this service, where the exact
|
||||||
|
path is the evidence and no client can provoke one at will.
|
||||||
|
|
||||||
The query string is never logged; it is replaced by the fixed marker
|
The query string is never logged; it is replaced by the fixed marker
|
||||||
`?(redacted)`. It is client-chosen on every route, and
|
`?(redacted)`. It is client-chosen on every route, and
|
||||||
@@ -2497,6 +2480,11 @@ to, `notice`, which names the line a page shows after an action, and the event
|
|||||||
log's `show`, which picks the events it lists. A query string sent to an
|
log's `show`, which picks the events it lists. A query string sent to an
|
||||||
entrypoint is not lost either: the event stores it, and the event log shows it.
|
entrypoint is not lost either: the event stores it, and the event log shows it.
|
||||||
|
|
||||||
|
The `referer` field keeps only the scheme and host of the request's `Referer`,
|
||||||
|
and logs `(redacted)` for one without them, because its path can be an
|
||||||
|
entrypoint URL: a sender that follows the `http://` to `https://` redirect in
|
||||||
|
the nginx example sends the URL it was redirected from.
|
||||||
|
|
||||||
Client-supplied request content does not leave the host by the other route
|
Client-supplied request content does not leave the host by the other route
|
||||||
either. The Sentry SDK attaches the request to every event it captures,
|
either. The Sentry SDK attaches the request to every event it captures,
|
||||||
independently of the access log, and `SendDefaultPII=false` does not cover all
|
independently of the access log, and `SendDefaultPII=false` does not cover all
|
||||||
@@ -2511,9 +2499,10 @@ The same hook rewrites the request URL. The SDK builds it as
|
|||||||
`scheme://host/path` from the concrete path, which on the receiver route is
|
`scheme://host/path` from the concrete path, which on the receiver route is
|
||||||
`/h/<uuid>` in full — and that UUID is a write capability, not an identifier:
|
`/h/<uuid>` in full — and that UUID is a write capability, not an identifier:
|
||||||
anyone holding it can post events this service accepts and its targets then
|
anyone holding it can post events this service accepts and its targets then
|
||||||
deliver. A tracker has its own retention, access control and deletion policy, so
|
deliver. What is sent is the chi route pattern instead: `http://host/h/{uuid}`.
|
||||||
the rule the access log follows above does not carry across that boundary. What
|
A tracker has its own retention, access control and deletion policy, so the
|
||||||
is sent is the chi route pattern instead: `http://host/h/{uuid}`.
|
pattern is sent on every route, including the 2xx and 5xx responses outside the
|
||||||
|
receiver for which the access log keeps the concrete path.
|
||||||
|
|
||||||
The scheme and the host are kept, and everything else in the URL is discarded
|
The scheme and the host are kept, and everything else in the URL is discarded
|
||||||
rather than edited, so a future SDK version that starts appending a query string
|
rather than edited, so a future SDK version that starts appending a query string
|
||||||
@@ -2550,10 +2539,23 @@ Where the pattern is out of reach — the transaction dispatch, an event capture
|
|||||||
outside the router, or a request that matched no route — the fallback is never
|
outside the router, or a request that matched no route — the fallback is never
|
||||||
the concrete path. The path becomes the literal `/(redacted)`, so the URL reads
|
the concrete path. The path becomes the literal `/(redacted)`, so the URL reads
|
||||||
`http://host/(redacted)`; a URL the rewrite cannot parse into a scheme is
|
`http://host/(redacted)`; a URL the rewrite cannot parse into a scheme is
|
||||||
withheld whole. A transaction event additionally carries the SDK's own
|
withheld whole.
|
||||||
`METHOD /path` name, built from the concrete path as well; it is rewritten on
|
|
||||||
the same terms, to `POST /h/{uuid}` where the pattern is known and
|
A transaction event also carries a name, which the SDK builds as `METHOD /path`
|
||||||
`POST /(redacted)` where it is not.
|
from the concrete path and gives the transaction source `url`. Sentry groups
|
||||||
|
transactions by name, so withholding it would put every request in the service
|
||||||
|
into one group per method, such as `POST /(redacted)`. Inside each Sentry
|
||||||
|
handler, a second middleware renames the transaction once the request has been
|
||||||
|
routed, to the method and the route pattern (`POST /h/{uuid}`), and sets its
|
||||||
|
source to `route`. The hook keeps a name whose source is `route` and rewrites
|
||||||
|
any other to `METHOD /(redacted)`. A path that matches nothing at the top level
|
||||||
|
has no pattern, so its transaction keeps the SDK's name and the hook withholds
|
||||||
|
it. Inside a route group, a path that matches no route in the group, and a
|
||||||
|
request the group's own middleware turns away before routing (a CSRF refusal,
|
||||||
|
the redirect to `/pages/login`), are named after the group's pattern, such as
|
||||||
|
`GET /pages/*` or `POST /hook/{sourceID}/*`, and the hook keeps that name. The
|
||||||
|
URL on a transaction event still falls back to `http://host/(redacted)`, because
|
||||||
|
the hook has no request there.
|
||||||
|
|
||||||
The headers are an allowlist for the same reason the rules above are
|
The headers are an allowlist for the same reason the rules above are
|
||||||
unconditional: the SDK's own filter removes four names and passes everything
|
unconditional: the SDK's own filter removes four names and passes everything
|
||||||
@@ -2629,9 +2631,7 @@ than it:
|
|||||||
| ------------------------------------------ | ------- | ------------------- | ------------------------------------------ |
|
| ------------------------------------------ | ------- | ------------------- | ------------------------------------------ |
|
||||||
| `request body exceeds limit` (413) | `WARN` | path, method | yes — `MaxBodySize` precedes `RequireAuth` |
|
| `request body exceeds limit` (413) | `WARN` | path, method | yes — `MaxBodySize` precedes `RequireAuth` |
|
||||||
| `csrf: token validation failed` (403) | `WARN` | path, method | yes — `CSRF` precedes `RequireAuth` |
|
| `csrf: token validation failed` (403) | `WARN` | path, method | yes — `CSRF` precedes `RequireAuth` |
|
||||||
| `... rate limit exceeded` (429) | `WARN` | path | yes, on the receiver |
|
|
||||||
| `auth middleware: unauthenticated request` | `DEBUG` | path, method | yes, by definition |
|
| `auth middleware: unauthenticated request` | `DEBUG` | path, method | yes, by definition |
|
||||||
| `entrypoint not found` | `DEBUG` | entrypoint UUID | yes, on the receiver |
|
|
||||||
| `user not found` / `invalid password` | `DEBUG` | username | yes, on the sign-in form |
|
| `user not found` / `invalid password` | `DEBUG` | username | yes, on the sign-in form |
|
||||||
| `login failure limit exceeded` (429) | `WARN` | path | yes, on the sign-in form |
|
| `login failure limit exceeded` (429) | `WARN` | path | yes, on the sign-in form |
|
||||||
| `password verification capacity exhausted` | `WARN` | path | yes, on the sign-in form |
|
| `password verification capacity exhausted` | `WARN` | path | yes, on the sign-in form |
|
||||||
@@ -2659,13 +2659,12 @@ wider of the two handlers spends more on than the client spent sending them.
|
|||||||
Every case holds each line to the 2,560-byte ceiling. That per-line ceiling is
|
Every case holds each line to the 2,560-byte ceiling. That per-line ceiling is
|
||||||
what the figure above states, and every row establishes it.
|
what the figure above states, and every row establishes it.
|
||||||
|
|
||||||
Three of the sites go further and bound the whole flood's output — the total
|
Two of the sites go further and bound the whole flood's output — the total bytes
|
||||||
bytes a run of distinct invented values wrote, which is the shape an operator
|
a run of distinct invented values wrote, which is the shape an operator sizing
|
||||||
sizing storage cares about. They are `request body exceeds limit`
|
storage cares about. They are `request body exceeds limit`
|
||||||
(`TestMaxBodySize_FloodOfOversizePathsDoesNotGrowTheLog`),
|
(`TestMaxBodySize_FloodOfOversizePathsDoesNotGrowTheLog`) and `user not found`
|
||||||
`entrypoint not found` and `user not found` (the last two through
|
(through `assertBoundedFlood`). The other rows carry no aggregate assertion; the
|
||||||
`assertBoundedFlood`). The other rows carry no aggregate assertion; the per-line
|
per-line ceiling is what they establish.
|
||||||
ceiling is what they establish.
|
|
||||||
|
|
||||||
`internal/logfield/logfield_test.go` measures the per-rune charge against what
|
`internal/logfield/logfield_test.go` measures the per-rune charge against what
|
||||||
the handlers really emit, over roughly 3,000 code points on each, so an
|
the handlers really emit, over roughly 3,000 code points on each, so an
|
||||||
@@ -2683,7 +2682,7 @@ everything else, so its lines take the level the operator set and the handler
|
|||||||
`internal/logger` selected, and every value it emits is spent through the same
|
`internal/logger` selected, and every value it emits is spent through the same
|
||||||
`internal/logfield` budget. A record-not-found is not logged as an error: it is
|
`internal/logfield` budget. A record-not-found is not logged as an error: it is
|
||||||
the expected outcome on both of those paths, and each handler already records
|
the expected outcome on both of those paths, and each handler already records
|
||||||
its own miss at `DEBUG` — bounded, per the table above — without the SQL. Slow
|
its own miss at `DEBUG`, without the SQL and within the ceiling above. Slow
|
||||||
statements are kept, at `WARN`, above the same 200 ms threshold GORM used and
|
statements are kept, at `WARN`, above the same 200 ms threshold GORM used and
|
||||||
with the statement bounded, because that report is the one thing GORM's logger
|
with the statement bounded, because that report is the one thing GORM's logger
|
||||||
gave an operator that nothing else here does. The adapter orders its cases
|
gave an operator that nothing else here does. The adapter orders its cases
|
||||||
@@ -3018,7 +3017,6 @@ webhooker/
|
|||||||
│ ├── database/
|
│ ├── database/
|
||||||
│ │ ├── base_model.go # BaseModel with UUID primary keys
|
│ │ ├── base_model.go # BaseModel with UUID primary keys
|
||||||
│ │ ├── database.go # GORM connection, migrations, admin seed
|
│ │ ├── database.go # GORM connection, migrations, admin seed
|
||||||
│ │ ├── models.go # AutoMigrate for config-tier models
|
|
||||||
│ │ ├── model_setting.go # Setting entity (key-value app config)
|
│ │ ├── model_setting.go # Setting entity (key-value app config)
|
||||||
│ │ ├── model_user.go # User entity
|
│ │ ├── model_user.go # User entity
|
||||||
│ │ ├── model_webhook.go # Webhook entity
|
│ │ ├── model_webhook.go # Webhook entity
|
||||||
@@ -3035,6 +3033,11 @@ webhooker/
|
|||||||
│ │ └── databasetest/ # Test support: a WebhookDBManager for tests in other packages
|
│ │ └── databasetest/ # Test support: a WebhookDBManager for tests in other packages
|
||||||
│ ├── datadir/
|
│ ├── datadir/
|
||||||
│ │ └── lock.go # Exclusive advisory lock on DATA_DIR (one instance)
|
│ │ └── lock.go # Exclusive advisory lock on DATA_DIR (one instance)
|
||||||
|
│ ├── db/migrations/
|
||||||
|
│ │ ├── migrations.go # Applies a database file's SQL files when it is opened
|
||||||
|
│ │ ├── main/ # 000_migration.sql and 001_schema.sql for webhooker.db
|
||||||
|
│ │ ├── events/ # The same for each events-{uuid}.db
|
||||||
|
│ │ └── archive/ # The same for each archive file
|
||||||
│ ├── globals/
|
│ ├── globals/
|
||||||
│ │ └── globals.go # Build-time variables (appname, version, arch)
|
│ │ └── globals.go # Build-time variables (appname, version, arch)
|
||||||
│ ├── gormlog/
|
│ ├── gormlog/
|
||||||
@@ -3108,8 +3111,8 @@ webhooker/
|
|||||||
│ ├── static.go # //go:embed directive
|
│ ├── static.go # //go:embed directive
|
||||||
│ ├── css/input.css # Tailwind input, source for tailwind.css (make css)
|
│ ├── css/input.css # Tailwind input, source for tailwind.css (make css)
|
||||||
│ ├── css/tailwind.css # Generated stylesheet the pages load
|
│ ├── css/tailwind.css # Generated stylesheet the pages load
|
||||||
│ ├── css/style.css # Hand-written, loaded after tailwind.css: btn-small, the pointer cursor for input.css's buttons, the webhook list cards' focus outline
|
│ ├── css/style.css # Hand-written, loaded after tailwind.css: btn-small, the pointer cursor for input.css's buttons, the webhook list cards' focus outline, the rule hiding x-cloak elements
|
||||||
│ ├── js/app.js # Copy-to-clipboard, and the Alpine.js components
|
│ ├── js/app.js # Copy-to-clipboard, the delete prompts, and the Alpine.js components
|
||||||
│ └── js/alpine.min.js # Alpine.js CSP build, extracted from 3p/ by make assets, not committed
|
│ └── js/alpine.min.js # Alpine.js CSP build, extracted from 3p/ by make assets, not committed
|
||||||
├── templates/ # Go HTML templates (base, login, sources, etc.)
|
├── templates/ # Go HTML templates (base, login, sources, etc.)
|
||||||
├── script/ # Scripts to Rule Them All entrypoints
|
├── script/ # Scripts to Rule Them All entrypoints
|
||||||
@@ -3163,8 +3166,9 @@ Applied to all routes in this order:
|
|||||||
|
|
||||||
1. **RequestID** — Generate unique request IDs (chi built-in)
|
1. **RequestID** — Generate unique request IDs (chi built-in)
|
||||||
2. **SecurityHeaders** — Production security headers on every response (HSTS
|
2. **SecurityHeaders** — Production security headers on every response (HSTS
|
||||||
unless `HSTS_MAX_AGE` is `0`, X-Content-Type-Options, X-Frame-Options, CSP,
|
unless `HSTS_MAX_AGE` is `0`, X-Content-Type-Options, X-Frame-Options, CSP
|
||||||
Referrer-Policy, Permissions-Policy)
|
`default-src 'self'; script-src 'self'; style-src 'self'`, Referrer-Policy,
|
||||||
|
Permissions-Policy)
|
||||||
3. **Logging** — Structured request logging (method, URL, status, latency,
|
3. **Logging** — Structured request logging (method, URL, status, latency,
|
||||||
remote IP, client IP, user agent, request ID)
|
remote IP, client IP, user agent, request ID)
|
||||||
4. **Metrics** — Prometheus HTTP metrics (if `METRICS_USERNAME` and
|
4. **Metrics** — Prometheus HTTP metrics (if `METRICS_USERNAME` and
|
||||||
@@ -3174,7 +3178,8 @@ Applied to all routes in this order:
|
|||||||
7. **Recoverer** — Panic recovery: one `ERROR` record through `internal/logger`
|
7. **Recoverer** — Panic recovery: one `ERROR` record through `internal/logger`
|
||||||
and a `500`
|
and a `500`
|
||||||
8. **Sentry** — Error reporting to Sentry (if `SENTRY_DSN` is set; configured
|
8. **Sentry** — Error reporting to Sentry (if `SENTRY_DSN` is set; configured
|
||||||
with `Repanic: true` so panics still reach Recoverer)
|
with `Repanic: true` so panics still reach Recoverer), and inside it the
|
||||||
|
naming of the request's Sentry transaction after its route pattern
|
||||||
|
|
||||||
Recoverer sits seventh rather than first, and both neighbours are the reason. It
|
Recoverer sits seventh rather than first, and both neighbours are the reason. It
|
||||||
runs **inside** everything that observes the response, so the `500` it writes
|
runs **inside** everything that observes the response, so the `500` it writes
|
||||||
@@ -3189,9 +3194,9 @@ set a header or start a timer.
|
|||||||
|
|
||||||
Each admin page route group (`/pages`, `/user/*`, `/settings`, `/hooks`,
|
Each admin page route group (`/pages`, `/user/*`, `/settings`, `/hooks`,
|
||||||
`/hook/*`) starts with its own **Recoverer** and, if `SENTRY_DSN` is set, its
|
`/hook/*`) starts with its own **Recoverer** and, if `SENTRY_DSN` is set, its
|
||||||
own **Sentry** error reporting. That Recoverer answers a panic with the `500`
|
own **Sentry** error reporting and transaction naming. That Recoverer answers a
|
||||||
error page in the normal layout; the global one keeps the plain-text `500` for
|
panic with the `500` error page in the normal layout; the global one keeps the
|
||||||
every other route.
|
plain-text `500` for every other route.
|
||||||
|
|
||||||
Additionally, form endpoints (`/pages`, `/user/*`, `/settings`, `/hooks`,
|
Additionally, form endpoints (`/pages`, `/user/*`, `/settings`, `/hooks`,
|
||||||
`/hook/*`) apply a **MaxBodySize** middleware that limits POST/PUT/PATCH request
|
`/hook/*`) apply a **MaxBodySize** middleware that limits POST/PUT/PATCH request
|
||||||
@@ -3248,6 +3253,12 @@ credential check, see [The login endpoint](#the-login-endpoint).
|
|||||||
- Production security headers on all responses: HSTS (unless `HSTS_MAX_AGE` is
|
- Production security headers on all responses: HSTS (unless `HSTS_MAX_AGE` is
|
||||||
`0`), X-Content-Type-Options (`nosniff`), X-Frame-Options (`DENY`),
|
`0`), X-Content-Type-Options (`nosniff`), X-Frame-Options (`DENY`),
|
||||||
Content-Security-Policy, Referrer-Policy, and Permissions-Policy
|
Content-Security-Policy, Referrer-Policy, and Permissions-Policy
|
||||||
|
- The Content-Security-Policy is
|
||||||
|
`default-src 'self'; script-src 'self'; style-src 'self'`: a page takes
|
||||||
|
scripts and styles only from files webhooker serves, and the browser runs no
|
||||||
|
inline script or event handler and ignores any `<style>` element or `style`
|
||||||
|
attribute in a page. The pages need none: their scripts are in `static/js/`
|
||||||
|
and their styles in `static/css/`
|
||||||
- Request body size limits (1 MB) on all form POST endpoints, enforced by
|
- Request body size limits (1 MB) on all form POST endpoints, enforced by
|
||||||
middleware that runs before CSRF parses the form
|
middleware that runs before CSRF parses the form
|
||||||
- **CSRF protection** via [gorilla/csrf](https://github.com/gorilla/csrf) on all
|
- **CSRF protection** via [gorilla/csrf](https://github.com/gorilla/csrf) on all
|
||||||
@@ -3315,7 +3326,8 @@ stop-hook log:
|
|||||||
3. `server` — the HTTP drain, bounded by `server.ShutdownTimeout` (**3
|
3. `server` — the HTTP drain, bounded by `server.ShutdownTimeout` (**3
|
||||||
seconds**) and by what the hooks before it left, then a Sentry flush if
|
seconds**) and by what the hooks before it left, then a Sentry flush if
|
||||||
`SENTRY_DSN` is set
|
`SENTRY_DSN` is set
|
||||||
4. `delivery.Engine` — waits for its workers, then closes the archive databases
|
4. `delivery.Engine` — cuts off any `http` or `slack` request in flight, waits
|
||||||
|
for its workers, then closes the archive databases
|
||||||
5. `healthcheck`
|
5. `healthcheck`
|
||||||
6. `WebhookDBManager`
|
6. `WebhookDBManager`
|
||||||
7. the database close
|
7. the database close
|
||||||
@@ -3326,6 +3338,12 @@ by the stop context, so a wedge there consumes the 5 seconds before the HTTP
|
|||||||
server hook is ever entered. The hooks after the server are microsecond-scale in
|
server hook is ever entered. The hooks after the server are microsecond-scale in
|
||||||
normal operation.
|
normal operation.
|
||||||
|
|
||||||
|
The delivery engine cancels a delivery request in flight. That attempt is not
|
||||||
|
recorded or counted in the metrics, and the delivery keeps its `pending` or
|
||||||
|
`retrying` status, so startup recovery sends it again on the next start (see
|
||||||
|
Recovery paths under Bounded Worker Pool). If the receiver already had it, it
|
||||||
|
receives it twice: the at-least-once duplicate described under Rationale.
|
||||||
|
|
||||||
The HTTP drain budget is deliberately **shorter** than the sequence budget. Were
|
The HTTP drain budget is deliberately **shorter** than the sequence budget. Were
|
||||||
the two equal, a drain that used its whole budget would exhaust the sequence
|
the two equal, a drain that used its whole budget would exhaust the sequence
|
||||||
budget at the instant it finished, and every later hook — the delivery engine,
|
budget at the instant it finished, and every later hook — the delivery engine,
|
||||||
@@ -3448,14 +3466,12 @@ fixed independently of the compiler's:
|
|||||||
a step's log, and then fails. `make test` builds this stage alone.
|
a step's log, and then fails. `make test` builds this stage alone.
|
||||||
6. **Builder stage** (`golang:1.26.1-bookworm`) — depends on the lint and test
|
6. **Builder stage** (`golang:1.26.1-bookworm`) — depends on the lint and test
|
||||||
phases and the `css-check` and `markdown-check` stages passing (it copies a
|
phases and the `css-check` and `markdown-check` stages passing (it copies a
|
||||||
file from each), runs `make build` (which extracts Alpine.js from `3p/`
|
file from each), and runs `make build` (which extracts Alpine.js from `3p/`
|
||||||
first), and then rebuilds the binary with `CGO_ENABLED=1` and static linking
|
first). `make build` compiles with cgo off, so the binary is statically
|
||||||
so it runs on musl. Both builds go through `make build`, the relink adding
|
linked and runs on the Alpine runtime image. The version is the `VERSION`
|
||||||
its `-extldflags` via `GO_LDFLAGS`, so neither can drop the `-X` that stamps
|
build arg if one is given, otherwise derived from the `.git` in the context,
|
||||||
the version. The version is the `VERSION` build arg if one is given,
|
and the stage fails if a context with `.git` would stamp an empty version,
|
||||||
otherwise derived from the `.git` in the context, and the stage fails if a
|
`dev` or `unknown` (see [Version stamping](#version-stamping)).
|
||||||
context with `.git` would stamp an empty version, `dev` or `unknown` (see
|
|
||||||
[Version stamping](#version-stamping)).
|
|
||||||
7. **Runtime stage** (`alpine:3.21`) — copies the static binary and
|
7. **Runtime stage** (`alpine:3.21`) — copies the static binary and
|
||||||
`deploy/docker-entrypoint.sh`, creates the `/var/lib/webhooker` directory for
|
`deploy/docker-entrypoint.sh`, creates the `/var/lib/webhooker` directory for
|
||||||
all SQLite databases, exposes port 8080, and includes a health check against
|
all SQLite databases, exposes port 8080, and includes a health check against
|
||||||
@@ -3467,11 +3483,6 @@ The lint and test phases invoke `gofmt`, `golangci-lint` and `go test` directly
|
|||||||
rather than `make fmt-check`, `make lint` and `make test`: those targets build
|
rather than `make fmt-check`, `make lint` and `make test`: those targets build
|
||||||
docker stages, which would need a docker daemon inside this build.
|
docker stages, which would need a docker daemon inside this build.
|
||||||
|
|
||||||
The lint phase and the Go stages use Debian rather than Alpine because
|
|
||||||
`gorm.io/driver/sqlite` pulls in `mattn/go-sqlite3`, which needs CGO and does
|
|
||||||
not compile against musl. Only the final binary is statically linked, which is
|
|
||||||
what lets it run on the Alpine runtime image.
|
|
||||||
|
|
||||||
`script/cibuild` is the CI gate: it runs `script/bootstrap`, then
|
`script/cibuild` is the CI gate: it runs `script/bootstrap`, then
|
||||||
`script/check`, then builds the image, whose build runs the lint and test phases
|
`script/check`, then builds the image, whose build runs the lint and test phases
|
||||||
and the stylesheet and Markdown checks again. A build that succeeds is a repo
|
and the stylesheet and Markdown checks again. A build that succeeds is a repo
|
||||||
|
|||||||
@@ -12,24 +12,14 @@ One issue per unit of work, one branch and one PR per issue:
|
|||||||
`main` without notice. One `next` -> `main` PR accumulates the milestone;
|
`main` without notice. One `next` -> `main` PR accumulates the milestone;
|
||||||
releases are cut from `main` separately.
|
releases are cut from `main` separately.
|
||||||
|
|
||||||
Issue branches do NOT touch this file — the manager maintains it on `next`.
|
The tracker (https://git.eeqj.de/sneak/webhooker/issues) is the record of what
|
||||||
Every branch editing `TODO.md` conflicts with every other (#112).
|
is done and what is under way. This file keeps no list of either.
|
||||||
|
|
||||||
# Status
|
# Status
|
||||||
|
|
||||||
The milestone (https://git.eeqj.de/sneak/webhooker/milestone/9) is the
|
What is still open for 1.0.0 is in its milestone
|
||||||
authoritative list, and the only place to read a count or a state of play from.
|
(https://git.eeqj.de/sneak/webhooker/milestone/9). `v1.0.0` is not tagged, and
|
||||||
This file records where the project is, not what is in flight: a sentence whose
|
only @sneak tags it.
|
||||||
truth depends on a branch being unmerged is wrong the moment it merges, and this
|
|
||||||
file has been wrong that way before.
|
|
||||||
|
|
||||||
The durability defect that held the tag has landed
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/256, commit `8d64259`). Every SQLite
|
|
||||||
handle opens with WAL journaling and a busy timeout, a bookkeeping write that
|
|
||||||
fails leaves its delivery in a recoverable state rather than a lying one, and
|
|
||||||
recovery skips a delivery that already has a successful result row. Final
|
|
||||||
pre-tag verification exercised it and confirmed it holds. Whatever the milestone
|
|
||||||
still shows open is what remains before `v1.0.0`.
|
|
||||||
|
|
||||||
Delivery is at-least-once by design, not by accident: a send whose result row
|
Delivery is at-least-once by design, not by accident: a send whose result row
|
||||||
does not land is attempted again, so a receiver can see a duplicate. That is
|
does not land is attempted again, so a receiver can see a duplicate. That is
|
||||||
@@ -38,338 +28,31 @@ under Rationale. It is not a defect to re-file.
|
|||||||
|
|
||||||
# Next Step
|
# Next Step
|
||||||
|
|
||||||
Clear the rest of the open 1.0.0 milestone
|
@sneak runs webhooker in limited production, then tags `v1.0.0`. Merging `next`
|
||||||
(https://git.eeqj.de/sneak/webhooker/milestone/9) and tag `v1.0.0`. Merging
|
into `main` is a separate act from tagging and waits on neither of those: `next`
|
||||||
`next` into `main` is a separate act from tagging and waits on neither of those:
|
is kept mergeable at all times, which is the point of the branch.
|
||||||
`next` is kept mergeable at all times, which is the point of the branch.
|
|
||||||
|
|
||||||
# Completed Steps
|
|
||||||
|
|
||||||
- 2026-08-24 Bind the plaintext HTTP listener deliberately, via `BIND_ADDRESS`
|
|
||||||
defaulting to `127.0.0.1`, and document the reverse-proxy deployment. A
|
|
||||||
hostname, an empty value or a value carrying a port is a startup error, and
|
|
||||||
the `Dockerfile` sets `0.0.0.0` because a loopback bind inside a container is
|
|
||||||
unreachable (https://git.eeqj.de/sneak/webhooker/issues/268). The same commit
|
|
||||||
removed the shutdown race: `httpServer` is built in the constructor rather
|
|
||||||
than assigned from the serving goroutine, which orders the write before every
|
|
||||||
fx hook and rules out the nil dereference a SIGTERM arriving first would have
|
|
||||||
caused, and `sentryEnabled` is an `atomic.Bool`
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/226)
|
|
||||||
- 2026-08-24 Remove inbound request signature verification. The entrypoint UUID
|
|
||||||
is the authentication secret, so the per-entrypoint shared secret, the
|
|
||||||
`internal/signature` package, the receiver check, the model fields and the
|
|
||||||
forms are all gone. This reverses the feature that landed earlier in the same
|
|
||||||
milestone (https://git.eeqj.de/sneak/webhooker/issues/67,
|
|
||||||
https://git.eeqj.de/sneak/webhooker/issues/279)
|
|
||||||
- 2026-08-24 Stamp the build version into the binary and render it in the UI
|
|
||||||
footer. `script/version` is the single source — `$VERSION`, else
|
|
||||||
`git describe --tags --always --dirty`, else `unknown` — so a `make build`
|
|
||||||
binary and a `make docker` image from one checkout report the same thing, and
|
|
||||||
nothing in it varies between two builds of the same commit, which the release
|
|
||||||
gate's byte-identical assertion would catch
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/253)
|
|
||||||
- 2026-08-24 Derive cookie `Secure` and CSRF strictness from the request
|
|
||||||
transport rather than from `WEBHOOKER_ENVIRONMENT`. Behind a real TLS proxy
|
|
||||||
with the environment left at its `dev` default, the session cookie silently
|
|
||||||
lost `Secure` while the CSRF cookie on the same response kept it.
|
|
||||||
`X-Forwarded-Proto` is now matched case-insensitively on its first
|
|
||||||
comma-separated element, so `HTTPS` and `https, http` no longer fall to the
|
|
||||||
relaxed CSRF path (https://git.eeqj.de/sneak/webhooker/issues/269)
|
|
||||||
- 2026-08-24 Roll back a failed webhook deletion instead of committing it. A
|
|
||||||
failing delete committed whatever had already succeeded, hard-deleted the
|
|
||||||
per-webhook event database anyway, and redirected as though it had worked —
|
|
||||||
orphaned config plus permanently destroyed history, reported as success. All
|
|
||||||
three delete positions now roll back with the event database intact
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/262)
|
|
||||||
- 2026-08-24 Name a deleted target on its historical deliveries, marked
|
|
||||||
`(deleted)`, rather than leaving the event log unable to say where a delivery
|
|
||||||
went. A deleted target's credentials stay masked exactly as a live one's, and
|
|
||||||
it cannot become deliverable again through the receiver, resubmit, replay, the
|
|
||||||
edit form or the toggle (https://git.eeqj.de/sneak/webhooker/issues/211)
|
|
||||||
- 2026-08-24 Bound both request-controlled `/metrics` label dimensions, so the
|
|
||||||
unauthenticated receiver is no longer a memory-exhaustion vector: `handler`
|
|
||||||
carries the chi route pattern, and `method` folds anything chi cannot route
|
|
||||||
onto a single `(unmatched)` sentinel. Both were reproduced before the fix —
|
|
||||||
300 random method tokens took the series count from 106 to 7,631, and path
|
|
||||||
flooding reached 62,532 — and a label audit across a live scrape found no
|
|
||||||
third unbounded dimension (https://git.eeqj.de/sneak/webhooker/issues/254,
|
|
||||||
https://git.eeqj.de/sneak/webhooker/issues/261)
|
|
||||||
- 2026-08-24 Validate `max_retries` on both target forms. `abc`, `2.7` and `-5`
|
|
||||||
silently became 0 — fire-and-forget — including on the edit path, where it
|
|
||||||
destroyed a working value, and `999999999` stored verbatim. The ceiling of 20
|
|
||||||
is the `max` both templates already declared
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/221)
|
|
||||||
- 2026-08-24 Resubmit a stored event as a new undelivered event, so a backend
|
|
||||||
under development can be tested against real captured traffic. Per-delivery
|
|
||||||
replay cannot serve that: it re-sends one finished delivery to its own
|
|
||||||
original target, and a target created for a dev backend has no prior delivery
|
|
||||||
to replay. Resubmit re-injects the stored event at the top of the receiver
|
|
||||||
path and fans it out to whatever targets are active now
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/250)
|
|
||||||
- 2026-08-20 Take an exclusive lock on `DATA_DIR` at startup, so two instances
|
|
||||||
on one directory cannot both deliver
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/201)
|
|
||||||
- 2026-08-20 Shut down the app when the HTTP listener fails. The `OnStart` hook
|
|
||||||
returned as soon as the serving goroutine was spawned, so a failed listen left
|
|
||||||
fx reporting RUNNING and a live process with nothing bound — invisible to
|
|
||||||
systemd and Docker restart policies
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/200)
|
|
||||||
- 2026-08-20 Stop target credentials leaking into the per-webhook event
|
|
||||||
databases (https://git.eeqj.de/sneak/webhooker/issues/206), log SQL with
|
|
||||||
placeholders rather than bound values
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/207), and fail loudly on half-set
|
|
||||||
metrics auth credentials (https://git.eeqj.de/sneak/webhooker/issues/205)
|
|
||||||
- 2026-08-20 Read queue depths with `Find`, not `Scan`. `Scan` swaps GORM's own
|
|
||||||
trace recorder in for the logging adapter, and that recorder does not
|
|
||||||
implement `gorm.ParamsFilter`, so those statements logged their bound values
|
|
||||||
interpolated and bypassed the suppression above. The two units gated green
|
|
||||||
against a `next` that lacked the other, and `next` went red when both landed
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/234)
|
|
||||||
- 2026-08-20 Render per-attempt delivery detail in the event log
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/202) and add replay of a
|
|
||||||
terminally failed delivery (https://git.eeqj.de/sneak/webhooker/issues/203)
|
|
||||||
- 2026-08-20 Expose delivery metrics on `/metrics`
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/209) and document the backup,
|
|
||||||
restore and upgrade procedures
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/210)
|
|
||||||
- 2026-08-20 Add a `webhooker resetpw` subcommand and a bootstrap banner. The
|
|
||||||
admin bootstrap password was printed once among roughly 45 fx lines, and under
|
|
||||||
`docker run -d` went to container logs subject to rotation; there was no reset
|
|
||||||
path at all, so recovery meant hand-deleting the users row, documented
|
|
||||||
nowhere. The password is read from stdin or generated, never from argv where
|
|
||||||
`/proc` would publish it (https://git.eeqj.de/sneak/webhooker/issues/208)
|
|
||||||
- 2026-08-20 Add `ALLOWED_EGRESS_CIDRS`, an allowlist-only escape hatch for the
|
|
||||||
SSRF guard, so a self-hosted proxy can forward into the operator's own
|
|
||||||
network. The guard's always-blocked set cannot be reopened by configuration
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/204)
|
|
||||||
- 2026-08-20 Harden operator-set target headers, which were carried unsafely
|
|
||||||
across a redirect (https://git.eeqj.de/sneak/webhooker/issues/233)
|
|
||||||
- 2026-08-20 Add a target edit form with headers and timeout fields
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/127)
|
|
||||||
- 2026-08-18 Raise `script/test`'s per-package timeout from 30s to 90s, matching
|
|
||||||
the org-wide backstop. `go test` applies `-timeout` per package, and
|
|
||||||
`internal/handlers` had grown past the old budget: a cache-defeated build
|
|
||||||
failed outright at `GOMAXPROCS=4`, and every run under deliberate host load
|
|
||||||
breached 30s. The measurement table lives in the script (#194)
|
|
||||||
- 2026-08-18 Re-sync `REPO_POLICIES.md` from `prompts`. The local copy was stale
|
|
||||||
and still mandated a 20s test target with a 30s timeout, which the org
|
|
||||||
replaced with a 60s cap and a 90s backstop. A synced copy is not a source;
|
|
||||||
reading it as one nearly produced a PR against `prompts` proposing a change
|
|
||||||
already merged there (#196)
|
|
||||||
- 2026-08-18 Report handler panics through the logger and answer 500. chi
|
|
||||||
v1.5.5's `Recoverer` scans for a `panic(0x` frame the runtime no longer emits,
|
|
||||||
then indexes `pkg[-1:]`, so it panicked inside its own stack printer before
|
|
||||||
writing a byte: the recovery never ran, the client got a dropped connection
|
|
||||||
instead of a 500, and the original panic was lost. A local middleware replaces
|
|
||||||
it, bounded by `MaxPanicLogLineBytes` (#187)
|
|
||||||
- 2026-08-18 Route GORM's logger through `slog` and bound it. Every `gorm.Open`
|
|
||||||
left `logger.Default` in place at `Warn` with `IgnoreRecordNotFoundError`
|
|
||||||
false, so **every record-not-found printed the fully interpolated SQL to
|
|
||||||
stdout** — including the client-chosen path on `/webhook/{uuid}` and the
|
|
||||||
submitted username on the login form, at no level the operator set and outside
|
|
||||||
`internal/logger` entirely. Three call sites, not the two the issue named
|
|
||||||
(#178)
|
|
||||||
- 2026-08-18 Bound every `slog` line against client-chosen text. Eight sites
|
|
||||||
reachable unauthenticated, found by reading every `slog` call in the tree
|
|
||||||
rather than only the one reported; the budget moved to a shared
|
|
||||||
`internal/logfield` so no second truncation exists. `DEBUG` being off by
|
|
||||||
default is not a bound and is not treated as one (#176)
|
|
||||||
- 2026-08-18 Stop a slow host turning a login-guard test into a segfault. A
|
|
||||||
non-fatal `assert` on an acquire result was dereferenced on the next line, so
|
|
||||||
one timing miss killed the whole `internal/middleware` binary and reddened CI
|
|
||||||
for unrelated PRs. The fix also removed a real production race — `acquire`
|
|
||||||
could shed a request with a slot standing free, because Go picks uniformly
|
|
||||||
among ready `select` cases (#186)
|
|
||||||
- 2026-08-18 Send the chi route pattern to Sentry rather than the concrete path.
|
|
||||||
The receiver's path carries the entrypoint capability token, so every Sentry
|
|
||||||
event from `/webhook/{uuid}` shipped a live credential to a third party.
|
|
||||||
Request `Data`, `QueryString`, `Cookies` and `Env` are dropped and headers
|
|
||||||
reduced to an allowlist (#179)
|
|
||||||
- 2026-08-18 Read form fields from the POST body only. `r.FormValue` merges the
|
|
||||||
query string, so a login could be driven by URL parameters — putting the
|
|
||||||
password somewhere that lands in access logs, proxy logs and browser history
|
|
||||||
(#160)
|
|
||||||
- 2026-08-18 Verify login credentials before spending rate-limit budget, so a
|
|
||||||
flood of wrong passwords cannot lock out the account it is guessing at. The
|
|
||||||
manager took this decision rather than stall the queue; it is flagged on the
|
|
||||||
issue for reversal (#150)
|
|
||||||
- 2026-08-18 Run all linting in Docker via `Dockerfile.lint`. Host lint was
|
|
||||||
wrong in both directions from version skew and shared caches. `script/lint`
|
|
||||||
asserts the summary line, because `--no-cache-filter` silently ignores a stage
|
|
||||||
name it does not match — the flag that makes the gate meaningful fails open
|
|
||||||
(#109)
|
|
||||||
- 2026-08-18 Serve an event's full stored body over HTTP. The list query
|
|
||||||
truncates for rendering, and that truncated value was the only way to read a
|
|
||||||
body, so the full payload was unreachable (#157)
|
|
||||||
- 2026-08-18 Bound the access log line against client-chosen text.
|
|
||||||
`internal/logfield` budgets by _encoded_ bytes, not runes, so a handler's JSON
|
|
||||||
escaping cannot multiply a field past its allowance (#146)
|
|
||||||
- 2026-08-18 Mark superseded CI commits `failure` rather than `skipped`. A
|
|
||||||
skipped run rolls up green, so a commit that was never tested reported success
|
|
||||||
(#152)
|
|
||||||
- 2026-08-18 Set `fx.StopTimeout` inside the container stop grace, so shutdown
|
|
||||||
hooks are bounded by a deadline the orchestrator will actually honour rather
|
|
||||||
than being killed mid-flush (#134)
|
|
||||||
- 2026-08-17 Bucket IPv6 rate-limit keys by `/64`. A single allocation hands out
|
|
||||||
2^64 addresses, so per-address keying let one client mint unlimited buckets.
|
|
||||||
Manager decision, recorded on the issue (#125)
|
|
||||||
- 2026-08-17 Correct release-blocking README and startup-warning inaccuracies,
|
|
||||||
including claims about behaviour the code does not have (#151)
|
|
||||||
- 2026-08-17 Fetch and verify Alpine.js at build time against
|
|
||||||
`static/vendor.sha256` instead of committing the minified blob, so the
|
|
||||||
dependency is pinned by hash rather than by trust (#145)
|
|
||||||
- 2026-08-17 Bound the event log's rendered bodies in the query itself, so a
|
|
||||||
large stored payload cannot be read into memory just to be truncated for
|
|
||||||
display (#135)
|
|
||||||
- 2026-08-17 Mask the `http` target's destination URL in the UI: it can carry a
|
|
||||||
bearer credential in its path or query, and was rendered verbatim. Manager
|
|
||||||
decision to mask unconditionally (#115)
|
|
||||||
- 2026-08-14 Bound shutdown hooks by their stop context, so a hook that hangs
|
|
||||||
cannot hold the process past its grace period (#102)
|
|
||||||
- 2026-08-14 Render templates via a buffer rather than the `ResponseWriter`, so
|
|
||||||
a template error part-way through cannot commit a 200 and then fail — the
|
|
||||||
response is written only once it is whole (#123)
|
|
||||||
- 2026-08-14 Align the session codec's max-age with the 7-day absolute cap. The
|
|
||||||
codec accepted cookies the session layer considered expired, so the cap was
|
|
||||||
enforced in one place and not the other (#108)
|
|
||||||
- 2026-08-12 Warn when `TRUSTED_PROXIES` is empty in production, where the safe
|
|
||||||
default silently discards forwarded headers and every client rate-limits as
|
|
||||||
the proxy's address (#149)
|
|
||||||
- 2026-08-12 Bound the receiver rate limit per client IP across the whole
|
|
||||||
`/webhook/*` route. The existing limiter keyed on the request path and
|
|
||||||
`/webhook/{uuid}` matches any single segment, so a client that invented a
|
|
||||||
fresh path per request minted a fresh bucket per request: the limit on the
|
|
||||||
only unauthenticated endpoint bounded nothing in aggregate, and every request
|
|
||||||
still cost an entrypoint lookup before it 404ed. An outer limiter keyed on the
|
|
||||||
client address alone now bounds that, chained in front of the unchanged
|
|
||||||
per-entrypoint limiter (#139)
|
|
||||||
- 2026-08-12 Correct release-blocking documentation inaccuracies: the README
|
|
||||||
promised manual redelivery in the present tense in three places when nothing
|
|
||||||
implements it (the same false claim also sat in the doc comment that was its
|
|
||||||
source text), the env table omitted `RETENTION_SWEEP_INTERVAL`, and `TODO.md`
|
|
||||||
itself omitted five landed units (#141)
|
|
||||||
- 2026-08-12 Make the CI gate execute the checks it reports on. The workflow now
|
|
||||||
writes a build-context fingerprint before calling `script/cibuild`, so a code
|
|
||||||
commit invalidates the `COPY` layer of the lint and builder stages while a
|
|
||||||
docs-only commit still replays from cache; a superseding run also rewrites the
|
|
||||||
`failure` status Gitea leaves on commits it cancelled and never tested.
|
|
||||||
Verified by pushing a deliberately broken test and watching CI go red (#119)
|
|
||||||
- 2026-08-12 Require a positive `RETENTION_SWEEP_INTERVAL`: a non-positive value
|
|
||||||
reached `time.NewTicker` in both the retention reaper and the archive sweeper,
|
|
||||||
panicking two goroutines with no recover after startup had already reported
|
|
||||||
success (#140)
|
|
||||||
- 2026-08-12 Bound the `X-Forwarded-For` scan's allocation to the hop cap: the
|
|
||||||
reverse walk cuts entries with `strings.LastIndexByte` instead of joining and
|
|
||||||
splitting, so a 1 MB header allocates 16 bytes rather than 1.6 MB per request
|
|
||||||
on the unauthenticated receiver. Semantics proven unchanged by differential
|
|
||||||
testing against the previous implementation (#133)
|
|
||||||
- 2026-08-12 Cap the `X-Forwarded-For` hop walk at 64 entries, so an
|
|
||||||
attacker-supplied chain cannot burn unbounded CPU in the rate-limit key
|
|
||||||
function; running off the end falls back to the peer address (#124)
|
|
||||||
- 2026-08-12 Gate forwarded-header trust behind a `TRUSTED_PROXIES` CIDR list:
|
|
||||||
all three rate limiters key on the connection's own address unless the direct
|
|
||||||
peer is a configured proxy, in which case `X-Forwarded-For` is walked right to
|
|
||||||
left for the first non-proxy hop. Default trusts nothing, and a
|
|
||||||
set-but-unparseable value aborts startup. Before this, any client could mint a
|
|
||||||
fresh bucket or drain another's by rotating a spoofed header (#88)
|
|
||||||
- 2026-08-11 Web UI cleanup: nav terminology unified on Webhooks, the Profile
|
|
||||||
settings placeholder removed, a progressive-enhancement copy button for the
|
|
||||||
entrypoint URL, and retention form copy that states the actual policy
|
|
||||||
(deletion by the reaper, 0 retains forever) (#57)
|
|
||||||
- 2026-08-11 Mask the webhook credential in delivery errors and logs: Go embeds
|
|
||||||
the request URL in `*url.Error`, so every transport failure persisted the full
|
|
||||||
Slack webhook URL into the per-webhook event database via
|
|
||||||
`DeliveryResult.Error`, a field a future REST API would have served.
|
|
||||||
`maskURLError` drops path, query and userinfo while preserving the wrapped
|
|
||||||
cause, so `errors.Is`/`As` and `Timeout()` still work and DNS, TLS and timeout
|
|
||||||
failures still read differently (#118)
|
|
||||||
- 2026-08-11 Rate-limit the public webhook receiver endpoint
|
|
||||||
(`RECEIVER_RATE_LIMIT`, default 120/min), keyed on client IP plus entrypoint
|
|
||||||
path so one entrypoint cannot exhaust another's budget; over-limit requests
|
|
||||||
get 429 with `Retry-After`. It was the one unauthenticated, internet-facing
|
|
||||||
endpoint with no limit at all (#64)
|
|
||||||
- 2026-08-11 Enforce the body size limit before CSRF parses the form:
|
|
||||||
`MaxBodySize` is now first in all four form-parsing route groups, so an
|
|
||||||
oversized request is rejected with 413 instead of being read in full by the
|
|
||||||
CSRF middleware before any cap applied (#90)
|
|
||||||
- 2026-08-11 Mask target config on the source detail page, which rendered the
|
|
||||||
stored blob verbatim and so exposed the Slack incoming-webhook URL — a bearer
|
|
||||||
credential that cannot be revoked per-holder. Config reaches the template only
|
|
||||||
as a `TargetView` of labelled fields, and header values are rendered as a
|
|
||||||
count (#113)
|
|
||||||
- 2026-08-11 Allow `retention_days` of 0 to mean retain forever, via a sentinel
|
|
||||||
written in `BeforeSave` so the GORM column default cannot win the race. Also
|
|
||||||
bounds the reaper's cutoff arithmetic: day counts above 106751 overflowed
|
|
||||||
`time.Duration` and wrapped the cutoff into the future, where every row
|
|
||||||
matched and the sweep deleted everything (#79)
|
|
||||||
- 2026-08-09 Inactivity-based session timeout: sliding idle expiry
|
|
||||||
(`SESSION_IDLE_TIMEOUT`, default `24h`) refreshed on authenticated requests,
|
|
||||||
with the 7-day absolute cap kept as an independent backstop that activity
|
|
||||||
never extends (#66)
|
|
||||||
- 2026-08-09 Restart recovery and the 60s retry sweep terminally fail an
|
|
||||||
orphaned `retrying` delivery whose target type no longer supports retries,
|
|
||||||
recording a `DeliveryResult` with the reason instead of leaving the delivery
|
|
||||||
stuck forever (#82)
|
|
||||||
- 2026-08-09 Root the delivery engine's worker pool and the retention reaper's
|
|
||||||
sweep loop at `context.Background()` rather than the fx `OnStart` hook context
|
|
||||||
(#97), which carries fx's 15s start timeout and killed both roughly fifteen
|
|
||||||
seconds after boot: the proxy silently stopped delivering webhooks entirely,
|
|
||||||
and the reaper never ran a single sweep under its default one-hour interval
|
|
||||||
- 2026-08-09 Archive writer lifecycle (#89): deleting a webhook (or its last
|
|
||||||
`database` target) evicts the cached archive writer and closes its handle
|
|
||||||
while deliberately leaving `archive-{webhookID}.db` on disk, and a new
|
|
||||||
`ArchiveSweeper` prunes idle archives on the existing
|
|
||||||
`RETENTION_SWEEP_INTERVAL` without ever creating an archive file
|
|
||||||
- 2026-08-09 Configuration parsing fails loudly on set-but-unparseable
|
|
||||||
environment values: `envInt` removed in favour of `envPositiveInt` plus a
|
|
||||||
`PORT` range check, `envBool` now parses with `strconv.ParseBool`, and
|
|
||||||
defaults apply only to unset variables (#80)
|
|
||||||
- 2026-08-07 Automatic event retention cleanup based on `retention_days`,
|
|
||||||
deleting expired events, deliveries, and delivery results from each
|
|
||||||
per-webhook event database (#63)
|
|
||||||
- 2026-08-07 Update golangci-lint to v2.12.2 (Docker image digest in
|
|
||||||
`Dockerfile`, release-archive sha256 pins in `script/bootstrap`), adopt the
|
|
||||||
canonical `.golangci.yml` (v2 `linters.settings` layout so
|
|
||||||
`lll`/`funlen`/`cyclop`/`dupl` thresholds actually apply), and fix all newly
|
|
||||||
surfaced lint findings
|
|
||||||
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, Makefile
|
|
||||||
shims, README Entrypoints section
|
|
||||||
- 2026-03-25 pin golangci-lint Docker image for linting (#55)
|
|
||||||
- 2026-03-18 CSRF middleware detects TLS per-request, fixing login over plain
|
|
||||||
HTTP and behind reverse proxies (#54)
|
|
||||||
- 2026-03-17 root path redirects based on auth state (#52)
|
|
||||||
- 2026-03-17 CSRF protection, SSRF prevention for HTTP delivery targets with DNS
|
|
||||||
rebinding defense, and per-IP login rate limiting (#42)
|
|
||||||
- 2026-03-17 Slack target type for incoming webhook notifications (#47)
|
|
||||||
- 2026-03-17 Dockerfile absolute paths and static linking (#49); absolute dev
|
|
||||||
DATA_DIR default and clarified env docs (#46)
|
|
||||||
- 2026-03-05 security headers middleware, session regeneration on login, request
|
|
||||||
body size limits (#41)
|
|
||||||
- 2026-03-04 tests for delivery, middleware, and session packages (#32); removed
|
|
||||||
the build-architecture global (#31)
|
|
||||||
- 2026-03-04 1.0 MVP merge: Webhook/Entrypoint/Target rename, core delivery
|
|
||||||
engine with bounded worker pool and circuit breaker, parallel fan-out,
|
|
||||||
per-webhook event databases, management UI (#16)
|
|
||||||
- 2026-03-01 repo brought to REPO_POLICIES standards; TODO.md folded into README
|
|
||||||
(#6)
|
|
||||||
|
|
||||||
# Future Steps
|
# Future Steps
|
||||||
|
|
||||||
- Delivery status and retry management UI. Replay of a terminally failed
|
- Per-webhook rate limits, set in the web UI and enforced in the receiver
|
||||||
delivery and per-attempt detail already landed
|
handler, on top of the receiver's existing limits, which `RECEIVER_RATE_LIMIT`
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/203,
|
sets for the whole server (see Rate Limiting in the README)
|
||||||
https://git.eeqj.de/sneak/webhooker/issues/202)
|
- API key authentication for programmatic access (the `APIKey` model exists, but
|
||||||
- Per-webhook rate limiting in the receiver handler (per-webhook config plus
|
nothing creates a key and no middleware checks one)
|
||||||
handler enforcement; global limits must not apply to receiver endpoints)
|
|
||||||
- API key authentication for programmatic access (APIKey model exists; Bearer
|
|
||||||
token middleware does not)
|
|
||||||
- REST API v1
|
- REST API v1
|
||||||
- CRUD for webhooks, entrypoints, targets
|
- CRUD for webhooks, entrypoints, targets
|
||||||
- event viewing and filtering endpoints
|
- event viewing and filtering endpoints
|
||||||
- event redelivery endpoint
|
- event redelivery endpoint
|
||||||
- OpenAPI specification
|
- OpenAPI specification
|
||||||
- Analytics dashboard: success rates, response times, volume
|
- Analytics in the web UI: response times summarised per webhook or per target,
|
||||||
- A remember-me option at login
|
and figures for all webhooks together. The webhook page already shows event,
|
||||||
|
delivery and failure counts and failure percentages, the target list counts
|
||||||
|
each target's deliveries, the event log and each event's page show every
|
||||||
|
attempt's duration, and `/metrics` carries server-wide counts and delivery
|
||||||
|
durations
|
||||||
|
- A remember-me option at login. Every session already survives a browser
|
||||||
|
restart: it lasts up to 7 days, and ends sooner after `SESSION_IDLE_TIMEOUT`
|
||||||
|
(default 24 hours) without activity
|
||||||
- Password reset flow for a forgotten password over the web. The authenticated
|
- Password reset flow for a forgotten password over the web. The authenticated
|
||||||
password _change_ flow already landed, and a lost password is recoverable from
|
password _change_ flow already landed, and a lost password is recoverable from
|
||||||
the console with `webhooker resetpw`
|
the console with `webhooker resetpw`
|
||||||
@@ -377,7 +60,13 @@ Clear the rest of the open 1.0.0 milestone
|
|||||||
- Later, nice to have
|
- Later, nice to have
|
||||||
- email delivery target type
|
- email delivery target type
|
||||||
- SNS and S3 delivery targets
|
- SNS and S3 delivery targets
|
||||||
- data transformations (e.g. webhook to Slack message formatting)
|
- data transformations (e.g. a configurable Slack message format; the Slack
|
||||||
- JSONL file delivery with periodic S3 upload
|
target sends one fixed format today)
|
||||||
- webhook event search and filtering
|
- JSONL file delivery with periodic S3 upload (the `database` target already
|
||||||
- multi-user with role-based access control
|
archives events to SQLite files, rotated hourly, daily or monthly if set,
|
||||||
|
and the webhook page offers them as one gzipped JSON download)
|
||||||
|
- webhook event search, and filtering beyond the event log's failed and
|
||||||
|
pending views
|
||||||
|
- multi-user with role-based access control (each webhook already belongs to
|
||||||
|
one account, but there is only the account made at first boot and no way
|
||||||
|
to add another)
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import (
|
|||||||
"sneak.berlin/go/webhooker/internal/banner"
|
"sneak.berlin/go/webhooker/internal/banner"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/datadir"
|
"sneak.berlin/go/webhooker/internal/datadir"
|
||||||
|
"sneak.berlin/go/webhooker/internal/db/migrations"
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
)
|
)
|
||||||
@@ -249,12 +250,21 @@ func (d *Database) connectTo(dataDir string) error {
|
|||||||
return fmt.Errorf("migrating %s: %w", dbPath, err)
|
return fmt.Errorf("migrating %s: %w", dbPath, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
err = parseModels(
|
||||||
|
db, &Setting{}, &User{}, &APIKey{},
|
||||||
|
&Webhook{}, &Entrypoint{}, &Target{},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("connecting to %s: %w", dbPath, err)
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// migrate applies the main database's SQL files. See
|
||||||
|
// internal/db/migrations.
|
||||||
func (d *Database) migrate() error {
|
func (d *Database) migrate() error {
|
||||||
// Run GORM auto-migrations
|
err := migrations.Apply(d.db, migrations.Main)
|
||||||
err := d.Migrate()
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
d.log.Error(
|
d.log.Error(
|
||||||
"failed to run database migrations",
|
"failed to run database migrations",
|
||||||
@@ -269,6 +279,21 @@ func (d *Database) migrate() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parseModels has GORM parse each model for db before the handle is
|
||||||
|
// shared. GORM parses a model the first time a handle uses it, and two
|
||||||
|
// goroutines doing that at once for models that refer to each other,
|
||||||
|
// as Webhook and Target do, race inside GORM.
|
||||||
|
func parseModels(db *gorm.DB, models ...any) error {
|
||||||
|
for _, model := range models {
|
||||||
|
err := (&gorm.Statement{DB: db}).Parse(model)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("parsing model %T: %w", model, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// ensureAdminUser creates the bootstrap admin account when the
|
// ensureAdminUser creates the bootstrap admin account when the
|
||||||
// deployment has no users at all.
|
// deployment has no users at all.
|
||||||
func (d *Database) ensureAdminUser() error {
|
func (d *Database) ensureAdminUser() error {
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package database
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
|
||||||
|
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"gorm.io/gorm/clause"
|
"gorm.io/gorm/clause"
|
||||||
@@ -18,14 +17,14 @@ const omitAssociationsCallback = "webhooker:omit_associations"
|
|||||||
//
|
//
|
||||||
// A per-webhook database holds the event tier only, but Delivery
|
// A per-webhook database holds the event tier only, but Delivery
|
||||||
// declares belongs-to Event and Target and the delivery engine fills
|
// declares belongs-to Event and Target and the delivery engine fills
|
||||||
// both in memory before writing. Without this guard GORM upserts
|
// both in memory before writing. Without this guard GORM upserts those
|
||||||
// those parent rows here on the delivery and retry write paths,
|
// parent rows here on the delivery and retry write paths, and the
|
||||||
// copying targets.config, which holds destination URLs and bearer
|
// target, whose config holds destination URLs and bearer credentials,
|
||||||
// credentials, into the file most likely to be backed up or handed
|
// goes to a targets table this file does not have, which fails the
|
||||||
// to someone else. Registering the guard on the connection covers
|
// write. Registering the guard on the connection
|
||||||
// every write path, including writes inside a transaction and write
|
// covers every write path, including writes inside a transaction and
|
||||||
// paths added later. Every event-tier row this file holds is written
|
// write paths added later. Every event-tier row this file holds is
|
||||||
// explicitly, so nothing depends on the automatic save.
|
// written explicitly, so nothing depends on the automatic save.
|
||||||
func omitAssociations(db *gorm.DB) error {
|
func omitAssociations(db *gorm.DB) error {
|
||||||
omit := func(tx *gorm.DB) {
|
omit := func(tx *gorm.DB) {
|
||||||
tx.Statement.Omits = append(
|
tx.Statement.Omits = append(
|
||||||
@@ -53,107 +52,3 @@ func omitAssociations(db *gorm.DB) error {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// eventDBSweptVersion is the PRAGMA user_version purgeTargetRows
|
|
||||||
// stamps into a per-webhook database once it has removed any leaked
|
|
||||||
// target rows *and* the VACUUM that removes their bytes has returned.
|
|
||||||
// Nothing else in the tree uses user_version, so 0 means "not swept
|
|
||||||
// by this build".
|
|
||||||
//
|
|
||||||
// The stamp, not the DELETE, is what records that a file is done. A
|
|
||||||
// DELETE commits on its own, so a sweep that is interrupted or whose
|
|
||||||
// VACUUM fails leaves a file whose rows are gone but whose credential
|
|
||||||
// bytes are still in the free pages -- indistinguishable, by row
|
|
||||||
// count, from a file that never leaked. Both leave the stamp unset,
|
|
||||||
// so the next open sweeps again.
|
|
||||||
const eventDBSweptVersion = 1
|
|
||||||
|
|
||||||
// purgeTargetRows deletes target rows that an earlier build's
|
|
||||||
// association upsert wrote into a per-webhook database, and rewrites
|
|
||||||
// the file so their bytes are gone with them. AutoMigrate creates a
|
|
||||||
// targets table in every one of these files because Delivery declares
|
|
||||||
// a belongs-to Target, but nothing in the event tier may put rows in
|
|
||||||
// it. The rows it did put there are junk, not history: they carry an
|
|
||||||
// empty webhook_id, and delivery rows resolve their target against
|
|
||||||
// the main database, so nothing here refers to them.
|
|
||||||
//
|
|
||||||
// The DELETE only unlinks the rows: modernc.org/sqlite leaves
|
|
||||||
// secure_delete at SQLite's default of off, so the credential bytes
|
|
||||||
// stay readable in the file's free pages and a backup of a swept file
|
|
||||||
// would still hand them over. VACUUM rewrites the file without them.
|
|
||||||
//
|
|
||||||
// This runs before every migration and is gated on
|
|
||||||
// eventDBSweptVersion, so a file pays for the rewrite once, on the
|
|
||||||
// first open that finds it unstamped, and every open after that is a
|
|
||||||
// PRAGMA read. A file this build created is stamped before its
|
|
||||||
// targets table exists, so it never vacuums at all. A failure here
|
|
||||||
// fails the open with the stamp left unset, so the sweep is retried
|
|
||||||
// rather than skipped -- a webhook whose file cannot be swept stays
|
|
||||||
// unusable instead of quietly serving from a file that still holds
|
|
||||||
// recoverable credentials.
|
|
||||||
func purgeTargetRows(
|
|
||||||
db *gorm.DB, log *slog.Logger, webhookID string,
|
|
||||||
) error {
|
|
||||||
var version int
|
|
||||||
|
|
||||||
// Row().Scan, not (*gorm.DB).Scan: see internal/gormlog.
|
|
||||||
err := db.Raw("PRAGMA user_version").Row().Scan(&version)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"reading sweep marker of webhook database %s: %w",
|
|
||||||
webhookID, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if version >= eventDBSweptVersion {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var purged int64
|
|
||||||
|
|
||||||
if db.Migrator().HasTable("targets") {
|
|
||||||
res := db.Exec("DELETE FROM targets")
|
|
||||||
if res.Error != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"purging target rows from webhook database %s: %w",
|
|
||||||
webhookID, res.Error,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
purged = res.RowsAffected
|
|
||||||
|
|
||||||
// Unconditional: a zero row count here does not mean there is
|
|
||||||
// nothing to remove, only that no *live* row is left. See
|
|
||||||
// eventDBSweptVersion.
|
|
||||||
err = db.Exec("VACUUM").Error
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"purged %d leaked target rows from webhook database "+
|
|
||||||
"%s but vacuuming it failed, so the deleted "+
|
|
||||||
"target credentials are still recoverable from "+
|
|
||||||
"the file; it stays marked unswept and the next "+
|
|
||||||
"open retries: %w",
|
|
||||||
purged, webhookID, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
err = db.Exec(fmt.Sprintf(
|
|
||||||
"PRAGMA user_version = %d", eventDBSweptVersion,
|
|
||||||
)).Error
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"marking webhook database %s swept: %w", webhookID, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if purged > 0 {
|
|
||||||
log.Warn(
|
|
||||||
"purged leaked target rows from per-webhook database",
|
|
||||||
"webhook_id", webhookID,
|
|
||||||
"rows", purged,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,392 +1,15 @@
|
|||||||
package database_test
|
package database_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"database/sql"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
_ "modernc.org/sqlite"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
||||||
)
|
)
|
||||||
|
|
||||||
// testDataDirPerm is the mode the test data directory is created
|
|
||||||
// with.
|
|
||||||
const testDataDirPerm = 0o750
|
|
||||||
|
|
||||||
// eventDBDataDir returns a data directory that a WebhookDBManager
|
|
||||||
// can be pointed at.
|
|
||||||
func eventDBDataDir(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
dir := filepath.Join(t.TempDir(), "events")
|
|
||||||
require.NoError(t, os.MkdirAll(dir, testDataDirPerm))
|
|
||||||
|
|
||||||
return dir
|
|
||||||
}
|
|
||||||
|
|
||||||
// openRawEventDB opens the per-webhook database file directly,
|
|
||||||
// without the manager, so a test can put a file on disk in a state
|
|
||||||
// the manager has to cope with, or inspect one afterwards.
|
|
||||||
func openRawEventDB(
|
|
||||||
t *testing.T, dataDir, webhookID string,
|
|
||||||
) *sql.DB {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
path := filepath.Join(
|
|
||||||
dataDir, fmt.Sprintf("events-%s.db", webhookID),
|
|
||||||
)
|
|
||||||
|
|
||||||
sqlDB, err := sql.Open(
|
|
||||||
"sqlite",
|
|
||||||
fmt.Sprintf("file:%s?mode=rwc", path),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
|
||||||
|
|
||||||
return sqlDB
|
|
||||||
}
|
|
||||||
|
|
||||||
// eventDBFileBytes reads a per-webhook database file off disk, so a
|
|
||||||
// test can assert on what the file itself still holds rather than on
|
|
||||||
// what a query returns.
|
|
||||||
func eventDBFileBytes(t *testing.T, dataDir, webhookID string) []byte {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
//nolint:gosec // reads a file the test just created under t.TempDir()
|
|
||||||
raw, err := os.ReadFile(filepath.Join(
|
|
||||||
dataDir, fmt.Sprintf("events-%s.db", webhookID),
|
|
||||||
))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
return raw
|
|
||||||
}
|
|
||||||
|
|
||||||
// eventDBUserVersion returns the PRAGMA user_version of a per-webhook
|
|
||||||
// database file, which is the marker purgeTargetRows stamps once it
|
|
||||||
// has swept and vacuumed.
|
|
||||||
func eventDBUserVersion(t *testing.T, sqlDB *sql.DB) int {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var version int
|
|
||||||
|
|
||||||
require.NoError(t, sqlDB.QueryRowContext(
|
|
||||||
t.Context(), "PRAGMA user_version",
|
|
||||||
).Scan(&version))
|
|
||||||
|
|
||||||
return version
|
|
||||||
}
|
|
||||||
|
|
||||||
// clearEventDBSweptMarker resets the sweep marker to 0, which is what
|
|
||||||
// a file written by a build without the sweep looks like. Tests that
|
|
||||||
// seed a leaked row have to create the file through the manager to
|
|
||||||
// get the real targets table shape, and that stamps it.
|
|
||||||
func clearEventDBSweptMarker(t *testing.T, sqlDB *sql.DB) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
_, err := sqlDB.ExecContext(t.Context(), "PRAGMA user_version = 0")
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// countTargetRows returns the number of rows in the targets table of
|
|
||||||
// a per-webhook database file, or -1 if the table does not exist.
|
|
||||||
func countTargetRows(t *testing.T, sqlDB *sql.DB) int {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var tables int
|
|
||||||
|
|
||||||
require.NoError(t, sqlDB.QueryRowContext(
|
|
||||||
t.Context(),
|
|
||||||
"SELECT count(*) FROM sqlite_master "+
|
|
||||||
"WHERE type = 'table' AND name = 'targets'",
|
|
||||||
).Scan(&tables))
|
|
||||||
|
|
||||||
if tables == 0 {
|
|
||||||
return -1
|
|
||||||
}
|
|
||||||
|
|
||||||
var rows int
|
|
||||||
|
|
||||||
require.NoError(t, sqlDB.QueryRowContext(
|
|
||||||
t.Context(), "SELECT count(*) FROM targets",
|
|
||||||
).Scan(&rows))
|
|
||||||
|
|
||||||
return rows
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestOpenPurgesLeakedTargetRows covers the sweep for event
|
|
||||||
// databases written by a build that let GORM upsert target rows
|
|
||||||
// into them: opening the database clears them, and opening it again
|
|
||||||
// is a no-op.
|
|
||||||
func TestOpenPurgesLeakedTargetRows(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dataDir := eventDBDataDir(t)
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
|
|
||||||
// Create the file the way the application does, so the targets
|
|
||||||
// table has exactly the shape AutoMigrate gives it, then write
|
|
||||||
// a leaked row into it the way the association upsert did.
|
|
||||||
initial := databasetest.NewWebhookDBManager(t, dataDir)
|
|
||||||
|
|
||||||
_, err := initial.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, initial.CloseAll())
|
|
||||||
|
|
||||||
seed := openRawEventDB(t, dataDir, webhookID)
|
|
||||||
|
|
||||||
_, err = seed.ExecContext(
|
|
||||||
t.Context(),
|
|
||||||
"INSERT INTO targets "+
|
|
||||||
"(id, webhook_id, name, type, config) "+
|
|
||||||
"VALUES (?, '', ?, ?, ?)",
|
|
||||||
uuid.New().String(),
|
|
||||||
"leaked-target",
|
|
||||||
"slack",
|
|
||||||
`{"webhookUrl":"https://hooks.example/T000/B000/secret"}`,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, 1, countTargetRows(t, seed))
|
|
||||||
clearEventDBSweptMarker(t, seed)
|
|
||||||
require.NoError(t, seed.Close())
|
|
||||||
|
|
||||||
mgr := databasetest.NewWebhookDBManager(t, dataDir)
|
|
||||||
|
|
||||||
_, err = mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, mgr.CloseAll())
|
|
||||||
|
|
||||||
check := openRawEventDB(t, dataDir, webhookID)
|
|
||||||
assert.Zero(t, countTargetRows(t, check))
|
|
||||||
assert.Equal(
|
|
||||||
t, 1, eventDBUserVersion(t, check),
|
|
||||||
"a completed sweep must mark the file so later opens skip it",
|
|
||||||
)
|
|
||||||
require.NoError(t, check.Close())
|
|
||||||
|
|
||||||
// Idempotent: a second open leaves it at zero and does not
|
|
||||||
// error.
|
|
||||||
again := databasetest.NewWebhookDBManager(t, dataDir)
|
|
||||||
|
|
||||||
_, err = again.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, again.CloseAll())
|
|
||||||
|
|
||||||
recheck := openRawEventDB(t, dataDir, webhookID)
|
|
||||||
assert.Zero(t, countTargetRows(t, recheck))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestOpenPurgeRemovesCredentialBytes covers the sweep at the level
|
|
||||||
// that matters for a backup handed to someone else: the leaked
|
|
||||||
// credential must be gone from the raw bytes of the file, not merely
|
|
||||||
// unreachable by query. A bare DELETE unlinks the row and leaves the
|
|
||||||
// bytes readable in the free pages, so this fails without the VACUUM
|
|
||||||
// in purgeTargetRows.
|
|
||||||
func TestOpenPurgeRemovesCredentialBytes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dataDir := eventDBDataDir(t)
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
credential := "T00000000/B00000000/" + uuid.New().String()
|
|
||||||
|
|
||||||
initial := databasetest.NewWebhookDBManager(t, dataDir)
|
|
||||||
|
|
||||||
_, err := initial.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, initial.CloseAll())
|
|
||||||
|
|
||||||
seed := openRawEventDB(t, dataDir, webhookID)
|
|
||||||
|
|
||||||
_, err = seed.ExecContext(
|
|
||||||
t.Context(),
|
|
||||||
"INSERT INTO targets "+
|
|
||||||
"(id, webhook_id, name, type, config) "+
|
|
||||||
"VALUES (?, '', ?, ?, ?)",
|
|
||||||
uuid.New().String(),
|
|
||||||
"leaked-target",
|
|
||||||
"slack",
|
|
||||||
fmt.Sprintf(
|
|
||||||
`{"webhookUrl":"https://hooks.example/%s"}`, credential,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
clearEventDBSweptMarker(t, seed)
|
|
||||||
require.NoError(t, seed.Close())
|
|
||||||
|
|
||||||
// The seed has to be in the file for its absence later to mean
|
|
||||||
// anything.
|
|
||||||
require.True(
|
|
||||||
t,
|
|
||||||
bytes.Contains(
|
|
||||||
eventDBFileBytes(t, dataDir, webhookID),
|
|
||||||
[]byte(credential),
|
|
||||||
),
|
|
||||||
"seeded credential is not in the file, so this test proves nothing",
|
|
||||||
)
|
|
||||||
|
|
||||||
mgr := databasetest.NewWebhookDBManager(t, dataDir)
|
|
||||||
|
|
||||||
_, err = mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, mgr.CloseAll())
|
|
||||||
|
|
||||||
assert.NotContains(
|
|
||||||
t,
|
|
||||||
string(eventDBFileBytes(t, dataDir, webhookID)),
|
|
||||||
credential,
|
|
||||||
"leaked credential is still recoverable from the raw file",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestOpenRevacuumsAfterIncompleteSweep covers the case a row count
|
|
||||||
// cannot see: the rows are already deleted but the file was never
|
|
||||||
// vacuumed, because an earlier sweep died between the two or its
|
|
||||||
// VACUUM failed. The credential bytes are still recoverable, and the
|
|
||||||
// unset marker is the only thing that says so, so the next open must
|
|
||||||
// vacuum rather than conclude from the empty table that there is
|
|
||||||
// nothing to do.
|
|
||||||
func TestOpenRevacuumsAfterIncompleteSweep(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dataDir := eventDBDataDir(t)
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
credential := "T00000000/B00000000/" + uuid.New().String()
|
|
||||||
|
|
||||||
initial := databasetest.NewWebhookDBManager(t, dataDir)
|
|
||||||
|
|
||||||
_, err := initial.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, initial.CloseAll())
|
|
||||||
|
|
||||||
seed := openRawEventDB(t, dataDir, webhookID)
|
|
||||||
|
|
||||||
_, err = seed.ExecContext(
|
|
||||||
t.Context(),
|
|
||||||
"INSERT INTO targets "+
|
|
||||||
"(id, webhook_id, name, type, config) "+
|
|
||||||
"VALUES (?, '', ?, ?, ?)",
|
|
||||||
uuid.New().String(),
|
|
||||||
"leaked-target",
|
|
||||||
"slack",
|
|
||||||
fmt.Sprintf(
|
|
||||||
`{"webhookUrl":"https://hooks.example/%s"}`, credential,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Exactly the state an interrupted sweep leaves: rows gone,
|
|
||||||
// marker unset, bytes still in the free pages.
|
|
||||||
_, err = seed.ExecContext(t.Context(), "DELETE FROM targets")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Zero(t, countTargetRows(t, seed))
|
|
||||||
clearEventDBSweptMarker(t, seed)
|
|
||||||
require.NoError(t, seed.Close())
|
|
||||||
|
|
||||||
require.True(
|
|
||||||
t,
|
|
||||||
bytes.Contains(
|
|
||||||
eventDBFileBytes(t, dataDir, webhookID),
|
|
||||||
[]byte(credential),
|
|
||||||
),
|
|
||||||
"the deleted row's bytes must still be in the file, or this "+
|
|
||||||
"test proves nothing",
|
|
||||||
)
|
|
||||||
|
|
||||||
mgr := databasetest.NewWebhookDBManager(t, dataDir)
|
|
||||||
|
|
||||||
_, err = mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, mgr.CloseAll())
|
|
||||||
|
|
||||||
assert.NotContains(
|
|
||||||
t,
|
|
||||||
string(eventDBFileBytes(t, dataDir, webhookID)),
|
|
||||||
credential,
|
|
||||||
"an interrupted sweep was not retried, so the credential is "+
|
|
||||||
"still recoverable from the raw file",
|
|
||||||
)
|
|
||||||
|
|
||||||
check := openRawEventDB(t, dataDir, webhookID)
|
|
||||||
assert.Equal(t, 1, eventDBUserVersion(t, check))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestOpenSkipsSweptDatabase covers the other half of the marker: a
|
|
||||||
// file this build created is marked without ever being vacuumed, and
|
|
||||||
// a marked file is not swept again.
|
|
||||||
func TestOpenSkipsSweptDatabase(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dataDir := eventDBDataDir(t)
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
|
|
||||||
mgr := databasetest.NewWebhookDBManager(t, dataDir)
|
|
||||||
|
|
||||||
_, err := mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, mgr.CloseAll())
|
|
||||||
|
|
||||||
marked := openRawEventDB(t, dataDir, webhookID)
|
|
||||||
assert.Equal(t, 1, eventDBUserVersion(t, marked))
|
|
||||||
|
|
||||||
// A marked file is left alone, so a row written into it survives
|
|
||||||
// a reopen. Nothing writes target rows any more; this stands in
|
|
||||||
// for the sweep having run.
|
|
||||||
_, err = marked.ExecContext(
|
|
||||||
t.Context(),
|
|
||||||
"INSERT INTO targets "+
|
|
||||||
"(id, webhook_id, name, type, config) "+
|
|
||||||
"VALUES (?, '', ?, ?, ?)",
|
|
||||||
uuid.New().String(), "sentinel", "slack", `{}`,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, marked.Close())
|
|
||||||
|
|
||||||
again := databasetest.NewWebhookDBManager(t, dataDir)
|
|
||||||
|
|
||||||
_, err = again.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, again.CloseAll())
|
|
||||||
|
|
||||||
check := openRawEventDB(t, dataDir, webhookID)
|
|
||||||
assert.Equal(
|
|
||||||
t, 1, countTargetRows(t, check),
|
|
||||||
"a marked file must not be swept again",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestOpenSucceedsWithoutTargetsTable covers an existing event
|
|
||||||
// database that never grew a targets table. The sweep must not fail
|
|
||||||
// startup on it.
|
|
||||||
func TestOpenSucceedsWithoutTargetsTable(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dataDir := eventDBDataDir(t)
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
|
|
||||||
seed := openRawEventDB(t, dataDir, webhookID)
|
|
||||||
|
|
||||||
_, err := seed.ExecContext(
|
|
||||||
t.Context(),
|
|
||||||
"CREATE TABLE events (id text PRIMARY KEY)",
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, seed.Close())
|
|
||||||
|
|
||||||
mgr := databasetest.NewWebhookDBManager(t, dataDir)
|
|
||||||
|
|
||||||
db, err := mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.NotNil(t, db)
|
|
||||||
require.NoError(t, mgr.CloseAll())
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventDBCreateOmitsAssociations covers the connection-level
|
// TestEventDBCreateOmitsAssociations covers the connection-level
|
||||||
// guard directly: a Delivery carrying its Event and Target in
|
// guard directly: a Delivery carrying its Event and Target in
|
||||||
// memory, written through the manager's handle, must store only the
|
// memory, written through the manager's handle, must store only the
|
||||||
@@ -394,10 +17,9 @@ func TestOpenSucceedsWithoutTargetsTable(t *testing.T) {
|
|||||||
func TestEventDBCreateOmitsAssociations(t *testing.T) {
|
func TestEventDBCreateOmitsAssociations(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
dataDir := eventDBDataDir(t)
|
|
||||||
webhookID := uuid.New().String()
|
webhookID := uuid.New().String()
|
||||||
|
|
||||||
mgr := databasetest.NewWebhookDBManager(t, dataDir)
|
mgr := databasetest.NewWebhookDBManager(t, t.TempDir())
|
||||||
|
|
||||||
db, err := mgr.GetDB(webhookID)
|
db, err := mgr.GetDB(webhookID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -432,8 +54,9 @@ func TestEventDBCreateOmitsAssociations(t *testing.T) {
|
|||||||
require.NoError(t, db.Model(d).
|
require.NoError(t, db.Model(d).
|
||||||
Update("status", database.DeliveryStatusDelivered).
|
Update("status", database.DeliveryStatusDelivered).
|
||||||
Error)
|
Error)
|
||||||
require.NoError(t, mgr.CloseAll())
|
|
||||||
|
|
||||||
check := openRawEventDB(t, dataDir, webhookID)
|
// The file has no targets table, so the create and update above
|
||||||
assert.Zero(t, countTargetRows(t, check))
|
// would have failed had GORM tried to save the target.
|
||||||
|
assert.False(t, db.Migrator().HasTable(&database.Target{}))
|
||||||
|
require.NoError(t, mgr.CloseAll())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,10 +13,8 @@ import (
|
|||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestWebhookDBManager_OpenAddsEventTierIndexes verifies that opening a
|
// TestWebhookDBManager_OpenAddsEventTierIndexes verifies that a new
|
||||||
// per-webhook database that predates these indexes creates them. It
|
// per-webhook database has these indexes.
|
||||||
// stands in for an older database file by dropping the indexes
|
|
||||||
// AutoMigrate just created, then reopening the same file.
|
|
||||||
func TestWebhookDBManager_OpenAddsEventTierIndexes(t *testing.T) {
|
func TestWebhookDBManager_OpenAddsEventTierIndexes(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -42,27 +40,9 @@ func TestWebhookDBManager_OpenAddsEventTierIndexes(t *testing.T) {
|
|||||||
db, err := mgr.GetDB(webhookID)
|
db, err := mgr.GetDB(webhookID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
// A fresh database has them.
|
|
||||||
for _, ix := range indexes {
|
|
||||||
require.True(t, db.Migrator().HasIndex(ix.model, ix.name))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stand in for a database file created before the indexes existed.
|
|
||||||
for _, ix := range indexes {
|
|
||||||
require.NoError(t, db.Migrator().DropIndex(ix.model, ix.name))
|
|
||||||
require.False(t, db.Migrator().HasIndex(ix.model, ix.name))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Drop the cached connection so the next open reopens the file and
|
|
||||||
// runs AutoMigrate against it, as a restart would.
|
|
||||||
require.NoError(t, mgr.CloseAll())
|
|
||||||
|
|
||||||
db, err = mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
for _, ix := range indexes {
|
for _, ix := range indexes {
|
||||||
assert.True(t, db.Migrator().HasIndex(ix.model, ix.name),
|
assert.True(t, db.Migrator().HasIndex(ix.model, ix.name),
|
||||||
"opening the existing database should create %s", ix.name)
|
"a new database should have %s", ix.name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -45,8 +45,7 @@ type Event struct {
|
|||||||
|
|
||||||
// ResubmittedFromID names the event this one was copied from by
|
// ResubmittedFromID names the event this one was copied from by
|
||||||
// an operator resubmit. It is nil for an event that arrived on
|
// an operator resubmit. It is nil for an event that arrived on
|
||||||
// the receiver, which is every event created before the column
|
// the receiver. It is not a foreign key: the source event can be
|
||||||
// existed. It is not a foreign key: the source event can be
|
|
||||||
// reaped by retention while its copies remain, and the id is
|
// reaped by retention while its copies remain, and the id is
|
||||||
// kept as the record of where the copy came from either way.
|
// kept as the record of where the copy came from either way.
|
||||||
ResubmittedFromID *string `gorm:"type:uuid;index:idx_events_resubmitted_from_id,priority:1;index:idx_events_entrypoint_id,priority:3" json:"resubmittedFromId,omitempty"`
|
ResubmittedFromID *string `gorm:"type:uuid;index:idx_events_resubmitted_from_id,priority:1;index:idx_events_entrypoint_id,priority:3" json:"resubmittedFromId,omitempty"`
|
||||||
|
|||||||
@@ -82,7 +82,7 @@ type Webhook struct {
|
|||||||
// value while building the insert statement, which runs after
|
// value while building the insert statement, which runs after
|
||||||
// BeforeSave; rewriting any later than this loses that race and the
|
// BeforeSave; rewriting any later than this loses that race and the
|
||||||
// row lands at 30 days. Living on the model also means a future call
|
// row lands at 30 days. Living on the model also means a future call
|
||||||
// site — a REST API, a fixture, a migration — cannot bypass it.
|
// site, such as a REST API or a fixture, cannot bypass it.
|
||||||
func (w *Webhook) BeforeSave(_ *gorm.DB) error {
|
func (w *Webhook) BeforeSave(_ *gorm.DB) error {
|
||||||
if w.RetentionDays <= 0 {
|
if w.RetentionDays <= 0 {
|
||||||
w.RetentionDays = RetentionForeverDays
|
w.RetentionDays = RetentionForeverDays
|
||||||
@@ -98,8 +98,8 @@ func (w *Webhook) BeforeSave(_ *gorm.DB) error {
|
|||||||
// are exempt from reaping.
|
// are exempt from reaping.
|
||||||
//
|
//
|
||||||
// It accepts the RetentionForeverDays sentinel written by BeforeSave
|
// It accepts the RetentionForeverDays sentinel written by BeforeSave
|
||||||
// and, defensively, the non-positive values that rows written before
|
// and, defensively, any non-positive value, which BeforeSave treats
|
||||||
// the sentinel existed may still carry.
|
// as a request to retain forever.
|
||||||
func retainsForever(retentionDays int) bool {
|
func retainsForever(retentionDays int) bool {
|
||||||
return retentionDays <= 0 ||
|
return retentionDays <= 0 ||
|
||||||
retentionDays >= RetentionForeverDays
|
retentionDays >= RetentionForeverDays
|
||||||
|
|||||||
@@ -1,17 +0,0 @@
|
|||||||
package database
|
|
||||||
|
|
||||||
// Migrate runs database migrations for the main application database.
|
|
||||||
// Only configuration-tier models are stored in the main database.
|
|
||||||
// Event-tier models (Event, Delivery, DeliveryResult, EventTotals,
|
|
||||||
// TargetTotals, EntrypointTotals) live in
|
|
||||||
// per-webhook dedicated databases managed by WebhookDBManager.
|
|
||||||
func (d *Database) Migrate() error {
|
|
||||||
return d.db.AutoMigrate(
|
|
||||||
&Setting{},
|
|
||||||
&User{},
|
|
||||||
&APIKey{},
|
|
||||||
&Webhook{},
|
|
||||||
&Entrypoint{},
|
|
||||||
&Target{},
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -253,7 +253,7 @@ func (r *RetentionReaper) reapWebhook(
|
|||||||
// row in the database: the sweep then deletes every event, delivery,
|
// row in the database: the sweep then deletes every event, delivery,
|
||||||
// and delivery result, including ones created seconds ago. Rejecting
|
// and delivery result, including ones created seconds ago. Rejecting
|
||||||
// out-of-range input at the form is the primary guard; saturating here
|
// out-of-range input at the form is the primary guard; saturating here
|
||||||
// means an old row, a migration, or a future call site cannot turn a
|
// means a value that reaches the row by any other path cannot turn a
|
||||||
// too-large retention into total data loss.
|
// too-large retention into total data loss.
|
||||||
func retentionCutoff(
|
func retentionCutoff(
|
||||||
now time.Time,
|
now time.Time,
|
||||||
|
|||||||
@@ -0,0 +1,168 @@
|
|||||||
|
package database_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
|
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
||||||
|
)
|
||||||
|
|
||||||
|
// withFixedNow returns db with GORM's clock stopped at a time in UTC,
|
||||||
|
// so that the created and updated times it writes come back from
|
||||||
|
// SQLite equal to the row written, which holds them in UTC.
|
||||||
|
func withFixedNow(db *gorm.DB) (*gorm.DB, time.Time) {
|
||||||
|
at := time.Date(2026, 10, 13, 12, 30, 45, 123456789, time.UTC)
|
||||||
|
|
||||||
|
return db.Session(&gorm.Session{
|
||||||
|
NowFunc: func() time.Time { return at },
|
||||||
|
}), at
|
||||||
|
}
|
||||||
|
|
||||||
|
// newBaseModel returns a BaseModel with a new ID.
|
||||||
|
func newBaseModel() database.BaseModel {
|
||||||
|
return database.BaseModel{ID: uuid.New().String()}
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertRoundTrip writes row, which must be the only row of its
|
||||||
|
// table, and reads back the same row.
|
||||||
|
func assertRoundTrip[T any](t *testing.T, db *gorm.DB, row *T) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
require.NoError(t, db.Create(row).Error)
|
||||||
|
|
||||||
|
var got T
|
||||||
|
|
||||||
|
require.NoError(t, db.First(&got).Error)
|
||||||
|
assert.Equal(t, *row, got)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMainSchemaHoldsEveryModel writes a row of each main database
|
||||||
|
// model, every field set, into a database made from its SQL files.
|
||||||
|
func TestMainSchemaHoldsEveryModel(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
mainDB, err := database.Open(
|
||||||
|
t.TempDir(), slog.New(slog.DiscardHandler),
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
t.Cleanup(func() { _ = mainDB.Close() })
|
||||||
|
|
||||||
|
db, at := withFixedNow(mainDB.DB())
|
||||||
|
|
||||||
|
assertRoundTrip(t, db, &database.Setting{
|
||||||
|
Key: "session_key", Value: "value",
|
||||||
|
})
|
||||||
|
assertRoundTrip(t, db, &database.User{
|
||||||
|
BaseModel: newBaseModel(),
|
||||||
|
Username: "user",
|
||||||
|
Password: "password hash",
|
||||||
|
})
|
||||||
|
assertRoundTrip(t, db, &database.APIKey{
|
||||||
|
BaseModel: newBaseModel(),
|
||||||
|
UserID: uuid.New().String(),
|
||||||
|
Key: "key",
|
||||||
|
Description: "API key description",
|
||||||
|
LastUsedAt: &at,
|
||||||
|
})
|
||||||
|
assertRoundTrip(t, db, &database.Webhook{
|
||||||
|
BaseModel: newBaseModel(),
|
||||||
|
UserID: uuid.New().String(),
|
||||||
|
Name: "webhook",
|
||||||
|
Description: "webhook description",
|
||||||
|
RetentionDays: 7,
|
||||||
|
})
|
||||||
|
assertRoundTrip(t, db, &database.Entrypoint{
|
||||||
|
BaseModel: newBaseModel(),
|
||||||
|
WebhookID: uuid.New().String(),
|
||||||
|
Path: uuid.New().String(),
|
||||||
|
Description: "entrypoint description",
|
||||||
|
Active: true,
|
||||||
|
})
|
||||||
|
assertRoundTrip(t, db, &database.Target{
|
||||||
|
BaseModel: newBaseModel(),
|
||||||
|
WebhookID: uuid.New().String(),
|
||||||
|
Name: "target",
|
||||||
|
Type: database.TargetTypeHTTP,
|
||||||
|
Active: true,
|
||||||
|
Config: `{"url":"https://example.com/"}`,
|
||||||
|
MaxRetries: 3,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestEventSchemaHoldsEveryModel does the same for a webhook's event
|
||||||
|
// database, which the manager makes from its SQL files.
|
||||||
|
func TestEventSchemaHoldsEveryModel(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
mgr := databasetest.NewWebhookDBManager(t, t.TempDir())
|
||||||
|
|
||||||
|
t.Cleanup(func() { _ = mgr.CloseAll() })
|
||||||
|
|
||||||
|
events, err := mgr.GetDB(uuid.New().String())
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
db, at := withFixedNow(events)
|
||||||
|
source := uuid.New().String()
|
||||||
|
|
||||||
|
assertRoundTrip(t, db, &database.Event{
|
||||||
|
BaseModel: newBaseModel(),
|
||||||
|
WebhookID: uuid.New().String(),
|
||||||
|
EntrypointID: uuid.New().String(),
|
||||||
|
Method: http.MethodPost,
|
||||||
|
RawQuery: "a=1",
|
||||||
|
Headers: `{"X":"Y"}`,
|
||||||
|
Body: "body",
|
||||||
|
ContentType: "text/plain",
|
||||||
|
BodyBytes: 4,
|
||||||
|
ResubmittedFromID: &source,
|
||||||
|
})
|
||||||
|
assertRoundTrip(t, db, &database.Delivery{
|
||||||
|
BaseModel: newBaseModel(),
|
||||||
|
EventID: uuid.New().String(),
|
||||||
|
TargetID: uuid.New().String(),
|
||||||
|
Status: database.DeliveryStatusDelivered,
|
||||||
|
FinishedAt: &at,
|
||||||
|
Replay: true,
|
||||||
|
})
|
||||||
|
assertRoundTrip(t, db, &database.DeliveryResult{
|
||||||
|
BaseModel: newBaseModel(),
|
||||||
|
DeliveryID: uuid.New().String(),
|
||||||
|
AttemptNum: 2,
|
||||||
|
Success: true,
|
||||||
|
StatusCode: 200,
|
||||||
|
ResponseBody: "ok",
|
||||||
|
Error: "error",
|
||||||
|
Duration: 15,
|
||||||
|
})
|
||||||
|
assertRoundTrip(t, db, &database.TargetTotals{
|
||||||
|
TargetID: uuid.New().String(),
|
||||||
|
Deliveries: 5,
|
||||||
|
Delivered: 3,
|
||||||
|
Failed: 1,
|
||||||
|
DeliveriesRemoved: 2,
|
||||||
|
FailedRemoved: 1,
|
||||||
|
})
|
||||||
|
assertRoundTrip(t, db, &database.EntrypointTotals{
|
||||||
|
EntrypointID: uuid.New().String(), LastEventAt: at,
|
||||||
|
})
|
||||||
|
|
||||||
|
// A new event database has its one row of event totals already.
|
||||||
|
require.NoError(t, database.AddEventTotals(db, database.EventTotals{
|
||||||
|
Events: 3, EventsRemoved: 1, LastEventAt: &at,
|
||||||
|
}))
|
||||||
|
|
||||||
|
var totals []database.EventTotals
|
||||||
|
|
||||||
|
require.NoError(t, db.Find(&totals).Error)
|
||||||
|
assert.Equal(t, []database.EventTotals{{
|
||||||
|
ID: 1, Events: 3, EventsRemoved: 1, LastEventAt: &at,
|
||||||
|
}}, totals)
|
||||||
|
}
|
||||||
@@ -118,9 +118,8 @@ const SQLiteFilePerm fs.FileMode = 0o600
|
|||||||
// SQLite still reports the absence rather than this function
|
// SQLite still reports the absence rather than this function
|
||||||
// materializing an empty database the caller asked not to create.
|
// materializing an empty database the caller asked not to create.
|
||||||
//
|
//
|
||||||
// Chmod of a file that already exists is what tightens a data
|
// The mode is set on every open, whoever created the file, so a
|
||||||
// directory an earlier build left at 0644 — including a developer's
|
// database or sidecar found at a wider mode is tightened too.
|
||||||
// own scratch directory — without any migration machinery.
|
|
||||||
func reserveSQLiteFile(path string, create bool) error {
|
func reserveSQLiteFile(path string, create bool) error {
|
||||||
if create {
|
if create {
|
||||||
// gosec G304: the path is the database file the caller asked
|
// gosec G304: the path is the database file the caller asked
|
||||||
@@ -242,7 +241,19 @@ func SQLiteDSN(path, mode string) string {
|
|||||||
)
|
)
|
||||||
q.Add("_pragma", "journal_mode(WAL)")
|
q.Add("_pragma", "journal_mode(WAL)")
|
||||||
|
|
||||||
return "file:" + path + "?" + q.Encode()
|
// SQLite reads a file: URI's path only up to the first ? or #, and
|
||||||
|
// decodes %XX escapes in it, so url.URL escapes the path. OmitHost
|
||||||
|
// keeps a relative path as "file:data/x.db"; written
|
||||||
|
// "file://data/x.db", SQLite would reject "data" as the URI's
|
||||||
|
// authority.
|
||||||
|
dsn := url.URL{
|
||||||
|
Scheme: "file",
|
||||||
|
Path: path,
|
||||||
|
OmitHost: true,
|
||||||
|
RawQuery: q.Encode(),
|
||||||
|
}
|
||||||
|
|
||||||
|
return dsn.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
// OpenSQLite opens the SQLite file at path with the service's
|
// OpenSQLite opens the SQLite file at path with the service's
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ package database_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -112,6 +114,79 @@ func TestMainDBAppliesPragmasOnALiveHandle(t *testing.T) {
|
|||||||
assert.Equal(t, "10000", busy)
|
assert.Equal(t, "10000", busy)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestOpenSQLiteOpensThePathAskedFor uses a data directory whose name
|
||||||
|
// holds ?, # and %. SQLite reads a file: URI's path only up to a ? or
|
||||||
|
// #, and decodes %41 to A, so an unescaped path sent every database in
|
||||||
|
// that directory to one file outside it.
|
||||||
|
func TestOpenSQLiteOpensThePathAskedFor(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dir := filepath.Join(t.TempDir(), "hooks#1?x%41")
|
||||||
|
require.NoError(t, os.Mkdir(dir, 0o700))
|
||||||
|
|
||||||
|
for _, name := range []string{
|
||||||
|
database.MainDBFileName,
|
||||||
|
"events-" + uuid.New().String() + ".db",
|
||||||
|
} {
|
||||||
|
path := filepath.Join(dir, name)
|
||||||
|
|
||||||
|
sqlDB, err := database.OpenSQLite(
|
||||||
|
path, database.SQLiteModeCreate,
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
t.Cleanup(func() { assert.NoError(t, sqlDB.Close()) })
|
||||||
|
|
||||||
|
assertOpenedFile(t, sqlDB, path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestOpenSQLiteOpensARelativePath opens a database at a path relative
|
||||||
|
// to the working directory, which is what a relative DATA_DIR such as
|
||||||
|
// the README's DATA_DIR=../webhooker-data gives OpenSQLite.
|
||||||
|
//
|
||||||
|
//nolint:paralleltest // t.Chdir moves the whole process.
|
||||||
|
func TestOpenSQLiteOpensARelativePath(t *testing.T) {
|
||||||
|
t.Chdir(t.TempDir())
|
||||||
|
require.NoError(t, os.Mkdir("data", 0o700))
|
||||||
|
|
||||||
|
path := filepath.Join("data", database.MainDBFileName)
|
||||||
|
|
||||||
|
sqlDB, err := database.OpenSQLite(path, database.SQLiteModeCreate)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
defer func() { require.NoError(t, sqlDB.Close()) }()
|
||||||
|
|
||||||
|
assertOpenedFile(t, sqlDB, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertOpenedFile checks that sqlDB is the SQLite file at path.
|
||||||
|
// database_list names the file SQLite actually opened, as an absolute
|
||||||
|
// path with symlinks resolved, so the files are compared, not the names.
|
||||||
|
func assertOpenedFile(t *testing.T, sqlDB *sql.DB, path string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var (
|
||||||
|
seq int
|
||||||
|
schema, file string
|
||||||
|
)
|
||||||
|
|
||||||
|
require.NoError(t, sqlDB.
|
||||||
|
QueryRowContext(t.Context(), "pragma database_list").
|
||||||
|
Scan(&seq, &schema, &file))
|
||||||
|
|
||||||
|
asked, err := os.Stat(path)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
opened, err := os.Stat(file)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
assert.True(
|
||||||
|
t, os.SameFile(asked, opened),
|
||||||
|
"asked for %s, SQLite opened %s", path, file,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
// TestConcurrentReaderDoesNotBlockWrites is the unit-scale form of the
|
// TestConcurrentReaderDoesNotBlockWrites is the unit-scale form of the
|
||||||
// reproduction in
|
// reproduction in
|
||||||
// https://git.eeqj.de/sneak/webhooker/issues/256: an operator's
|
// https://git.eeqj.de/sneak/webhooker/issues/256: an operator's
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/datadir"
|
"sneak.berlin/go/webhooker/internal/datadir"
|
||||||
|
"sneak.berlin/go/webhooker/internal/db/migrations"
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -370,18 +371,7 @@ func (m *WebhookDBManager) openDB(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = purgeTargetRows(db, m.log, webhookID)
|
err = migrations.Apply(db, migrations.Events)
|
||||||
if err != nil {
|
|
||||||
_ = sqlDB.Close()
|
|
||||||
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run migrations for event-tier models only
|
|
||||||
err = db.AutoMigrate(
|
|
||||||
&Event{}, &Delivery{}, &DeliveryResult{},
|
|
||||||
&EventTotals{}, &TargetTotals{}, &EntrypointTotals{},
|
|
||||||
)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = sqlDB.Close()
|
_ = sqlDB.Close()
|
||||||
|
|
||||||
@@ -391,14 +381,15 @@ func (m *WebhookDBManager) openDB(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// A new database gets its row of event totals, all zero. Target
|
err = parseModels(
|
||||||
// totals rows are created by the first delivery to each target.
|
db, &Event{}, &Delivery{}, &DeliveryResult{},
|
||||||
err = db.FirstOrCreate(&EventTotals{}).Error
|
&EventTotals{}, &TargetTotals{}, &EntrypointTotals{},
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = sqlDB.Close()
|
_ = sqlDB.Close()
|
||||||
|
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"creating event totals for webhook database %s: %w",
|
"connecting to webhook database %s: %w",
|
||||||
webhookID, err,
|
webhookID, err,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,9 @@
|
|||||||
|
-- Migration 000: the table recording which migrations this file has
|
||||||
|
-- had. Applied before the others; it records itself.
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||||
|
version INTEGER PRIMARY KEY,
|
||||||
|
applied_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT OR IGNORE INTO schema_migrations (version) VALUES (0);
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
-- Migration 001: the schema of a database target's archive file. The
|
||||||
|
-- archivedEvent model in internal/delivery must agree with it.
|
||||||
|
|
||||||
|
CREATE TABLE archived_events (
|
||||||
|
id integer PRIMARY KEY AUTOINCREMENT,
|
||||||
|
event_id text,
|
||||||
|
webhook_id text,
|
||||||
|
entrypoint_id text,
|
||||||
|
method text,
|
||||||
|
raw_query text,
|
||||||
|
headers text,
|
||||||
|
body text,
|
||||||
|
content_type text,
|
||||||
|
archived_at datetime
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_archived_events_event_id ON archived_events (event_id);
|
||||||
|
CREATE INDEX idx_archived_events_archived_at ON archived_events (archived_at);
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
-- Migration 000: the table recording which migrations this file has
|
||||||
|
-- had. Applied before the others; it records itself.
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||||
|
version INTEGER PRIMARY KEY,
|
||||||
|
applied_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT OR IGNORE INTO schema_migrations (version) VALUES (0);
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
-- Migration 001: the schema of a webhook's event database. The models
|
||||||
|
-- in internal/database must agree with it. The webhook, entrypoint and
|
||||||
|
-- target ids here name rows in the main database, which has those
|
||||||
|
-- tables; this file does not.
|
||||||
|
|
||||||
|
CREATE TABLE events (
|
||||||
|
id uuid,
|
||||||
|
created_at datetime,
|
||||||
|
updated_at datetime,
|
||||||
|
deleted_at datetime,
|
||||||
|
webhook_id uuid NOT NULL,
|
||||||
|
entrypoint_id uuid NOT NULL,
|
||||||
|
method text NOT NULL,
|
||||||
|
raw_query text,
|
||||||
|
headers text,
|
||||||
|
body text,
|
||||||
|
content_type text,
|
||||||
|
body_bytes integer NOT NULL,
|
||||||
|
resubmitted_from_id uuid,
|
||||||
|
PRIMARY KEY (id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_events_created_at ON events (created_at);
|
||||||
|
CREATE INDEX idx_events_deleted_at ON events (deleted_at);
|
||||||
|
CREATE INDEX idx_events_deleted_at_created_at
|
||||||
|
ON events (deleted_at, created_at);
|
||||||
|
CREATE INDEX idx_events_resubmitted_from_id
|
||||||
|
ON events (resubmitted_from_id, deleted_at);
|
||||||
|
CREATE INDEX idx_events_entrypoint_id
|
||||||
|
ON events (entrypoint_id, deleted_at, resubmitted_from_id, created_at);
|
||||||
|
|
||||||
|
CREATE TABLE deliveries (
|
||||||
|
id uuid,
|
||||||
|
created_at datetime,
|
||||||
|
updated_at datetime,
|
||||||
|
deleted_at datetime,
|
||||||
|
event_id uuid NOT NULL,
|
||||||
|
target_id uuid NOT NULL,
|
||||||
|
status text NOT NULL DEFAULT 'pending',
|
||||||
|
finished_at datetime,
|
||||||
|
replay numeric NOT NULL DEFAULT false,
|
||||||
|
PRIMARY KEY (id),
|
||||||
|
CONSTRAINT fk_events_deliveries
|
||||||
|
FOREIGN KEY (event_id) REFERENCES events (id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_deliveries_deleted_at ON deliveries (deleted_at);
|
||||||
|
CREATE INDEX idx_deliveries_event_id ON deliveries (event_id, deleted_at);
|
||||||
|
CREATE INDEX idx_deliveries_status
|
||||||
|
ON deliveries (status, deleted_at, finished_at, target_id);
|
||||||
|
|
||||||
|
CREATE TABLE delivery_results (
|
||||||
|
id uuid,
|
||||||
|
created_at datetime,
|
||||||
|
updated_at datetime,
|
||||||
|
deleted_at datetime,
|
||||||
|
delivery_id uuid NOT NULL,
|
||||||
|
attempt_num integer NOT NULL,
|
||||||
|
success numeric,
|
||||||
|
status_code integer,
|
||||||
|
response_body text,
|
||||||
|
error text,
|
||||||
|
duration integer,
|
||||||
|
PRIMARY KEY (id),
|
||||||
|
CONSTRAINT fk_deliveries_delivery_results
|
||||||
|
FOREIGN KEY (delivery_id) REFERENCES deliveries (id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_delivery_results_deleted_at
|
||||||
|
ON delivery_results (deleted_at);
|
||||||
|
CREATE INDEX idx_delivery_results_delivery_id
|
||||||
|
ON delivery_results (delivery_id, deleted_at);
|
||||||
|
|
||||||
|
CREATE TABLE event_totals (
|
||||||
|
id integer PRIMARY KEY AUTOINCREMENT,
|
||||||
|
events integer NOT NULL,
|
||||||
|
events_removed integer NOT NULL,
|
||||||
|
last_event_at datetime
|
||||||
|
);
|
||||||
|
|
||||||
|
-- The one row of event totals, all zero.
|
||||||
|
INSERT INTO event_totals (events, events_removed) VALUES (0, 0);
|
||||||
|
|
||||||
|
CREATE TABLE target_totals (
|
||||||
|
target_id uuid,
|
||||||
|
deliveries integer NOT NULL,
|
||||||
|
delivered integer NOT NULL,
|
||||||
|
failed integer NOT NULL,
|
||||||
|
deliveries_removed integer NOT NULL,
|
||||||
|
failed_removed integer NOT NULL,
|
||||||
|
PRIMARY KEY (target_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE entrypoint_totals (
|
||||||
|
entrypoint_id uuid,
|
||||||
|
last_event_at datetime NOT NULL,
|
||||||
|
PRIMARY KEY (entrypoint_id)
|
||||||
|
);
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
-- Migration 000: the table recording which migrations this file has
|
||||||
|
-- had. Applied before the others; it records itself.
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||||
|
version INTEGER PRIMARY KEY,
|
||||||
|
applied_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT OR IGNORE INTO schema_migrations (version) VALUES (0);
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
-- Migration 001: the main database's schema. The models in
|
||||||
|
-- internal/database must agree with it.
|
||||||
|
|
||||||
|
CREATE TABLE settings (
|
||||||
|
"key" text,
|
||||||
|
value text NOT NULL,
|
||||||
|
PRIMARY KEY ("key")
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE users (
|
||||||
|
id uuid,
|
||||||
|
created_at datetime,
|
||||||
|
updated_at datetime,
|
||||||
|
deleted_at datetime,
|
||||||
|
username text NOT NULL,
|
||||||
|
password text NOT NULL,
|
||||||
|
PRIMARY KEY (id),
|
||||||
|
CONSTRAINT chk_users_username
|
||||||
|
CHECK (length(CAST(username AS BLOB)) <= 1024)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_users_deleted_at ON users (deleted_at);
|
||||||
|
CREATE UNIQUE INDEX idx_users_username ON users (username);
|
||||||
|
|
||||||
|
CREATE TABLE api_keys (
|
||||||
|
id uuid,
|
||||||
|
created_at datetime,
|
||||||
|
updated_at datetime,
|
||||||
|
deleted_at datetime,
|
||||||
|
user_id uuid NOT NULL,
|
||||||
|
"key" text NOT NULL,
|
||||||
|
description text,
|
||||||
|
last_used_at datetime,
|
||||||
|
PRIMARY KEY (id),
|
||||||
|
CONSTRAINT fk_users_api_keys FOREIGN KEY (user_id) REFERENCES users (id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_api_keys_deleted_at ON api_keys (deleted_at);
|
||||||
|
CREATE UNIQUE INDEX idx_api_keys_key ON api_keys ("key");
|
||||||
|
|
||||||
|
CREATE TABLE webhooks (
|
||||||
|
id uuid,
|
||||||
|
created_at datetime,
|
||||||
|
updated_at datetime,
|
||||||
|
deleted_at datetime,
|
||||||
|
user_id uuid NOT NULL,
|
||||||
|
name text NOT NULL,
|
||||||
|
description text,
|
||||||
|
retention_days integer DEFAULT 30,
|
||||||
|
PRIMARY KEY (id),
|
||||||
|
CONSTRAINT fk_users_webhooks FOREIGN KEY (user_id) REFERENCES users (id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_webhooks_deleted_at ON webhooks (deleted_at);
|
||||||
|
|
||||||
|
CREATE TABLE entrypoints (
|
||||||
|
id uuid,
|
||||||
|
created_at datetime,
|
||||||
|
updated_at datetime,
|
||||||
|
deleted_at datetime,
|
||||||
|
webhook_id uuid NOT NULL,
|
||||||
|
path text NOT NULL,
|
||||||
|
description text,
|
||||||
|
active numeric DEFAULT true,
|
||||||
|
PRIMARY KEY (id),
|
||||||
|
CONSTRAINT fk_webhooks_entrypoints
|
||||||
|
FOREIGN KEY (webhook_id) REFERENCES webhooks (id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_entrypoints_deleted_at ON entrypoints (deleted_at);
|
||||||
|
CREATE UNIQUE INDEX idx_entrypoints_path ON entrypoints (path);
|
||||||
|
|
||||||
|
CREATE TABLE targets (
|
||||||
|
id uuid,
|
||||||
|
created_at datetime,
|
||||||
|
updated_at datetime,
|
||||||
|
deleted_at datetime,
|
||||||
|
webhook_id uuid NOT NULL,
|
||||||
|
name text NOT NULL,
|
||||||
|
type text NOT NULL,
|
||||||
|
active numeric DEFAULT true,
|
||||||
|
config text,
|
||||||
|
max_retries integer,
|
||||||
|
PRIMARY KEY (id),
|
||||||
|
CONSTRAINT fk_webhooks_targets
|
||||||
|
FOREIGN KEY (webhook_id) REFERENCES webhooks (id)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX idx_targets_deleted_at ON targets (deleted_at);
|
||||||
@@ -0,0 +1,135 @@
|
|||||||
|
// Package migrations holds the schema of each kind of SQLite file
|
||||||
|
// webhooker writes, as numbered SQL files embedded in the binary, and
|
||||||
|
// applies it when a file is opened.
|
||||||
|
//
|
||||||
|
// Each kind of file has its own directory. In each, 000_migration.sql
|
||||||
|
// creates only the schema_migrations table, and 001_schema.sql holds
|
||||||
|
// the whole schema. Before 1.0 a schema change is an edit to
|
||||||
|
// 001_schema.sql, never a new file.
|
||||||
|
package migrations
|
||||||
|
|
||||||
|
import (
|
||||||
|
"embed"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"path"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gorm.io/gorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The directories, one for each kind of database file.
|
||||||
|
const (
|
||||||
|
// Main is the main database, webhooker.db.
|
||||||
|
Main = "main"
|
||||||
|
|
||||||
|
// Events is a webhook's event database, events-{uuid}.db.
|
||||||
|
Events = "events"
|
||||||
|
|
||||||
|
// Archive is a database target's archive file.
|
||||||
|
Archive = "archive"
|
||||||
|
)
|
||||||
|
|
||||||
|
//go:embed main/*.sql events/*.sql archive/*.sql
|
||||||
|
var files embed.FS
|
||||||
|
|
||||||
|
// Apply runs each SQL file in dir, which is Main, Events or Archive,
|
||||||
|
// that the database has not had yet, in the order of the number its
|
||||||
|
// name starts with, and records that number in schema_migrations.
|
||||||
|
// 000_migration.sql creates that table and records itself, so it runs
|
||||||
|
// first, and only when the table is missing.
|
||||||
|
//
|
||||||
|
// It all runs in one transaction, so a failure leaves the file as it
|
||||||
|
// was.
|
||||||
|
func Apply(db *gorm.DB, dir string) error {
|
||||||
|
return db.Transaction(func(tx *gorm.DB) error {
|
||||||
|
var tables int
|
||||||
|
|
||||||
|
// Row().Scan, not (*gorm.DB).Scan: see internal/gormlog.
|
||||||
|
err := tx.Raw(
|
||||||
|
"SELECT count(*) FROM sqlite_master "+
|
||||||
|
"WHERE type = 'table' AND name = ?",
|
||||||
|
"schema_migrations",
|
||||||
|
).Row().Scan(&tables)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("looking for schema_migrations: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if tables == 0 {
|
||||||
|
err = run(tx, path.Join(dir, "000_migration.sql"))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// fs.Glob lists the files in name order, which is number order.
|
||||||
|
names, err := fs.Glob(files, path.Join(dir, "*.sql"))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("listing migrations: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, name := range names {
|
||||||
|
err = applyOnce(tx, name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// applyOnce runs the migration file name and records its number,
|
||||||
|
// unless schema_migrations already has that number.
|
||||||
|
func applyOnce(tx *gorm.DB, name string) error {
|
||||||
|
number, _, _ := strings.Cut(path.Base(name), "_")
|
||||||
|
|
||||||
|
version, err := strconv.Atoi(number)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("numbering migration %s: %w", name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var applied int
|
||||||
|
|
||||||
|
err = tx.Raw(
|
||||||
|
"SELECT count(*) FROM schema_migrations WHERE version = ?",
|
||||||
|
version,
|
||||||
|
).Row().Scan(&applied)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("checking migration %s: %w", name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if applied > 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
err = run(tx, name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = tx.Exec(
|
||||||
|
"INSERT INTO schema_migrations (version) VALUES (?)", version,
|
||||||
|
).Error
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("recording migration %s: %w", name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// run executes every statement in the migration file name.
|
||||||
|
func run(tx *gorm.DB, name string) error {
|
||||||
|
script, err := files.ReadFile(name)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("reading migration %s: %w", name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = tx.Exec(string(script)).Error
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("applying migration %s: %w", name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
package migrations_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"log/slog"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"gorm.io/driver/sqlite"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
_ "modernc.org/sqlite"
|
||||||
|
"sneak.berlin/go/webhooker/internal/db/migrations"
|
||||||
|
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestApplyRunsEachFileOnce applies each kind of database file's
|
||||||
|
// migrations to a new file twice. The first time records every file.
|
||||||
|
// The second finds them all recorded and runs none, which is the only
|
||||||
|
// way it can succeed: 001_schema.sql creates tables that then exist.
|
||||||
|
func TestApplyRunsEachFileOnce(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, dir := range []string{
|
||||||
|
migrations.Main, migrations.Events, migrations.Archive,
|
||||||
|
} {
|
||||||
|
t.Run(dir, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
sqlDB, err := sql.Open(
|
||||||
|
"sqlite", filepath.Join(t.TempDir(), "test.db"),
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||||
|
|
||||||
|
db, err := gorm.Open(
|
||||||
|
sqlite.Dialector{Conn: sqlDB},
|
||||||
|
&gorm.Config{
|
||||||
|
Logger: gormlog.New(slog.New(slog.DiscardHandler)),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
require.NoError(t, migrations.Apply(db, dir))
|
||||||
|
require.NoError(t, migrations.Apply(db, dir))
|
||||||
|
|
||||||
|
var versions string
|
||||||
|
|
||||||
|
require.NoError(t, sqlDB.QueryRowContext(t.Context(),
|
||||||
|
"SELECT group_concat(version) FROM schema_migrations",
|
||||||
|
).Scan(&versions))
|
||||||
|
assert.Equal(t, "0,1", versions)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -201,8 +201,7 @@ func (s *ArchiveSweeper) sweep(ctx context.Context) {
|
|||||||
|
|
||||||
// sweepTarget prunes the archive of a single database target.
|
// sweepTarget prunes the archive of a single database target.
|
||||||
// A missing, empty, or "never" expiry parses as a zero duration
|
// A missing, empty, or "never" expiry parses as a zero duration
|
||||||
// and is skipped entirely, so those archives keep exactly the
|
// and is skipped entirely, so those archives are never pruned.
|
||||||
// behaviour they had before the sweep existed.
|
|
||||||
func (s *ArchiveSweeper) sweepTarget(target *database.Target) {
|
func (s *ArchiveSweeper) sweepTarget(target *database.Target) {
|
||||||
expiry, err := parseArchiveExpiry(target.Config)
|
expiry, err := parseArchiveExpiry(target.Config)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ import (
|
|||||||
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
||||||
|
"sneak.berlin/go/webhooker/internal/db/migrations"
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||||
)
|
)
|
||||||
@@ -172,9 +173,7 @@ func seedArchiveFile(
|
|||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
require.NoError(
|
require.NoError(t, migrations.Apply(gdb, migrations.Archive))
|
||||||
t, gdb.AutoMigrate(&delivery.ExportArchivedEvent{}),
|
|
||||||
)
|
|
||||||
|
|
||||||
for i, at := range archivedAt {
|
for i, at := range archivedAt {
|
||||||
row := delivery.ExportArchivedEvent{
|
row := delivery.ExportArchivedEvent{
|
||||||
@@ -781,9 +780,9 @@ func TestArchiveSweep_NeverExpiryUntouched(t *testing.T) {
|
|||||||
//
|
//
|
||||||
// The spec is stronger than that — a "never" archive is skipped
|
// The spec is stronger than that — a "never" archive is skipped
|
||||||
// before any file is touched — so the archive here exists but has
|
// before any file is touched — so the archive here exists but has
|
||||||
// never been migrated. Opening it at all would run AutoMigrate
|
// never been migrated. Opening it at all would apply the archive
|
||||||
// and create the archive table, which is exactly what must not
|
// schema and create the archive table, which is exactly what must
|
||||||
// happen.
|
// not happen.
|
||||||
func TestArchiveSweep_NeverExpirySkipsBeforeOpening(
|
func TestArchiveSweep_NeverExpirySkipsBeforeOpening(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
@@ -807,7 +806,7 @@ func TestArchiveSweep_NeverExpirySkipsBeforeOpening(
|
|||||||
|
|
||||||
// seedUnmigratedArchive creates an archive file that exists but
|
// seedUnmigratedArchive creates an archive file that exists but
|
||||||
// carries no archive schema, so any open of it is observable: the
|
// carries no archive schema, so any open of it is observable: the
|
||||||
// archive table appears only if something ran AutoMigrate.
|
// archive table appears only if something applied the schema.
|
||||||
func seedUnmigratedArchive(t *testing.T, path string) {
|
func seedUnmigratedArchive(t *testing.T, path string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
|||||||
@@ -32,8 +32,9 @@ func targetTotals(
|
|||||||
|
|
||||||
// TestUpdateDeliveryStatus_FinishTimeAndTargetTotals pins what a status
|
// TestUpdateDeliveryStatus_FinishTimeAndTargetTotals pins what a status
|
||||||
// write records for the webhook page's statistics: the time a delivery
|
// write records for the webhook page's statistics: the time a delivery
|
||||||
// finished, set only when it becomes delivered or failed, and one more
|
// finished, set only when it becomes delivered or failed and stored in
|
||||||
// on its target's delivered or failed total.
|
// the same form as created_at, and one more on its target's delivered
|
||||||
|
// or failed total.
|
||||||
func TestUpdateDeliveryStatus_FinishTimeAndTargetTotals(t *testing.T) {
|
func TestUpdateDeliveryStatus_FinishTimeAndTargetTotals(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -75,6 +76,16 @@ func TestUpdateDeliveryStatus_FinishTimeAndTargetTotals(t *testing.T) {
|
|||||||
if tt.finished {
|
if tt.finished {
|
||||||
require.NotNil(t, stored.FinishedAt)
|
require.NotNil(t, stored.FinishedAt)
|
||||||
assert.False(t, stored.FinishedAt.Before(before))
|
assert.False(t, stored.FinishedAt.Before(before))
|
||||||
|
|
||||||
|
var createdAt, finishedAt string
|
||||||
|
|
||||||
|
require.NoError(t, db.Raw(
|
||||||
|
`SELECT CAST(created_at AS TEXT),
|
||||||
|
CAST(finished_at AS TEXT)
|
||||||
|
FROM deliveries WHERE id = ?`, d.ID,
|
||||||
|
).Row().Scan(&createdAt, &finishedAt))
|
||||||
|
assertStoredTime(t, createdAt)
|
||||||
|
assertStoredTime(t, finishedAt)
|
||||||
} else {
|
} else {
|
||||||
assert.Nil(t, stored.FinishedAt)
|
assert.Nil(t, stored.FinishedAt)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1701,8 +1701,11 @@ func writeDeliveryStatus(
|
|||||||
database.DeliveryStatusFailed,
|
database.DeliveryStatusFailed,
|
||||||
}).
|
}).
|
||||||
Updates(map[string]any{
|
Updates(map[string]any{
|
||||||
"status": status,
|
"status": status,
|
||||||
"finished_at": time.Now(),
|
// GORM's clock, the one that stamps created_at. A bare
|
||||||
|
// time.Now() carries a monotonic clock reading, which the
|
||||||
|
// driver would store as an " m=+…" suffix.
|
||||||
|
"finished_at": tx.NowFunc(),
|
||||||
})
|
})
|
||||||
if res.Error != nil || res.RowsAffected == 0 {
|
if res.Error != nil || res.RowsAffected == 0 {
|
||||||
return res.Error
|
return res.Error
|
||||||
@@ -1906,12 +1909,13 @@ func (e *Engine) takeForRedispatch(
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GORM's clock, as writeDeliveryStatus explains.
|
||||||
res := webhookDB.
|
res := webhookDB.
|
||||||
Model(&database.Delivery{}).
|
Model(&database.Delivery{}).
|
||||||
Where(
|
Where(
|
||||||
"id = ? AND status = ?", deliveryID, eligible,
|
"id = ? AND status = ?", deliveryID, eligible,
|
||||||
).
|
).
|
||||||
UpdateColumn("updated_at", time.Now())
|
UpdateColumn("updated_at", webhookDB.NowFunc())
|
||||||
|
|
||||||
if res.Error != nil {
|
if res.Error != nil {
|
||||||
e.log.Error(
|
e.log.Error(
|
||||||
|
|||||||
@@ -963,29 +963,12 @@ func TestFailUnretryableRetry_WritesNoTargetRow(
|
|||||||
database.DeliveryStatusFailed,
|
database.DeliveryStatusFailed,
|
||||||
)
|
)
|
||||||
|
|
||||||
// The table exists in the per-webhook database because GORM
|
// The per-webhook database has no targets table, so writing a
|
||||||
// migrates the Delivery relation's model alongside it. It must
|
// target row there would have failed the status write above.
|
||||||
// stay empty.
|
assert.False(t, s.WebhookDB.Migrator().HasTable("targets"),
|
||||||
var targetRows int64
|
|
||||||
|
|
||||||
require.NoError(t, s.WebhookDB.
|
|
||||||
Table("targets").
|
|
||||||
Count(&targetRows).Error)
|
|
||||||
|
|
||||||
assert.Zero(t, targetRows,
|
|
||||||
"orphaned-retry terminal failure wrote a target row "+
|
"orphaned-retry terminal failure wrote a target row "+
|
||||||
"into the per-webhook event database",
|
"into the per-webhook event database",
|
||||||
)
|
)
|
||||||
|
|
||||||
var configs []string
|
|
||||||
|
|
||||||
require.NoError(t, s.WebhookDB.
|
|
||||||
Table("targets").
|
|
||||||
Pluck("config", &configs).Error)
|
|
||||||
|
|
||||||
assert.NotContains(
|
|
||||||
t, strings.Join(configs, " "), hookURL,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRecoverSingleRetry_UnknownTargetType(
|
func TestRecoverSingleRetry_UnknownTargetType(
|
||||||
|
|||||||
@@ -2,11 +2,16 @@ package delivery_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"gorm.io/gorm/clause"
|
"gorm.io/gorm/clause"
|
||||||
@@ -272,6 +277,154 @@ func TestEngine_StopHookHonoursStopTimeout(t *testing.T) {
|
|||||||
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// seedHeldDelivery gives the setup's webhook an http target with
|
||||||
|
// maxRetries, pointed at a receiver that keeps every request open until
|
||||||
|
// the client gives up on it, and seeds one delivery to that target at
|
||||||
|
// status. It returns the task that sends attempt attemptNum of the
|
||||||
|
// delivery, and a channel that is closed when a request reaches the
|
||||||
|
// receiver.
|
||||||
|
func seedHeldDelivery(
|
||||||
|
t *testing.T,
|
||||||
|
s iSetup,
|
||||||
|
maxRetries int,
|
||||||
|
status database.DeliveryStatus,
|
||||||
|
attemptNum int,
|
||||||
|
) (delivery.Task, <-chan struct{}) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
arrived := make(chan struct{})
|
||||||
|
|
||||||
|
var once sync.Once
|
||||||
|
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(
|
||||||
|
func(_ http.ResponseWriter, r *http.Request) {
|
||||||
|
// The server notices the client hanging up only once the
|
||||||
|
// body has been read; until then r.Context() never ends.
|
||||||
|
_, _ = io.Copy(io.Discard, r.Body)
|
||||||
|
|
||||||
|
once.Do(func() { close(arrived) })
|
||||||
|
|
||||||
|
<-r.Context().Done()
|
||||||
|
},
|
||||||
|
))
|
||||||
|
t.Cleanup(ts.Close)
|
||||||
|
|
||||||
|
cfg := iHTTPConfig(ts.URL)
|
||||||
|
targetID := uuid.New().String()
|
||||||
|
|
||||||
|
iCreateWebhook(t, s.MainDB, s.WebhookID, "hook")
|
||||||
|
iCreateTarget(
|
||||||
|
t, s.MainDB, targetID, s.WebhookID, "held",
|
||||||
|
database.TargetTypeHTTP, cfg, maxRetries,
|
||||||
|
)
|
||||||
|
|
||||||
|
event := iSeedEvent(t, s.WebhookDB, s.WebhookID, `{"held":true}`)
|
||||||
|
d := iSeedDelivery(t, s.WebhookDB, event.ID, targetID, status)
|
||||||
|
|
||||||
|
bodyStr := event.Body
|
||||||
|
|
||||||
|
return iTask(
|
||||||
|
d, event, s.WebhookID, targetID, "held", cfg,
|
||||||
|
maxRetries, attemptNum, &bodyStr,
|
||||||
|
), arrived
|
||||||
|
}
|
||||||
|
|
||||||
|
// stopDuringAttempt waits until the receiver holds an attempt open, then
|
||||||
|
// stops the engine through its real stop hook.
|
||||||
|
func stopDuringAttempt(
|
||||||
|
t *testing.T, lc *recordingLifecycle, arrived <-chan struct{},
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-arrived:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("the delivery attempt never reached the receiver")
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, lc.hooks[0].OnStop(context.Background()))
|
||||||
|
}
|
||||||
|
|
||||||
|
// requireRequeued runs restart recovery on the stopped engine and
|
||||||
|
// requires it to queue task's attempt again on ch.
|
||||||
|
func requireRequeued(
|
||||||
|
t *testing.T, s iSetup, ch <-chan delivery.Task, task delivery.Task,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
s.Engine.ExportRecoverWebhookDeliveries(
|
||||||
|
context.Background(), s.WebhookID,
|
||||||
|
)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case got := <-ch:
|
||||||
|
assert.Equal(t, task.DeliveryID, got.DeliveryID)
|
||||||
|
assert.Equal(t, task.AttemptNum, got.AttemptNum)
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("restart recovery did not queue the delivery again")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestEngine_StopLeavesFireAndForgetAttemptPending is the regression
|
||||||
|
// test for a stop that turned an attempt in flight into a terminal
|
||||||
|
// failure. Stopping the engine cancels the request; for a target with
|
||||||
|
// no retries that was the delivery's only attempt, recorded as failed
|
||||||
|
// with "context canceled", and nothing ever sent it again. The
|
||||||
|
// delivery must instead stay pending, with no result and no count in
|
||||||
|
// the attempt metrics, for restart recovery to send.
|
||||||
|
func TestEngine_StopLeavesFireAndForgetAttemptPending(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
s := newISetup(t)
|
||||||
|
reg := mIsolate(t, s)
|
||||||
|
lc := startEngineViaHook(t, s.Engine)
|
||||||
|
|
||||||
|
task, arrived := seedHeldDelivery(
|
||||||
|
t, s, 0, database.DeliveryStatusPending, 1,
|
||||||
|
)
|
||||||
|
|
||||||
|
s.Engine.Notify([]delivery.Task{task})
|
||||||
|
stopDuringAttempt(t, lc, arrived)
|
||||||
|
|
||||||
|
iAssertStatus(
|
||||||
|
t, s.WebhookDB, task.DeliveryID, database.DeliveryStatusPending,
|
||||||
|
)
|
||||||
|
assert.Empty(t, iResults(t, s.WebhookDB, task.DeliveryID))
|
||||||
|
assert.Zero(t, mCounter(t, reg, mAttempts, mTypeHTTP))
|
||||||
|
assert.Zero(t, mHTTPDurations(t, reg))
|
||||||
|
|
||||||
|
requireRequeued(t, s, s.Engine.ExportDeliveryCh(), task)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestEngine_StopLeavesLastRetryAttemptRetrying is the same regression
|
||||||
|
// for a target with retries, on its last attempt: the cancelled attempt
|
||||||
|
// used to exhaust the retries and end the delivery failed. It must stay
|
||||||
|
// retrying with only its earlier attempt recorded, and restart recovery
|
||||||
|
// must send that last attempt again.
|
||||||
|
func TestEngine_StopLeavesLastRetryAttemptRetrying(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const maxRetries = 2
|
||||||
|
|
||||||
|
s := newISetup(t)
|
||||||
|
lc := startEngineViaHook(t, s.Engine)
|
||||||
|
|
||||||
|
task, arrived := seedHeldDelivery(
|
||||||
|
t, s, maxRetries, database.DeliveryStatusRetrying, maxRetries,
|
||||||
|
)
|
||||||
|
iSeedFailedResult(t, s.WebhookDB, task.DeliveryID)
|
||||||
|
|
||||||
|
s.Engine.ExportScheduleRetry(task, 0)
|
||||||
|
stopDuringAttempt(t, lc, arrived)
|
||||||
|
|
||||||
|
iAssertStatus(
|
||||||
|
t, s.WebhookDB, task.DeliveryID, database.DeliveryStatusRetrying,
|
||||||
|
)
|
||||||
|
assert.Len(t, iResults(t, s.WebhookDB, task.DeliveryID), 1)
|
||||||
|
|
||||||
|
requireRequeued(t, s, s.Engine.ExportRetryCh(), task)
|
||||||
|
}
|
||||||
|
|
||||||
// deliverToArchive gives the setup's webhook a database target,
|
// deliverToArchive gives the setup's webhook a database target,
|
||||||
// runs one delivery to it through the running engine, and returns
|
// runs one delivery to it through the running engine, and returns
|
||||||
// the target's ID and archive file path. The archive writer holds
|
// the target's ID and archive file path. The archive writer holds
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -22,12 +21,11 @@ import (
|
|||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"gorm.io/driver/sqlite"
|
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
_ "modernc.org/sqlite"
|
_ "modernc.org/sqlite"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
|
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
|
||||||
"sneak.berlin/go/webhooker/internal/metrics"
|
"sneak.berlin/go/webhooker/internal/metrics"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -37,35 +35,18 @@ const testContentType = "application/json"
|
|||||||
func testWebhookDB(t *testing.T) *gorm.DB {
|
func testWebhookDB(t *testing.T) *gorm.DB {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dbPath := filepath.Join(
|
// Opened by the manager the service uses, so these tests cannot
|
||||||
t.TempDir(), "events-test.db",
|
// pass against a schema, journal and locking settings or write
|
||||||
|
// guard production does not use.
|
||||||
|
mgr := databasetest.NewWebhookDBManagerWithLogger(
|
||||||
|
t, t.TempDir(), slog.New(slog.DiscardHandler),
|
||||||
)
|
)
|
||||||
|
|
||||||
// Opened the way the service opens a per-webhook database, so
|
t.Cleanup(func() { _ = mgr.CloseAll() })
|
||||||
// these tests cannot pass against journal and locking settings
|
|
||||||
// production does not use.
|
db, err := mgr.GetDB(uuid.New().String())
|
||||||
sqlDB, err := database.OpenSQLite(
|
|
||||||
dbPath, database.SQLiteModeCreate,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
|
||||||
|
|
||||||
db, err := gorm.Open(
|
|
||||||
sqlite.Dialector{Conn: sqlDB},
|
|
||||||
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
require.NoError(t, db.AutoMigrate(
|
|
||||||
&database.Event{},
|
|
||||||
&database.Delivery{},
|
|
||||||
&database.DeliveryResult{},
|
|
||||||
&database.EventTotals{},
|
|
||||||
&database.TargetTotals{},
|
|
||||||
))
|
|
||||||
require.NoError(t, db.Create(&database.EventTotals{}).Error)
|
|
||||||
|
|
||||||
return db
|
return db
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -136,6 +117,18 @@ func seedDelivery(
|
|||||||
return d
|
return d
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// assertStoredTime asserts that text, a timestamp as SQLite holds it,
|
||||||
|
// is in the form GORM stores created_at in: the layout of
|
||||||
|
// time.Time.String, with no " m=+…" monotonic clock reading after it.
|
||||||
|
func assertStoredTime(t *testing.T, text string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
_, err := time.Parse(
|
||||||
|
"2006-01-02 15:04:05.999999999 -0700 MST", text,
|
||||||
|
)
|
||||||
|
assert.NoError(t, err, "stored as %q", text)
|
||||||
|
}
|
||||||
|
|
||||||
// httpDeliveryFixture holds the task and delivery needed
|
// httpDeliveryFixture holds the task and delivery needed
|
||||||
// for HTTP delivery tests.
|
// for HTTP delivery tests.
|
||||||
type httpDeliveryFixture struct {
|
type httpDeliveryFixture struct {
|
||||||
|
|||||||
@@ -462,7 +462,8 @@ func TestScheduledRetryIsNotSweptDuringBackoff(t *testing.T) {
|
|||||||
|
|
||||||
// TestRedispatchStampsTheRow pins the cadence control: a stranded
|
// TestRedispatchStampsTheRow pins the cadence control: a stranded
|
||||||
// delivery that has just been handed out is not selected again by the
|
// delivery that has just been handed out is not selected again by the
|
||||||
// next tick a minute later.
|
// next tick a minute later. The stamp is stored in the same form as
|
||||||
|
// created_at.
|
||||||
func TestRedispatchStampsTheRow(t *testing.T) {
|
func TestRedispatchStampsTheRow(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -493,4 +494,13 @@ func TestRedispatchStampsTheRow(t *testing.T) {
|
|||||||
"a re-dispatched delivery must be stamped so the "+
|
"a re-dispatched delivery must be stamped so the "+
|
||||||
"next tick does not select it again",
|
"next tick does not select it again",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var createdAt, updatedAt string
|
||||||
|
|
||||||
|
require.NoError(t, s.WebhookDB.Raw(
|
||||||
|
`SELECT CAST(created_at AS TEXT), CAST(updated_at AS TEXT)
|
||||||
|
FROM deliveries WHERE id = ?`, d.ID,
|
||||||
|
).Row().Scan(&createdAt, &updatedAt))
|
||||||
|
assertStoredTime(t, createdAt)
|
||||||
|
assertStoredTime(t, updatedAt)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,7 +32,8 @@ type Scheduler interface {
|
|||||||
// Engine.observeAttempt, alongside the DeliveryResult it records for
|
// Engine.observeAttempt, alongside the DeliveryResult it records for
|
||||||
// it. Deliver is also entered for attempts that never happen — an
|
// it. Deliver is also entered for attempts that never happen — an
|
||||||
// open circuit breaker refuses one — so the count cannot be taken
|
// open circuit breaker refuses one — so the count cannot be taken
|
||||||
// from around this call.
|
// from around this call. An attempt cut off by a stop is neither
|
||||||
|
// counted nor recorded.
|
||||||
type Target interface {
|
type Target interface {
|
||||||
Deliver(
|
Deliver(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
"gorm.io/driver/sqlite"
|
"gorm.io/driver/sqlite"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
|
"sneak.berlin/go/webhooker/internal/db/migrations"
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -276,7 +277,11 @@ func (w *archiveWriter) write(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
row.ArchivedAt = time.Now()
|
// GORM's clock, the one that stamps created_at in the other
|
||||||
|
// databases, so this is stored in the same form. A bare time.Now()
|
||||||
|
// carries a monotonic clock reading, which the driver would store
|
||||||
|
// as an " m=+…" suffix.
|
||||||
|
row.ArchivedAt = w.db.NowFunc()
|
||||||
|
|
||||||
err := w.db.Create(&row).Error
|
err := w.db.Create(&row).Error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -335,7 +340,7 @@ func (w *archiveWriter) openMode(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = gdb.AutoMigrate(&archivedEvent{})
|
err = migrations.Apply(gdb, migrations.Archive)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = sqlDB.Close()
|
_ = sqlDB.Close()
|
||||||
|
|
||||||
|
|||||||
@@ -150,6 +150,13 @@ func TestArchiveWriter_WritesRow(t *testing.T) {
|
|||||||
assert.Equal(t, row.ContentType, got[0].ContentType)
|
assert.Equal(t, row.ContentType, got[0].ContentType)
|
||||||
assert.JSONEq(t, `{"hello":"world"}`, got[0].Body)
|
assert.JSONEq(t, `{"hello":"world"}`, got[0].Body)
|
||||||
assert.False(t, got[0].ArchivedAt.IsZero())
|
assert.False(t, got[0].ArchivedAt.IsZero())
|
||||||
|
|
||||||
|
var archivedAt string
|
||||||
|
|
||||||
|
require.NoError(t, w.DB().Raw(
|
||||||
|
"SELECT CAST(archived_at AS TEXT) FROM archived_events",
|
||||||
|
).Row().Scan(&archivedAt))
|
||||||
|
assertStoredTime(t, archivedAt)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestArchiveWriter_RecreatesAfterRemoval(
|
func TestArchiveWriter_RecreatesAfterRemoval(
|
||||||
|
|||||||
@@ -82,8 +82,8 @@ func isReservedTargetHeader(name string) bool {
|
|||||||
// one pair silently overwriting the other.
|
// one pair silently overwriting the other.
|
||||||
//
|
//
|
||||||
// An input with no pairs yields an empty map, which omitempty drops
|
// An input with no pairs yields an empty map, which omitempty drops
|
||||||
// from the stored config: a target configured with no headers keeps
|
// from the stored config, so a target configured with no headers
|
||||||
// the same config JSON it had before this field existed.
|
// stores no headers key at all.
|
||||||
func ParseTargetHeaders(raw string) (map[string]string, error) {
|
func ParseTargetHeaders(raw string) (map[string]string, error) {
|
||||||
headers := make(map[string]string)
|
headers := make(map[string]string)
|
||||||
|
|
||||||
|
|||||||
@@ -57,7 +57,11 @@ type httpCore struct {
|
|||||||
// recorded and no circuit breaker is consulted. A positive
|
// recorded and no circuit breaker is consulted. A positive
|
||||||
// maxRetries gates the attempt on the circuit breaker and
|
// maxRetries gates the attempt on the circuit breaker and
|
||||||
// schedules a backed-off retry on failure.
|
// schedules a backed-off retry on failure.
|
||||||
|
//
|
||||||
|
// ctx is the context attempt sends its request with. Stopping the
|
||||||
|
// engine cancels it; see interruptedByStop.
|
||||||
func (c *httpCore) deliver(
|
func (c *httpCore) deliver(
|
||||||
|
ctx context.Context,
|
||||||
webhookDB *gorm.DB,
|
webhookDB *gorm.DB,
|
||||||
d *database.Delivery,
|
d *database.Delivery,
|
||||||
task *Task,
|
task *Task,
|
||||||
@@ -66,21 +70,55 @@ func (c *httpCore) deliver(
|
|||||||
attempt func() attemptResult,
|
attempt func() attemptResult,
|
||||||
) {
|
) {
|
||||||
if maxRetries == 0 {
|
if maxRetries == 0 {
|
||||||
c.fireAndForget(webhookDB, d, attempt())
|
c.fireAndForget(ctx, webhookDB, d, attempt())
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
c.withRetry(
|
c.withRetry(
|
||||||
webhookDB, d, task, sched, maxRetries, attempt,
|
ctx, webhookDB, d, task, sched, maxRetries, attempt,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// interruptedByStop reports whether an attempt failed because the
|
||||||
|
// engine is stopping, which cancels the context its request is made
|
||||||
|
// with. Such an attempt is not recorded: no DeliveryResult, no circuit
|
||||||
|
// breaker outcome, no status change, no count in the attempt metrics.
|
||||||
|
// The delivery stays pending or retrying, and startup recovery sends
|
||||||
|
// it again on the next start.
|
||||||
|
//
|
||||||
|
// A target's own timeout leaves that context alone, so its attempt is
|
||||||
|
// still recorded as the failure it is.
|
||||||
|
func (c *httpCore) interruptedByStop(
|
||||||
|
ctx context.Context,
|
||||||
|
d *database.Delivery,
|
||||||
|
res attemptResult,
|
||||||
|
) bool {
|
||||||
|
if res.success || ctx.Err() == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
c.eng.log.Warn(
|
||||||
|
"delivery attempt interrupted by shutdown; not recorded, "+
|
||||||
|
"it is sent again on the next start",
|
||||||
|
"delivery_id", d.ID,
|
||||||
|
"event_id", d.EventID,
|
||||||
|
"target_id", d.TargetID,
|
||||||
|
)
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
func (c *httpCore) fireAndForget(
|
func (c *httpCore) fireAndForget(
|
||||||
|
ctx context.Context,
|
||||||
webhookDB *gorm.DB,
|
webhookDB *gorm.DB,
|
||||||
d *database.Delivery,
|
d *database.Delivery,
|
||||||
res attemptResult,
|
res attemptResult,
|
||||||
) {
|
) {
|
||||||
|
if c.interruptedByStop(ctx, d, res) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
c.eng.observeAttempt(d.Target.Type, res.elapsed())
|
c.eng.observeAttempt(d.Target.Type, res.elapsed())
|
||||||
|
|
||||||
err := c.eng.recordResult(
|
err := c.eng.recordResult(
|
||||||
@@ -110,6 +148,7 @@ func (c *httpCore) fireAndForget(
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *httpCore) withRetry(
|
func (c *httpCore) withRetry(
|
||||||
|
ctx context.Context,
|
||||||
webhookDB *gorm.DB,
|
webhookDB *gorm.DB,
|
||||||
d *database.Delivery,
|
d *database.Delivery,
|
||||||
task *Task,
|
task *Task,
|
||||||
@@ -131,6 +170,10 @@ func (c *httpCore) withRetry(
|
|||||||
|
|
||||||
res := attempt()
|
res := attempt()
|
||||||
|
|
||||||
|
if c.interruptedByStop(ctx, d, res) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
c.eng.observeAttempt(d.Target.Type, res.elapsed())
|
c.eng.observeAttempt(d.Target.Type, res.elapsed())
|
||||||
|
|
||||||
err := c.eng.recordResult(
|
err := c.eng.recordResult(
|
||||||
@@ -395,7 +438,7 @@ func (t *httpTarget) Deliver(
|
|||||||
}
|
}
|
||||||
|
|
||||||
t.deliver(
|
t.deliver(
|
||||||
webhookDB, d, task, sched,
|
ctx, webhookDB, d, task, sched,
|
||||||
d.Target.MaxRetries, attempt,
|
d.Target.MaxRetries, attempt,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,9 +13,9 @@ import (
|
|||||||
const RedactionMarker = "(redacted)"
|
const RedactionMarker = "(redacted)"
|
||||||
|
|
||||||
// Redactor removes one target's own credential material from
|
// Redactor removes one target's own credential material from
|
||||||
// text that target's remote peer chose: a delivery response
|
// a delivery response body, whose text the target's remote
|
||||||
// body, or a delivery error stored before the delivery path
|
// peer chose, and from a stored delivery error as a second
|
||||||
// learned to mask the URLs it embeds.
|
// line behind the delivery path's own URL masking.
|
||||||
//
|
//
|
||||||
// It removes byte-identical echoes of strings taken from the
|
// It removes byte-identical echoes of strings taken from the
|
||||||
// target's stored configuration, and nothing else. Anything
|
// target's stored configuration, and nothing else. Anything
|
||||||
|
|||||||
@@ -82,7 +82,7 @@ func (t *slackTarget) Deliver(
|
|||||||
}
|
}
|
||||||
|
|
||||||
t.deliver(
|
t.deliver(
|
||||||
webhookDB, d, task, sched,
|
ctx, webhookDB, d, task, sched,
|
||||||
d.Target.MaxRetries, attempt,
|
d.Target.MaxRetries, attempt,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -312,13 +312,9 @@ func TestFailMissingTarget_WritesNoTargetRow(
|
|||||||
database.DeliveryStatusFailed,
|
database.DeliveryStatusFailed,
|
||||||
)
|
)
|
||||||
|
|
||||||
var configs []string
|
// The per-webhook database has no targets table, so writing a
|
||||||
|
// target row there would have failed the status write above.
|
||||||
require.NoError(t, s.WebhookDB.
|
assert.False(t, s.WebhookDB.Migrator().HasTable("targets"),
|
||||||
Table("targets").
|
|
||||||
Pluck("config", &configs).Error)
|
|
||||||
|
|
||||||
assert.Empty(t, configs,
|
|
||||||
"the deleted-target terminal path wrote a target row "+
|
"the deleted-target terminal path wrote a target row "+
|
||||||
"into the per-webhook event database",
|
"into the per-webhook event database",
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -135,9 +135,11 @@ func openDB(
|
|||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
require.NoError(t, gdb.AutoMigrate(&thing{}))
|
require.NoError(t, gdb.Exec(
|
||||||
|
"CREATE TABLE things (id text PRIMARY KEY, name text)",
|
||||||
|
).Error)
|
||||||
|
|
||||||
// Migration chatter is not what any of these cases is about.
|
// Creating the table is not what any of these cases is about.
|
||||||
buf.Reset()
|
buf.Reset()
|
||||||
|
|
||||||
return gdb
|
return gdb
|
||||||
|
|||||||
@@ -115,8 +115,8 @@ func skipDir(name string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// walkNonTestGo parses every non-test .go file under root. It returns
|
// walkNonTestGo parses every non-test .go file under root. It returns
|
||||||
// the directories, relative to root, it parsed a file in, along with
|
// the directories, relative to root, it parsed a file in or below,
|
||||||
// every unguarded Scan it found.
|
// along with every unguarded Scan it found.
|
||||||
func walkNonTestGo(t *testing.T, root string) (map[string]bool, []string) {
|
func walkNonTestGo(t *testing.T, root string) (map[string]bool, []string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -155,7 +155,12 @@ func walkNonTestGo(t *testing.T, root string) (map[string]bool, []string) {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
walked[dir] = true
|
// The directories above count too, so that one holding
|
||||||
|
// only a nested package, as internal/db does, is not
|
||||||
|
// taken for one the walk skipped.
|
||||||
|
for ; dir != "."; dir = filepath.Dir(dir) {
|
||||||
|
walked[dir] = true
|
||||||
|
}
|
||||||
|
|
||||||
for _, pos := range unguardedScans(fset, file) {
|
for _, pos := range unguardedScans(fset, file) {
|
||||||
hits = append(hits, relPosition(root, pos))
|
hits = append(hits, relPosition(root, pos))
|
||||||
@@ -205,9 +210,9 @@ func TestGormScanIsNeverCalledOutsideTests(t *testing.T) {
|
|||||||
walked, offenders := walkNonTestGo(t, root)
|
walked, offenders := walkNonTestGo(t, root)
|
||||||
|
|
||||||
// The module's packages are static, templates, and every directory
|
// The module's packages are static, templates, and every directory
|
||||||
// directly under cmd and internal. Each holds non-test code, so one
|
// directly under cmd and internal. Each holds non-test code, itself
|
||||||
// the walk parsed nothing in was skipped, and a Scan there would
|
// or in a package below it, so one the walk parsed nothing in or
|
||||||
// pass unseen.
|
// below was skipped, and a Scan there would pass unseen.
|
||||||
packages := []string{"static", "templates"}
|
packages := []string{"static", "templates"}
|
||||||
|
|
||||||
for _, parent := range []string{"cmd", "internal"} {
|
for _, parent := range []string{"cmd", "internal"} {
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ func TestDeliveryAttempts_ReadInTheTargetTypesOwnTerms(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
t.Run(string(tc.targetType), func(t *testing.T) {
|
t.Run(string(tc.targetType)+" "+tc.outcome, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
f := newRecentEventsFixture(t)
|
||||||
|
|||||||
@@ -314,21 +314,15 @@ func assertReplayTask(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// assertNoLeakedTarget proves the per-webhook database holds no target
|
// assertNoLeakedTarget proves the per-webhook database holds no
|
||||||
// rows. AutoMigrate creates the table there because Delivery declares
|
// target: an association write would have upserted the whole target,
|
||||||
// the relation, so it is a ROW that signals a leak: an association
|
// plaintext config and all, into the event database. See
|
||||||
// write would have upserted the whole target, plaintext config and
|
|
||||||
// all, into the event database. See
|
|
||||||
// https://git.eeqj.de/sneak/webhooker/issues/206.
|
// https://git.eeqj.de/sneak/webhooker/issues/206.
|
||||||
func assertNoLeakedTarget(t *testing.T, webhookDB *gorm.DB) {
|
func assertNoLeakedTarget(t *testing.T, webhookDB *gorm.DB) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
var leaked int64
|
assert.False(
|
||||||
|
t, webhookDB.Migrator().HasTable(&database.Target{}),
|
||||||
require.NoError(t, webhookDB.Unscoped().
|
|
||||||
Model(&database.Target{}).Count(&leaked).Error)
|
|
||||||
assert.Zero(
|
|
||||||
t, leaked,
|
|
||||||
"replay must not write the target into the event database",
|
"replay must not write the target into the event database",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,13 +12,12 @@ import (
|
|||||||
//
|
//
|
||||||
// The bound is enforced here and in SQL, because this page's
|
// The bound is enforced here and in SQL, because this page's
|
||||||
// memory profile must not depend on a constant in another
|
// memory profile must not depend on a constant in another
|
||||||
// package staying where it is, and because rows predating the
|
// package staying where it is, or on every stored row having
|
||||||
// delivery engine's own cap or restored from an archive are
|
// been written by the delivery engine under its own cap.
|
||||||
// not covered by it at all.
|
|
||||||
//
|
//
|
||||||
// It happens to equal that engine cap, so a row written by the
|
// It happens to equal that engine cap, so a row written by the
|
||||||
// current engine reaches this one exactly and is never cut
|
// engine reaches this one exactly and is never cut twice.
|
||||||
// twice. Nothing here may assume the two differ: see view.
|
// Nothing here may assume the two differ: see view.
|
||||||
const maxRenderedResponseBytes = 4096
|
const maxRenderedResponseBytes = 4096
|
||||||
|
|
||||||
// deliveryResultColumns is the delivery attempt projection.
|
// deliveryResultColumns is the delivery attempt projection.
|
||||||
@@ -40,9 +39,8 @@ const deliveryResultColumns = "delivery_id, attempt_num, created_at, " +
|
|||||||
//
|
//
|
||||||
// Both Error and ResponseBody have been through the target's
|
// Both Error and ResponseBody have been through the target's
|
||||||
// Redactor. The engine already masks the URL out of the
|
// Redactor. The engine already masks the URL out of the
|
||||||
// errors it stores, so for errors this is a second line
|
// errors it stores, so for errors this is a second line; for
|
||||||
// covering rows written before it did; for response bodies it
|
// response bodies it is the only line, and its reach is what
|
||||||
// is the only line, and its reach is what
|
|
||||||
// delivery.Redactor documents.
|
// delivery.Redactor documents.
|
||||||
type DeliveryResultView struct {
|
type DeliveryResultView struct {
|
||||||
AttemptNum int
|
AttemptNum int
|
||||||
|
|||||||
@@ -158,8 +158,7 @@ func TestHandleSourceLogs_RendersFailedAttempt(t *testing.T) {
|
|||||||
// TestHandleSourceLogs_EscapesResponseBody proves the
|
// TestHandleSourceLogs_EscapesResponseBody proves the
|
||||||
// response body is treated as the untrusted remote content it
|
// response body is treated as the untrusted remote content it
|
||||||
// is. The remote chooses these bytes and the page is rendered
|
// is. The remote chooses these bytes and the page is rendered
|
||||||
// inside the operator's authenticated origin, where the
|
// inside the operator's authenticated origin.
|
||||||
// application's own CSP allows inline script from 'self'.
|
|
||||||
func TestHandleSourceLogs_EscapesResponseBody(t *testing.T) {
|
func TestHandleSourceLogs_EscapesResponseBody(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -35,11 +35,10 @@ const eventBodyQuery = "SELECT cast(body as blob) " +
|
|||||||
// response is deliberately not renderable. Content-Disposition
|
// response is deliberately not renderable. Content-Disposition
|
||||||
// makes the browser download rather than display it, and the
|
// makes the browser download rather than display it, and the
|
||||||
// octet-stream type plus nosniff stop it being interpreted as
|
// octet-stream type plus nosniff stop it being interpreted as
|
||||||
// HTML or script. Without those a stored payload would execute
|
// HTML or script. Without those a stored payload would be shown
|
||||||
// as the logged-in operator. The application's CSP does not
|
// as a page of the operator's own origin. The application's CSP
|
||||||
// help here: script-src allows 'unsafe-inline' from 'self', so
|
// is a second defence behind them: it allows no inline script,
|
||||||
// a document served from this origin could run its own inline
|
// so such a page could not run script of its own.
|
||||||
// script.
|
|
||||||
func (h *Handlers) HandleEventBodyDownload() http.HandlerFunc {
|
func (h *Handlers) HandleEventBodyDownload() http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
webhook, ok := h.ownedWebhook(w, r)
|
webhook, ok := h.ownedWebhook(w, r)
|
||||||
|
|||||||
@@ -84,7 +84,7 @@ func TestHandleEventDetail_ShowsEventWholeWithDeliveries(t *testing.T) {
|
|||||||
assert.Contains(t, page, strconv.Itoa(len(body))+" bytes")
|
assert.Contains(t, page, strconv.Itoa(len(body))+" bytes")
|
||||||
assert.Contains(t, page, "{\n "pad": "xxx")
|
assert.Contains(t, page, "{\n "pad": "xxx")
|
||||||
assert.Contains(t, page, ""tail": ""+sentinel+""\n}")
|
assert.Contains(t, page, ""tail": ""+sentinel+""\n}")
|
||||||
assert.Contains(t, page, `style="max-height: 32rem; overflow-y: auto"`)
|
assert.Contains(t, page, `break-all max-h-[32rem] overflow-y-auto"`)
|
||||||
assert.NotContains(t, page, "Showing the first")
|
assert.NotContains(t, page, "Showing the first")
|
||||||
assert.Contains(t, page, target.Name)
|
assert.Contains(t, page, target.Name)
|
||||||
assert.Contains(t, page, ">failed</span>")
|
assert.Contains(t, page, ">failed</span>")
|
||||||
|
|||||||
@@ -1,19 +1,15 @@
|
|||||||
package handlers_test
|
package handlers_test
|
||||||
|
|
||||||
// The handler-side half of the log-field audit. Two slog calls in
|
// The handler-side half of the log-field audit. The slog calls in
|
||||||
// this package reach a value an UNAUTHENTICATED client picks outright
|
// this package that reach a value an UNAUTHENTICATED client picks
|
||||||
// and of a length it picks outright:
|
// outright, and of a length it picks outright, are the failed-login
|
||||||
|
// DEBUG lines, whose username is a form field.
|
||||||
//
|
//
|
||||||
// - the unknown-entrypoint DEBUG line on /h/{uuid}, whose
|
// They are at DEBUG, which is off in production by default. That is
|
||||||
// path segment matched no stored entrypoint and so is bounded by
|
|
||||||
// nothing;
|
|
||||||
// - the failed-login DEBUG lines, whose username is a form field.
|
|
||||||
//
|
|
||||||
// Both are at DEBUG, which is off in production by default. That is
|
|
||||||
// not a bound: an operator turning DEBUG on to diagnose a flood must
|
// not a bound: an operator turning DEBUG on to diagnose a flood must
|
||||||
// not thereby hand the flood an unbounded write. Both spend the same
|
// not thereby hand the flood an unbounded write. They spend the same
|
||||||
// internal/logfield budget as the access log, and both are held here
|
// internal/logfield budget as the access log, and are held here to
|
||||||
// to middleware.MaxAccessLogLineBytes.
|
// middleware.MaxAccessLogLineBytes.
|
||||||
//
|
//
|
||||||
// The two login lines past the username lookup — "invalid password"
|
// The two login lines past the username lookup — "invalid password"
|
||||||
// and "user logged in" — carry the same cap without needing it, since
|
// and "user logged in" — carry the same cap without needing it, since
|
||||||
@@ -260,43 +256,6 @@ func postLoginWithPassword(
|
|||||||
return w.Code
|
return w.Code
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestUnknownEntrypoint_LogLineDoesNotTrackPathSize drives 8 KB of
|
|
||||||
// client-chosen path at the unauthenticated receiver's
|
|
||||||
// unknown-entrypoint DEBUG line and holds it to the same ceiling the
|
|
||||||
// access log states.
|
|
||||||
func TestUnknownEntrypoint_LogLineDoesNotTrackPathSize(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for handlerName, newHandler := range logHandlers() {
|
|
||||||
for fillName, fill := range escapeFills() {
|
|
||||||
t.Run(handlerName+"/"+fillName, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
h, buf := capturingHandlers(t, newHandler)
|
|
||||||
router := receiverRouter(h)
|
|
||||||
|
|
||||||
for i := range floodRequests {
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
http.StatusNotFound,
|
|
||||||
postReceiver(
|
|
||||||
t, router,
|
|
||||||
oversizedFill(fill)+
|
|
||||||
strings.Repeat("y", i),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
lines := logLines(t, buf)
|
|
||||||
require.Len(t, lines, floodRequests)
|
|
||||||
assertNoClientText(t, buf)
|
|
||||||
|
|
||||||
assertBoundedFlood(t, buf.Len())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFailedLogin_LogLineDoesNotTrackUsernameSize drives 8 KB of
|
// TestFailedLogin_LogLineDoesNotTrackUsernameSize drives 8 KB of
|
||||||
// client-chosen username at the unauthenticated login endpoint's
|
// client-chosen username at the unauthenticated login endpoint's
|
||||||
// DEBUG line and holds it to the same ceiling.
|
// DEBUG line and holds it to the same ceiling.
|
||||||
|
|||||||
@@ -273,7 +273,7 @@ func TestHandleSourceDetail_FitsWideAndNarrowWindows(t *testing.T) {
|
|||||||
|
|
||||||
assert.Contains(
|
assert.Contains(
|
||||||
t, body,
|
t, body,
|
||||||
`<div class="mx-auto px-6 py-8" style="max-width: 108rem"`,
|
`<div class="max-w-[108rem] mx-auto px-6 py-8">`,
|
||||||
)
|
)
|
||||||
assert.Contains(
|
assert.Contains(
|
||||||
t, body,
|
t, body,
|
||||||
@@ -286,8 +286,8 @@ func TestHandleSourceDetail_FitsWideAndNarrowWindows(t *testing.T) {
|
|||||||
// target and says what deleting it loses, that the webhook's gives its
|
// target and says what deleting it loses, that the webhook's gives its
|
||||||
// number of stored events (5 received, 2 removed by retention, so 3,
|
// number of stored events (5 received, 2 removed by retention, so 3,
|
||||||
// the statistics pane's "Within retention" figure), and that an
|
// the statistics pane's "Within retention" figure), and that an
|
||||||
// entrypoint with no description is named by its URL. The template
|
// entrypoint with no description is named by its URL. Each prompt is
|
||||||
// writes the slashes after http: as \/, which the browser reads as /.
|
// its form's data-confirm attribute, where is a line break.
|
||||||
func TestHandleSourceDetail_DeletePromptsNameWhatIsLost(t *testing.T) {
|
func TestHandleSourceDetail_DeletePromptsNameWhatIsLost(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -325,29 +325,28 @@ func TestHandleSourceDetail_DeletePromptsNameWhatIsLost(t *testing.T) {
|
|||||||
body := renderSourceDetailPage(t, h, sess, wh.ID)
|
body := renderSourceDetailPage(t, h, sess, wh.ID)
|
||||||
|
|
||||||
assert.Contains(t, body,
|
assert.Contains(t, body,
|
||||||
`Delete webhook "delete-me"?\n\n`+
|
`data-confirm="Delete webhook "delete-me"? `+
|
||||||
`This deletes its stored events (3) and their deliveries. `+
|
`This deletes its stored events (3) and their deliveries. `+
|
||||||
`Any archive files it wrote are kept.`)
|
`Any archive files it wrote are kept."`)
|
||||||
assert.Contains(t, body,
|
assert.Contains(t, body,
|
||||||
`Delete entrypoint "Stripe"?\n\n`+
|
`data-confirm="Delete entrypoint "Stripe"? `+
|
||||||
`Senders using its URL get an error from now on, `+
|
`Senders using its URL get an error from now on, `+
|
||||||
`and the URL cannot be restored.`)
|
`and the URL cannot be restored."`)
|
||||||
assert.Contains(t, body,
|
assert.Contains(t, body,
|
||||||
`Delete entrypoint "http:\/\/example.com/h/`+
|
`data-confirm="Delete entrypoint "http://example.com/h/`+
|
||||||
unnamed.Path+`"?`)
|
unnamed.Path+`"? `)
|
||||||
assert.Contains(t, body,
|
assert.Contains(t, body,
|
||||||
`Delete target "t-log"?\n\n`+
|
`data-confirm="Delete target "t-log"? `+
|
||||||
`Nothing more is delivered to it. `+
|
`Nothing more is delivered to it. `+
|
||||||
`Its past deliveries stay in the event log.`)
|
`Its past deliveries stay in the event log."`)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleSourceDetail_DeletePromptKeepsQuotesInName checks that a
|
// TestHandleSourceDetail_DeletePromptKeepsQuotesInName checks that a
|
||||||
// webhook name with quotes, a backslash, a closing script tag and a
|
// webhook name with quotes, a backslash, a closing script tag and a
|
||||||
// newline reaches its delete prompt escaped for the script, which the
|
// newline reaches its delete prompt escaped for the attribute, which
|
||||||
// browser reads back as the name typed: each quote and angle bracket
|
// the browser reads back as the name typed: each quote and angle
|
||||||
// as a \u escape, the slash as \/, the newline as \n and the backslash
|
// bracket as a character reference, the rest as typed. An unescaped
|
||||||
// doubled. An unescaped newline would break the prompt's script, and
|
// double quote would end the attribute and cut the prompt short.
|
||||||
// the form would then submit without asking.
|
|
||||||
func TestHandleSourceDetail_DeletePromptKeepsQuotesInName(t *testing.T) {
|
func TestHandleSourceDetail_DeletePromptKeepsQuotesInName(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -373,6 +372,6 @@ func TestHandleSourceDetail_DeletePromptKeepsQuotesInName(t *testing.T) {
|
|||||||
body := renderSourceDetailPage(t, h, sess, wh.ID)
|
body := renderSourceDetailPage(t, h, sess, wh.ID)
|
||||||
|
|
||||||
assert.Contains(t, body,
|
assert.Contains(t, body,
|
||||||
"Delete webhook "Bob\\u0027s \\u0022best\\u0022 \\\\ hook"+
|
"Delete webhook "Bob's "best" \\ hook"+
|
||||||
"\\u003c\\/script\\u003e\\nline two"?")
|
"</script>\nline two"?")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,13 +49,6 @@ var (
|
|||||||
// working retry configuration over a typo. A default answers a
|
// working retry configuration over a typo. A default answers a
|
||||||
// question that was not asked; it never answers one that was asked
|
// question that was not asked; it never answers one that was asked
|
||||||
// badly.
|
// badly.
|
||||||
//
|
|
||||||
// A target stored with a count above the ceiling before this
|
|
||||||
// validation existed keeps rendering and keeps delivering — nothing
|
|
||||||
// clamps the row. Re-saving it from the edit form does have to bring
|
|
||||||
// it into range, because the form submits the pre-filled value back
|
|
||||||
// and accepting it would be the ceiling not applying to the edit
|
|
||||||
// path. The 400 names the ceiling, so the fix is one field.
|
|
||||||
func parseMaxRetries(raw string, fallback int) (int, error) {
|
func parseMaxRetries(raw string, fallback int) (int, error) {
|
||||||
raw = strings.TrimSpace(raw)
|
raw = strings.TrimSpace(raw)
|
||||||
if raw == "" {
|
if raw == "" {
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ import (
|
|||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
"sneak.berlin/go/webhooker/internal/logfield"
|
|
||||||
"sneak.berlin/go/webhooker/internal/middleware"
|
"sneak.berlin/go/webhooker/internal/middleware"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -49,14 +48,13 @@ func (h *Handlers) HandleWebhook() http.HandlerFunc {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Logged only once the UUID is known to name a real
|
// Logged only once the lookup has found the entrypoint, so
|
||||||
// entrypoint. The UUID comes straight out of the path on
|
// an invented path writes no INFO line; a miss is logged at
|
||||||
// the one unauthenticated endpoint, so logging it before
|
// DEBUG. The entrypoint is named by its ID: the UUID is its
|
||||||
// the lookup let a client write an INFO line per invented
|
// credential and is never logged.
|
||||||
// path; the request itself is already in the access log
|
|
||||||
// and a miss is already logged at DEBUG.
|
|
||||||
h.log.Info("webhook request received",
|
h.log.Info("webhook request received",
|
||||||
"entrypoint_uuid", entrypointUUID,
|
"entrypoint_id", entrypoint.ID,
|
||||||
|
"webhook_id", entrypoint.WebhookID,
|
||||||
"method", r.Method,
|
"method", r.Method,
|
||||||
"remoteIP", middleware.RemoteIP(r),
|
"remoteIP", middleware.RemoteIP(r),
|
||||||
"clientIP", middleware.ClientIP(r),
|
"clientIP", middleware.ClientIP(r),
|
||||||
@@ -133,17 +131,10 @@ func (h *Handlers) lookupEntrypoint(
|
|||||||
"path = ?", entrypointUUID,
|
"path = ?", entrypointUUID,
|
||||||
).First(&entrypoint)
|
).First(&entrypoint)
|
||||||
if result.Error != nil {
|
if result.Error != nil {
|
||||||
// The receiver is unauthenticated and /h/{uuid}
|
// Not even an unknown UUID is logged. The receiver's log
|
||||||
// matches any single segment, so this value is entirely
|
// lines never carry the path segment, which is a credential
|
||||||
// client-chosen on exactly the branch where the lookup
|
// whenever it names an entrypoint.
|
||||||
// failed. DEBUG is off by default; the cap is what keeps
|
h.log.Debug("entrypoint not found")
|
||||||
// turning it on from restoring an unbounded write.
|
|
||||||
h.log.Debug(
|
|
||||||
"entrypoint not found",
|
|
||||||
"path", logfield.Truncate(
|
|
||||||
entrypointUUID, logfield.MaxBytes,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
http.NotFound(w, r)
|
http.NotFound(w, r)
|
||||||
|
|
||||||
return entrypoint, false
|
return entrypoint, false
|
||||||
|
|||||||
@@ -12,12 +12,15 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
"github.com/go-chi/chi"
|
||||||
|
"github.com/google/uuid"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
"gorm.io/gorm/clause"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
"sneak.berlin/go/webhooker/internal/handlers"
|
||||||
"sneak.berlin/go/webhooker/internal/middleware"
|
"sneak.berlin/go/webhooker/internal/middleware"
|
||||||
|
"sneak.berlin/go/webhooker/internal/middleware/middlewaretest"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestHandleWebhook_LogsClientNextToThePeer checks that the
|
// TestHandleWebhook_LogsClientNextToThePeer checks that the
|
||||||
@@ -99,6 +102,87 @@ func TestHandleWebhook_LogsClientNextToThePeer(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestHandleWebhook_LogsNoEntrypointUUID sends an accepted event, a
|
||||||
|
// request over the receiver rate limit and a request for an unknown
|
||||||
|
// UUID through the receiver as routes.go mounts it, and checks that
|
||||||
|
// nothing logged, at any level, carries either UUID.
|
||||||
|
func TestHandleWebhook_LogsNoEntrypointUUID(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
var (
|
||||||
|
h *handlers.Handlers
|
||||||
|
db *database.Database
|
||||||
|
)
|
||||||
|
|
||||||
|
cfg := &config.Config{DataDir: t.TempDir(), ReceiverRateLimit: 1}
|
||||||
|
|
||||||
|
app := newTestAppWithConfig(t, cfg, &h, &db)
|
||||||
|
app.RequireStart()
|
||||||
|
|
||||||
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
|
buf := new(bytes.Buffer)
|
||||||
|
log := slog.New(slog.NewJSONHandler(
|
||||||
|
buf, &slog.HandlerOptions{Level: slog.LevelDebug},
|
||||||
|
))
|
||||||
|
h.SetLogForTest(log)
|
||||||
|
mw := middlewaretest.New(t, log, cfg, nil)
|
||||||
|
|
||||||
|
webhook := seedWebhook(t, db)
|
||||||
|
entrypoint := &database.Entrypoint{
|
||||||
|
WebhookID: webhook.ID,
|
||||||
|
Path: uuid.New().String(),
|
||||||
|
Active: true,
|
||||||
|
}
|
||||||
|
require.NoError(t, db.DB().Omit(clause.Associations).
|
||||||
|
Create(entrypoint).Error)
|
||||||
|
|
||||||
|
unknownUUID := uuid.New().String()
|
||||||
|
|
||||||
|
router := chi.NewRouter()
|
||||||
|
router.Use(mw.Logging())
|
||||||
|
router.With(mw.ReceiverRateLimit()).HandleFunc(
|
||||||
|
middleware.ReceiverRoute, h.HandleWebhook(),
|
||||||
|
)
|
||||||
|
|
||||||
|
// In order: the limit is one request per minute per path, so the
|
||||||
|
// second request for the known UUID is the rejected one.
|
||||||
|
for _, step := range []struct {
|
||||||
|
entrypointUUID string
|
||||||
|
wantStatus int
|
||||||
|
}{
|
||||||
|
{entrypoint.Path, http.StatusOK},
|
||||||
|
{entrypoint.Path, http.StatusTooManyRequests},
|
||||||
|
{unknownUUID, http.StatusNotFound},
|
||||||
|
} {
|
||||||
|
req := httptest.NewRequestWithContext(
|
||||||
|
context.Background(), http.MethodPost,
|
||||||
|
"/h/"+step.entrypointUUID, strings.NewReader("{}"),
|
||||||
|
)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
require.Equal(t, step.wantStatus, w.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The lines this test is about were all written, so the checks
|
||||||
|
// below are not passing on an empty log.
|
||||||
|
for _, msg := range []string{
|
||||||
|
"webhook request received",
|
||||||
|
"webhook receiver rate limit exceeded",
|
||||||
|
"entrypoint not found",
|
||||||
|
} {
|
||||||
|
assert.Contains(t, buf.String(), `"msg":"`+msg+`"`)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.NotContains(t, buf.String(), entrypoint.Path)
|
||||||
|
assert.NotContains(t, buf.String(), unknownUUID)
|
||||||
|
|
||||||
|
line := receivedLine(t, buf)
|
||||||
|
assert.Equal(t, entrypoint.ID, line["entrypoint_id"])
|
||||||
|
assert.Equal(t, webhook.ID, line["webhook_id"])
|
||||||
|
}
|
||||||
|
|
||||||
// receivedLine returns the one "webhook request received" line in the
|
// receivedLine returns the one "webhook request received" line in the
|
||||||
// captured JSON log.
|
// captured JSON log.
|
||||||
func receivedLine(t *testing.T, buf *bytes.Buffer) map[string]any {
|
func receivedLine(t *testing.T, buf *bytes.Buffer) map[string]any {
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -27,6 +28,13 @@ const floodRequests = 64
|
|||||||
// line for a redirected or rejected request may contain it.
|
// line for a redirected or rejected request may contain it.
|
||||||
const attackerMarker = "QQATTACKERTEXTQQ"
|
const attackerMarker = "QQATTACKERTEXTQQ"
|
||||||
|
|
||||||
|
// knownEntrypointUUID and failingEntrypointUUID are the UUIDs the
|
||||||
|
// stand-in receiver in accessLogRouter answers 200 and 500 to.
|
||||||
|
const (
|
||||||
|
knownEntrypointUUID = "6f1e2a9c-4b7d-4e3a-9c2f-1d8b5a7e3c60"
|
||||||
|
failingEntrypointUUID = "0b7c3e5a-2d9f-4a61-8e4b-7c1d6f2a9e35"
|
||||||
|
)
|
||||||
|
|
||||||
// maxLineBytes bounds a single access log line whose client-supplied
|
// maxLineBytes bounds a single access log line whose client-supplied
|
||||||
// fields are of ordinary size. Well above what the fixed fields need,
|
// fields are of ordinary size. Well above what the fixed fields need,
|
||||||
// well below the length of the oversized input the amplification tests
|
// well below the length of the oversized input the amplification tests
|
||||||
@@ -132,17 +140,21 @@ func accessLogRouter(m *middleware.Middleware) *chi.Mux {
|
|||||||
)
|
)
|
||||||
|
|
||||||
router.HandleFunc(
|
router.HandleFunc(
|
||||||
"/h/{uuid}",
|
middleware.ReceiverRoute,
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
// Stands in for the real handler: an unknown entrypoint
|
// Stands in for the real handler: a known entrypoint
|
||||||
// UUID 404s, a known one succeeds.
|
// UUID succeeds, a failing one is a server error, and
|
||||||
if chi.URLParam(r, "uuid") != "known" {
|
// any other 404s.
|
||||||
|
switch chi.URLParam(r, "uuid") {
|
||||||
|
case knownEntrypointUUID:
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
case failingEntrypointUUID:
|
||||||
|
http.Error(
|
||||||
|
w, "boom", http.StatusInternalServerError,
|
||||||
|
)
|
||||||
|
default:
|
||||||
http.Error(w, "not found", http.StatusNotFound)
|
http.Error(w, "not found", http.StatusNotFound)
|
||||||
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -335,11 +347,13 @@ func oversizedValue(ch string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// oversizedHeaders fills every client-supplied header the access log
|
// oversizedHeaders fills every client-supplied header the access log
|
||||||
// reads with the same value.
|
// reads with the same value. The Referer carries it as the host of an
|
||||||
|
// absolute URL, because the access log keeps only a Referer's scheme
|
||||||
|
// and host.
|
||||||
func oversizedHeaders(value string) map[string]string {
|
func oversizedHeaders(value string) map[string]string {
|
||||||
return map[string]string{
|
return map[string]string{
|
||||||
"User-Agent": value,
|
"User-Agent": value,
|
||||||
"Referer": value,
|
"Referer": "http://" + value,
|
||||||
"X-Request-Id": value,
|
"X-Request-Id": value,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -649,17 +663,111 @@ func TestAccessLog_SuccessKeepsConcretePathAndRedactsQuery(
|
|||||||
router := accessLogRouter(m)
|
router := accessLogRouter(m)
|
||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, http.StatusOK, get(t, router, "/h/known?src=ci"),
|
t, http.StatusOK,
|
||||||
|
get(t, router, "/.well-known/healthcheck?src=ci"),
|
||||||
)
|
)
|
||||||
|
|
||||||
// The path resolved against a stored entrypoint, so it stays. The
|
|
||||||
// query never does: see TestAccessLog_UnauthenticatedSuccess...
|
|
||||||
entries := accessLogEntries(t, buf)
|
entries := accessLogEntries(t, buf)
|
||||||
require.Len(t, entries, 1)
|
require.Len(t, entries, 1)
|
||||||
assert.Equal(t, "/h/known?(redacted)", entries[0]["url"])
|
assert.Equal(
|
||||||
|
t, "/.well-known/healthcheck?(redacted)", entries[0]["url"],
|
||||||
|
)
|
||||||
assert.NotContains(t, buf.String(), "src=ci")
|
assert.NotContains(t, buf.String(), "src=ci")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestAccessLog_ReceiverNeverLogsTheUUID covers the two statuses that
|
||||||
|
// keep the concrete path on every other route. On the receiver that
|
||||||
|
// path carries the entrypoint UUID, so the line names the route
|
||||||
|
// pattern instead.
|
||||||
|
func TestAccessLog_ReceiverNeverLogsTheUUID(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for entrypointUUID, wantStatus := range map[string]int{
|
||||||
|
knownEntrypointUUID: http.StatusOK,
|
||||||
|
failingEntrypointUUID: http.StatusInternalServerError,
|
||||||
|
} {
|
||||||
|
t.Run(strconv.Itoa(wantStatus), func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
m, buf := capturingMiddleware(t)
|
||||||
|
router := accessLogRouter(m)
|
||||||
|
|
||||||
|
assert.Equal(
|
||||||
|
t, wantStatus,
|
||||||
|
get(t, router, "/h/"+entrypointUUID+"?src=ci"),
|
||||||
|
)
|
||||||
|
|
||||||
|
entries := accessLogEntries(t, buf)
|
||||||
|
require.Len(t, entries, 1)
|
||||||
|
assert.Equal(t, middleware.ReceiverRoute, entries[0]["url"])
|
||||||
|
assert.NotContains(t, buf.String(), entrypointUUID)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAccessLog_ReceiverPreflightNeverLogsTheUUID sends a CORS
|
||||||
|
// preflight to the receiver in dev, in the production middleware
|
||||||
|
// order. CORS answers it with 200 before chi routes it, so the line
|
||||||
|
// names no route and carries no UUID.
|
||||||
|
func TestAccessLog_ReceiverPreflightNeverLogsTheUUID(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
m, buf := capturingMiddleware(t)
|
||||||
|
|
||||||
|
router := chi.NewRouter()
|
||||||
|
router.Use(m.Logging())
|
||||||
|
router.Use(m.CORS())
|
||||||
|
router.HandleFunc(
|
||||||
|
middleware.ReceiverRoute,
|
||||||
|
func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
req := httptest.NewRequestWithContext(
|
||||||
|
context.Background(), http.MethodOptions,
|
||||||
|
"/h/"+knownEntrypointUUID, nil,
|
||||||
|
)
|
||||||
|
req.Header.Set("Origin", "https://sender.example")
|
||||||
|
req.Header.Set("Access-Control-Request-Method", http.MethodPost)
|
||||||
|
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(w, req)
|
||||||
|
assert.Equal(t, http.StatusOK, w.Code)
|
||||||
|
|
||||||
|
entries := accessLogEntries(t, buf)
|
||||||
|
require.Len(t, entries, 1)
|
||||||
|
assert.Equal(t, unmatchedRouteLiteral, entries[0]["url"])
|
||||||
|
assert.NotContains(t, buf.String(), knownEntrypointUUID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAccessLog_RefererNeverLogsTheUUID sends receiver requests whose
|
||||||
|
// Referer is the entrypoint URL, as a sender's is after it follows a
|
||||||
|
// redirect from http:// to https://. The referer field keeps only the
|
||||||
|
// scheme and host, and a Referer without them is logged as a marker.
|
||||||
|
func TestAccessLog_RefererNeverLogsTheUUID(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
m, buf := capturingMiddleware(t)
|
||||||
|
router := accessLogRouter(m)
|
||||||
|
|
||||||
|
for _, referer := range []string{
|
||||||
|
"http://webhooker.example/h/" + knownEntrypointUUID + "?src=ci",
|
||||||
|
"/h/" + knownEntrypointUUID,
|
||||||
|
} {
|
||||||
|
assert.Equal(t, http.StatusOK, getWithHeaders(
|
||||||
|
t, router, "/h/"+knownEntrypointUUID,
|
||||||
|
map[string]string{"Referer": referer},
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
entries := accessLogEntries(t, buf)
|
||||||
|
require.Len(t, entries, 2)
|
||||||
|
assert.Equal(t, "http://webhooker.example", entries[0]["referer"])
|
||||||
|
assert.Equal(t, "(redacted)", entries[1]["referer"])
|
||||||
|
assert.NotContains(t, buf.String(), knownEntrypointUUID)
|
||||||
|
}
|
||||||
|
|
||||||
func TestAccessLog_ServerErrorKeepsConcreteURL(t *testing.T) {
|
func TestAccessLog_ServerErrorKeepsConcreteURL(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -10,8 +10,10 @@ package middleware_test
|
|||||||
// the client's choosing.
|
// the client's choosing.
|
||||||
// - CSRF's 403 rejection, at WARN, also registered ahead of
|
// - CSRF's 403 rejection, at WARN, also registered ahead of
|
||||||
// RequireAuth.
|
// RequireAuth.
|
||||||
// - The rate limiters' 429 rejection, at WARN, on the
|
// - The rate limiters' 429 rejection, at WARN. On the receiver
|
||||||
// unauthenticated receiver among others.
|
// route it names the route pattern rather than the path, and
|
||||||
|
// the limiters that log a path sit behind RequireAuth, so its cap
|
||||||
|
// is defensive.
|
||||||
// - RequireAuth's own unauthenticated-request line, at DEBUG.
|
// - RequireAuth's own unauthenticated-request line, at DEBUG.
|
||||||
// - RecordLoginFailure's throttle rejection, at WARN. Its cap is
|
// - RecordLoginFailure's throttle rejection, at WARN. Its cap is
|
||||||
// defensive rather than load-bearing today: chi pins the one
|
// defensive rather than load-bearing today: chi pins the one
|
||||||
@@ -268,10 +270,14 @@ func logSites() map[string]logSite {
|
|||||||
send: postNoToken,
|
send: postNoToken,
|
||||||
wantStatus: http.StatusForbidden,
|
wantStatus: http.StatusForbidden,
|
||||||
},
|
},
|
||||||
// The per-entrypoint receiver limiter, unauthenticated. Its
|
// The 429 line every limiter but the aggregate one shares.
|
||||||
// bucket is keyed on the path, so the first request through a
|
// Mounted outside any route, as here, it logs the path, as
|
||||||
// fresh path is served and only the ones after it are
|
// it does for the limiters behind RequireAuth; on
|
||||||
// rejected; sendUntilLimited absorbs that.
|
// ReceiverRoute it names the pattern instead. The
|
||||||
|
// per-entrypoint receiver limiter's bucket is keyed on the
|
||||||
|
// path, so the first request through a fresh path is served
|
||||||
|
// and only the ones after it are rejected; sendUntilLimited
|
||||||
|
// absorbs that.
|
||||||
"receiver rate limit 429": {
|
"receiver rate limit 429": {
|
||||||
build: func(
|
build: func(
|
||||||
t *testing.T, m *middleware.Middleware,
|
t *testing.T, m *middleware.Middleware,
|
||||||
|
|||||||
@@ -41,7 +41,8 @@ const unmatchedMethod = unmatchedRoute
|
|||||||
// after /h/ is client-chosen, so labelling by path lets any
|
// after /h/ is client-chosen, so labelling by path lets any
|
||||||
// unauthenticated client mint permanent series at will, and publishes
|
// unauthenticated client mint permanent series at will, and publishes
|
||||||
// the entrypoint UUID (the receiver's only credential) in the scrape
|
// the entrypoint UUID (the receiver's only credential) in the scrape
|
||||||
// while doing it.
|
// while doing it. The access log and the 429 line read it for the
|
||||||
|
// same reason: see ReceiverRoute.
|
||||||
//
|
//
|
||||||
// chi populates the route context during routeHTTP, so this is only
|
// chi populates the route context during routeHTTP, so this is only
|
||||||
// valid once routing has run. Every caller below is on the recording
|
// valid once routing has run. Every caller below is on the recording
|
||||||
|
|||||||
@@ -13,7 +13,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
basicauth "github.com/99designs/basicauth-go"
|
basicauth "github.com/99designs/basicauth-go"
|
||||||
"github.com/go-chi/chi"
|
|
||||||
"github.com/go-chi/chi/middleware"
|
"github.com/go-chi/chi/middleware"
|
||||||
"github.com/go-chi/cors"
|
"github.com/go-chi/cors"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
@@ -49,6 +48,10 @@ const (
|
|||||||
// pick the size of the line it writes.
|
// pick the size of the line it writes.
|
||||||
redactedQuery = "?(redacted)"
|
redactedQuery = "?(redacted)"
|
||||||
|
|
||||||
|
// redactedReferer stands in for a Referer the access log cannot
|
||||||
|
// reduce to its scheme and host; see accessLogReferer.
|
||||||
|
redactedReferer = "(redacted)"
|
||||||
|
|
||||||
// maxLogRequestIDBytes bounds the request id, which is also
|
// maxLogRequestIDBytes bounds the request id, which is also
|
||||||
// client-supplied: chi's RequestID middleware passes an inbound
|
// client-supplied: chi's RequestID middleware passes an inbound
|
||||||
// X-Request-Id header through verbatim. Its generated form is an
|
// X-Request-Id header through verbatim. Its generated form is an
|
||||||
@@ -96,14 +99,14 @@ const (
|
|||||||
// It is also the ceiling on every OTHER line this service writes
|
// It is also the ceiling on every OTHER line this service writes
|
||||||
// THROUGH SLOG that carries text an UNAUTHENTICATED client
|
// THROUGH SLOG that carries text an UNAUTHENTICATED client
|
||||||
// supplies. Those lines — the MaxBodySize rejection, the CSRF
|
// supplies. Those lines — the MaxBodySize rejection, the CSRF
|
||||||
// rejection, the rate-limit rejection, the unauthenticated-request
|
// rejection, the unauthenticated-request DEBUG line, the
|
||||||
// and unknown-entrypoint DEBUG lines, the failed-login DEBUG
|
// failed-login DEBUG lines, and the two login-throttle WARN
|
||||||
// lines, and the two login-throttle WARN lines ("login failure
|
// lines ("login failure limit exceeded" in loginguard.go and
|
||||||
// limit exceeded" in loginguard.go and "password verification
|
// "password verification capacity exhausted" in
|
||||||
// capacity exhausted" in internal/handlers/auth.go) — spend the
|
// internal/handlers/auth.go) — spend the same per-field budgets,
|
||||||
// same per-field budgets, and each carries strictly fewer
|
// and each carries strictly fewer client-supplied fields than the
|
||||||
// client-supplied fields than the access log does,
|
// access log does, so none of them can reach a width the access
|
||||||
// so none of them can reach a width the access log cannot. That is
|
// log cannot. That is
|
||||||
// asserted directly, per line and under both handlers, rather than
|
// asserted directly, per line and under both handlers, rather than
|
||||||
// left to the reasoning: see logbound_test.go in this package and
|
// left to the reasoning: see logbound_test.go in this package and
|
||||||
// in internal/handlers.
|
// in internal/handlers.
|
||||||
@@ -151,6 +154,12 @@ const (
|
|||||||
MaxAccessLogLineBytes = 2560
|
MaxAccessLogLineBytes = 2560
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ReceiverRoute is the route pattern the webhook receiver is served
|
||||||
|
// at. Its path carries the entrypoint UUID, which is the receiver's
|
||||||
|
// only credential, so the log lines this package writes name a
|
||||||
|
// request on this route by the pattern, never by its path.
|
||||||
|
const ReceiverRoute = "/h/{uuid}"
|
||||||
|
|
||||||
//nolint:revive // MiddlewareParams is a standard fx naming convention.
|
//nolint:revive // MiddlewareParams is a standard fx naming convention.
|
||||||
type MiddlewareParams struct {
|
type MiddlewareParams struct {
|
||||||
fx.In
|
fx.In
|
||||||
@@ -268,14 +277,14 @@ func (lrw *loggingResponseWriter) Unwrap() http.ResponseWriter {
|
|||||||
// marker.
|
// marker.
|
||||||
//
|
//
|
||||||
// The path on those branches is bounded by the service's routes or by
|
// The path on those branches is bounded by the service's routes or by
|
||||||
// the operator's data — a 2xx on the receiver means the UUID named a
|
// the operator's data — a 2xx under /s means the file is in the
|
||||||
// stored entrypoint, a 2xx under /s means the file is in the embedded
|
// embedded tree. The query is not bounded by anything:
|
||||||
// tree. The query is not bounded by anything: /.well-known/healthcheck
|
// /.well-known/healthcheck and /s/* take no authentication and sit
|
||||||
// and /s/* take no authentication and sit behind no rate limiter, and
|
// behind no rate limiter, and /pages/login behind only the login
|
||||||
// /pages/login behind only the login limiter, so any of them will
|
// limiter, so any of them will answer 200 to a URL carrying an
|
||||||
// answer 200 to a URL carrying an arbitrary number of arbitrary bytes
|
// arbitrary number of arbitrary bytes after the '?'. Keeping the path
|
||||||
// after the '?'. Keeping the path and dropping the query is what makes
|
// and dropping the query is what makes this branch as bounded as the
|
||||||
// this branch as bounded as the pattern branches below.
|
// pattern branches below.
|
||||||
//
|
//
|
||||||
// Nothing debuggable is lost. The only query parameters the service
|
// Nothing debuggable is lost. The only query parameters the service
|
||||||
// reads are the login page's `next`, the page to return to,
|
// reads are the login page's `next`, the page to return to,
|
||||||
@@ -297,36 +306,61 @@ func concreteLogURL(r *http.Request) string {
|
|||||||
|
|
||||||
// accessLogURL returns the value for the access log's url field.
|
// accessLogURL returns the value for the access log's url field.
|
||||||
//
|
//
|
||||||
// 2xx and 5xx responses get the concrete path (see concreteLogURL). A
|
// On ReceiverRoute it is the route pattern whatever the status, so
|
||||||
// success resolved against a static route or against the operator's
|
// the entrypoint UUID in the path is never logged. A request chi never
|
||||||
// own data — on the receiver, a 2xx means the UUID named a stored
|
// routed gets unmatchedRoute whatever the status, for the same reason:
|
||||||
// entrypoint — and a server error is our own bug, where the exact URL
|
// in dev, CORS answers a preflight to the receiver before routing.
|
||||||
// is the primary evidence and which no client can provoke at will.
|
//
|
||||||
|
// On every other route, 2xx and 5xx responses get the concrete path
|
||||||
|
// (see concreteLogURL). A success resolved against a static route or
|
||||||
|
// against the operator's own data, and a server error is our own bug,
|
||||||
|
// where the exact URL is the primary evidence and which no client can
|
||||||
|
// provoke at will.
|
||||||
//
|
//
|
||||||
// 3xx and 4xx responses get the chi route pattern instead. Those are
|
// 3xx and 4xx responses get the chi route pattern instead. Those are
|
||||||
// the outcomes an unauthenticated client drives for free: 404 or 429
|
// the outcomes an unauthenticated client drives for free, such as a
|
||||||
// on any invented /h/ path, 303 to the login page on any
|
// 303 to the login page on any invented /user/ path. Logging the
|
||||||
// invented /user/ path. Logging the concrete URL there lets a flood
|
// concrete URL there lets a flood write attacker-chosen text, of
|
||||||
// write attacker-chosen text, of attacker-chosen length, into the
|
// attacker-chosen length, into the operator's log at one line per
|
||||||
// operator's log at one line per request. The pattern comes from the
|
// request. The pattern comes from the router's own table, so it is
|
||||||
// router's own table, so it is bounded by the service's routes while
|
// bounded by the service's routes while still naming which class of
|
||||||
// still naming which class of request was rejected.
|
// request was rejected.
|
||||||
//
|
//
|
||||||
// The pattern is only populated once routing has run, so this must be
|
// The pattern is only populated once routing has run, so this must be
|
||||||
// called after the handler returns, not before.
|
// called after the handler returns, not before.
|
||||||
func accessLogURL(r *http.Request, status int) string {
|
func accessLogURL(r *http.Request, status int) string {
|
||||||
|
pattern := routePatternID(r.Context())
|
||||||
|
if pattern == ReceiverRoute || pattern == unmatchedRoute {
|
||||||
|
return pattern
|
||||||
|
}
|
||||||
|
|
||||||
if status < http.StatusMultipleChoices ||
|
if status < http.StatusMultipleChoices ||
|
||||||
status >= http.StatusInternalServerError {
|
status >= http.StatusInternalServerError {
|
||||||
return concreteLogURL(r)
|
return concreteLogURL(r)
|
||||||
}
|
}
|
||||||
|
|
||||||
if rc := chi.RouteContext(r.Context()); rc != nil {
|
return pattern
|
||||||
if pattern := rc.RoutePattern(); pattern != "" {
|
}
|
||||||
return pattern
|
|
||||||
}
|
// accessLogReferer returns the value for the access log's referer
|
||||||
|
// field: the Referer's scheme and host, never its path or query. The
|
||||||
|
// path can be an entrypoint URL, which carries the UUID: a sender that
|
||||||
|
// follows a redirect from http:// to https:// sends the URL it was
|
||||||
|
// redirected from, and a browser sends the 404 page's URL under /h/
|
||||||
|
// when it fetches that page's stylesheet. A Referer without both a
|
||||||
|
// scheme and a host is logged as redactedReferer.
|
||||||
|
func accessLogReferer(r *http.Request) string {
|
||||||
|
referer := r.Referer()
|
||||||
|
if referer == "" {
|
||||||
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
return unmatchedRoute
|
parsed, err := url.Parse(referer)
|
||||||
|
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
||||||
|
return redactedReferer
|
||||||
|
}
|
||||||
|
|
||||||
|
return parsed.Scheme + "://" + parsed.Host
|
||||||
}
|
}
|
||||||
|
|
||||||
// Logging returns middleware that logs each HTTP request with
|
// Logging returns middleware that logs each HTTP request with
|
||||||
@@ -379,7 +413,7 @@ func (s *Middleware) Logging() func(http.Handler) http.Handler {
|
|||||||
requestID, maxLogRequestIDBytes,
|
requestID, maxLogRequestIDBytes,
|
||||||
),
|
),
|
||||||
"referer", logfield.Truncate(
|
"referer", logfield.Truncate(
|
||||||
r.Referer(), logfield.MaxBytes,
|
accessLogReferer(r), logfield.MaxBytes,
|
||||||
),
|
),
|
||||||
"proto", r.Proto,
|
"proto", r.Proto,
|
||||||
"remoteIP", RemoteIP(r),
|
"remoteIP", RemoteIP(r),
|
||||||
@@ -538,7 +572,9 @@ func (s *Middleware) MetricsAuth() func(http.Handler) http.Handler {
|
|||||||
// SecurityHeaders returns middleware that sets production security
|
// SecurityHeaders returns middleware that sets production security
|
||||||
// headers on every response: HSTS (unless HSTS_MAX_AGE is zero),
|
// headers on every response: HSTS (unless HSTS_MAX_AGE is zero),
|
||||||
// X-Content-Type-Options, X-Frame-Options, CSP, Referrer-Policy, and
|
// X-Content-Type-Options, X-Frame-Options, CSP, Referrer-Policy, and
|
||||||
// Permissions-Policy.
|
// Permissions-Policy. The CSP takes scripts and styles only from
|
||||||
|
// files on this origin, so a template may hold no inline script,
|
||||||
|
// event handler or style attribute.
|
||||||
func (s *Middleware) SecurityHeaders() func(http.Handler) http.Handler {
|
func (s *Middleware) SecurityHeaders() func(http.Handler) http.Handler {
|
||||||
maxAge := s.params.Config.HSTSMaxAge
|
maxAge := s.params.Config.HSTSMaxAge
|
||||||
hsts := "max-age=" + strconv.FormatInt(int64(maxAge/time.Second), 10) +
|
hsts := "max-age=" + strconv.FormatInt(int64(maxAge/time.Second), 10) +
|
||||||
@@ -560,8 +596,8 @@ func (s *Middleware) SecurityHeaders() func(http.Handler) http.Handler {
|
|||||||
w.Header().Set(
|
w.Header().Set(
|
||||||
"Content-Security-Policy",
|
"Content-Security-Policy",
|
||||||
"default-src 'self'; "+
|
"default-src 'self'; "+
|
||||||
"script-src 'self' 'unsafe-inline'; "+
|
"script-src 'self'; "+
|
||||||
"style-src 'self' 'unsafe-inline'",
|
"style-src 'self'",
|
||||||
)
|
)
|
||||||
w.Header().Set(
|
w.Header().Set(
|
||||||
"Referrer-Policy",
|
"Referrer-Policy",
|
||||||
|
|||||||
@@ -747,10 +747,10 @@ func TestNoCache_SetsHeaders(t *testing.T) {
|
|||||||
|
|
||||||
// --- SecurityHeaders Middleware Tests ---
|
// --- SecurityHeaders Middleware Tests ---
|
||||||
|
|
||||||
// hstsHeaders serves one request through SecurityHeaders built over
|
// securityHeaders serves one request through SecurityHeaders built
|
||||||
// a Config with the given HSTSMaxAge, and returns every
|
// over a Config with the given HSTSMaxAge, and returns the response's
|
||||||
// Strict-Transport-Security value on the response.
|
// headers.
|
||||||
func hstsHeaders(t *testing.T, maxAge time.Duration) []string {
|
func securityHeaders(t *testing.T, maxAge time.Duration) http.Header {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
cfg := &config.Config{HSTSMaxAge: maxAge}
|
cfg := &config.Config{HSTSMaxAge: maxAge}
|
||||||
@@ -772,7 +772,7 @@ func hstsHeaders(t *testing.T, maxAge time.Duration) []string {
|
|||||||
|
|
||||||
handler.ServeHTTP(w, req)
|
handler.ServeHTTP(w, req)
|
||||||
|
|
||||||
return w.Header().Values("Strict-Transport-Security")
|
return w.Header()
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSecurityHeaders_HSTS(t *testing.T) {
|
func TestSecurityHeaders_HSTS(t *testing.T) {
|
||||||
@@ -804,11 +804,28 @@ func TestSecurityHeaders_HSTS(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
assert.Equal(t, tt.expected, hstsHeaders(t, tt.maxAge))
|
assert.Equal(
|
||||||
|
t, tt.expected,
|
||||||
|
securityHeaders(t, tt.maxAge).Values(
|
||||||
|
"Strict-Transport-Security",
|
||||||
|
),
|
||||||
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestSecurityHeaders_CSP pins the Content-Security-Policy: a page
|
||||||
|
// takes its scripts and styles only from files on its own origin,
|
||||||
|
// never from an inline script, event handler or style.
|
||||||
|
func TestSecurityHeaders_CSP(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
assert.Equal(
|
||||||
|
t, "default-src 'self'; script-src 'self'; style-src 'self'",
|
||||||
|
securityHeaders(t, 0).Get("Content-Security-Policy"),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
// --- MaxBodySize Middleware Tests ---
|
// --- MaxBodySize Middleware Tests ---
|
||||||
|
|
||||||
const testBodyLimit int64 = 64
|
const testBodyLimit int64 = 64
|
||||||
|
|||||||
@@ -259,26 +259,28 @@ func (m *Middleware) clientAddr(r *http.Request) (netip.Addr, bool) {
|
|||||||
return peer, true
|
return peer, true
|
||||||
}
|
}
|
||||||
|
|
||||||
// tooManyRequests returns the 429 handler used by the
|
// tooManyRequests returns the 429 handler used by the postRateLimit
|
||||||
// password-change and per-entrypoint receiver limiters: it logs the
|
// limiters and the per-entrypoint receiver limiter: it logs the
|
||||||
// rejection with logMessage and answers with responseMessage.
|
// rejection with logMessage and answers with responseMessage.
|
||||||
// httprate adds the Retry-After header (RFC 6585). The aggregate
|
// httprate adds the Retry-After header (RFC 6585). The aggregate
|
||||||
// receiver limiter uses floodTooManyRequests instead.
|
// receiver limiter uses floodTooManyRequests instead.
|
||||||
//
|
//
|
||||||
// The path is capped against the same budget as the access log's url
|
// On ReceiverRoute the line names the route pattern, so the
|
||||||
// field. The per-entrypoint receiver limiter is unauthenticated and
|
// entrypoint UUID in the path is never logged. Elsewhere it names the
|
||||||
// its path is a client-chosen segment of client-chosen length, so at
|
// path, capped against the same budget as the access log's url field;
|
||||||
// WARN an uncapped path would let a sender pick the size of the line
|
// the limiters there sit behind RequireAuth, so the cap is defensive.
|
||||||
// it writes — the same defect the access log capping closed.
|
|
||||||
func (m *Middleware) tooManyRequests(
|
func (m *Middleware) tooManyRequests(
|
||||||
logMessage, responseMessage string,
|
logMessage, responseMessage string,
|
||||||
) http.HandlerFunc {
|
) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
path := r.URL.Path
|
||||||
|
if routePatternID(r.Context()) == ReceiverRoute {
|
||||||
|
path = ReceiverRoute
|
||||||
|
}
|
||||||
|
|
||||||
m.log.Warn(
|
m.log.Warn(
|
||||||
logMessage,
|
logMessage,
|
||||||
"path", logfield.Truncate(
|
"path", logfield.Truncate(path, logfield.MaxBytes),
|
||||||
r.URL.Path, logfield.MaxBytes,
|
|
||||||
),
|
|
||||||
"remoteIP", RemoteIP(r),
|
"remoteIP", RemoteIP(r),
|
||||||
"clientIP", ClientIP(r),
|
"clientIP", ClientIP(r),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -60,8 +60,8 @@ const (
|
|||||||
// TestAlpineRunsUnderTheSecurityPolicy loads the webhook page and the
|
// TestAlpineRunsUnderTheSecurityPolicy loads the webhook page and the
|
||||||
// event log in a headless browser, served by the real router and so
|
// event log in a headless browser, served by the real router and so
|
||||||
// under the real Content-Security-Policy, and checks that the pages'
|
// under the real Content-Security-Policy, and checks that the pages'
|
||||||
// Alpine.js directives and the copy control work, and that a target's
|
// Alpine.js directives, the copy control and the delete prompts work,
|
||||||
// row shows its delivery counts.
|
// and that a target's row shows its delivery counts.
|
||||||
func TestAlpineRunsUnderTheSecurityPolicy(t *testing.T) {
|
func TestAlpineRunsUnderTheSecurityPolicy(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -74,6 +74,10 @@ func TestAlpineRunsUnderTheSecurityPolicy(t *testing.T) {
|
|||||||
userID, _ := env.seedUser(t, "browser", "browser-password")
|
userID, _ := env.seedUser(t, "browser", "browser-password")
|
||||||
webhook, older, event, target := seedBrowserWebhook(t, env, userID)
|
webhook, older, event, target := seedBrowserWebhook(t, env, userID)
|
||||||
|
|
||||||
|
// The sign-in page is loaded before the browser signs in, since a
|
||||||
|
// signed-in visit to it is sent on to another page.
|
||||||
|
checkOtherPages(ctx, t, srv.URL+"/pages/login")
|
||||||
|
|
||||||
require.NoError(t, chromedp.Run(
|
require.NoError(t, chromedp.Run(
|
||||||
ctx, setCookies(srv.URL, env.authCookies(t, userID, "browser")),
|
ctx, setCookies(srv.URL, env.authCookies(t, userID, "browser")),
|
||||||
))
|
))
|
||||||
@@ -99,6 +103,11 @@ func TestAlpineRunsUnderTheSecurityPolicy(t *testing.T) {
|
|||||||
checkEventLog(ctx, t, page+"/events", event.ID, older.ID, target.Name)
|
checkEventLog(ctx, t, page+"/events", event.ID, older.ID, target.Name)
|
||||||
checkMobileMenu(ctx, t, page)
|
checkMobileMenu(ctx, t, page)
|
||||||
checkPhoneWidth(ctx, t, page, page+"/events", target.Name)
|
checkPhoneWidth(ctx, t, page, page+"/events", target.Name)
|
||||||
|
checkOtherPages(ctx, t,
|
||||||
|
srv.URL+"/hooks", srv.URL+"/settings", srv.URL+"/user/browser",
|
||||||
|
// A path that does not exist, which gets the 404 error page.
|
||||||
|
srv.URL+"/no-such-page")
|
||||||
|
checkDeletePrompt(ctx, t, page, target)
|
||||||
|
|
||||||
assert.Empty(t, problems(), "the browser reported problems")
|
assert.Empty(t, problems(), "the browser reported problems")
|
||||||
}
|
}
|
||||||
@@ -1388,3 +1397,75 @@ func checkPhoneWidth(
|
|||||||
assert.False(t, sideways, "the event log scrolls sideways on a phone")
|
assert.False(t, sideways, "the event log scrolls sideways on a phone")
|
||||||
assert.Empty(t, cutOff, "the event log cuts these off on a phone")
|
assert.Empty(t, cutOff, "the event log cuts these off on a phone")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// checkOtherPages loads the pages at urls, which no other check opens,
|
||||||
|
// so that anything the browser reports going wrong on them fails the
|
||||||
|
// test too.
|
||||||
|
func checkOtherPages(ctx context.Context, t *testing.T, urls ...string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
for _, url := range urls {
|
||||||
|
require.NoError(t, chromedp.Run(ctx, loadPage(url)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// answerPrompt returns a script that stands in for the user at the
|
||||||
|
// browser's prompt: it makes window.confirm record the question in
|
||||||
|
// window.asked and give answer. It also records in window.sent whether
|
||||||
|
// a form is sent once its submit event is handled, from a listener on
|
||||||
|
// window, which the event reaches after the listener static/js/app.js
|
||||||
|
// adds to document.
|
||||||
|
func answerPrompt(answer bool) string {
|
||||||
|
return fmt.Sprintf(`
|
||||||
|
window.asked = "";
|
||||||
|
window.sent = false;
|
||||||
|
window.confirm = (question) => {
|
||||||
|
window.asked = question;
|
||||||
|
return %t;
|
||||||
|
};
|
||||||
|
window.addEventListener("submit", (event) => {
|
||||||
|
window.sent = !event.defaultPrevented;
|
||||||
|
});`, answer)
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkDeletePrompt loads a webhook page and clicks Delete beside
|
||||||
|
// target. It checks that the browser asks first, naming the target,
|
||||||
|
// that answering no leaves the form unsent, and that answering yes
|
||||||
|
// sends it and the target is deleted.
|
||||||
|
func checkDeletePrompt(
|
||||||
|
ctx context.Context, t *testing.T, url string, target *database.Target,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
form := `form[action$="/targets/` + target.ID + `/delete"]`
|
||||||
|
question := `Delete target "` + target.Name + `"?` + "\n\n" +
|
||||||
|
"Nothing more is delivered to it. " +
|
||||||
|
"Its past deliveries stay in the event log."
|
||||||
|
|
||||||
|
var (
|
||||||
|
asked string
|
||||||
|
sent bool
|
||||||
|
listed bool
|
||||||
|
)
|
||||||
|
|
||||||
|
require.NoError(t, chromedp.Run(
|
||||||
|
ctx, loadPage(url), chromedp.Evaluate(answerPrompt(false), nil),
|
||||||
|
))
|
||||||
|
click(ctx, t, form+" button")
|
||||||
|
require.NoError(t, chromedp.Run(
|
||||||
|
ctx,
|
||||||
|
chromedp.Evaluate(`window.asked`, &asked),
|
||||||
|
chromedp.Evaluate(`window.sent`, &sent),
|
||||||
|
))
|
||||||
|
assert.Equal(t, question, asked, "Delete does not ask about the target")
|
||||||
|
assert.False(t, sent, "answering no still sends the delete form")
|
||||||
|
|
||||||
|
require.NoError(t, chromedp.Run(
|
||||||
|
ctx, chromedp.Evaluate(answerPrompt(true), nil),
|
||||||
|
))
|
||||||
|
clickAndLoad(ctx, t, form+" button")
|
||||||
|
require.NoError(t, chromedp.Run(ctx, chromedp.Evaluate(
|
||||||
|
`document.querySelector('`+form+`') !== null`, &listed,
|
||||||
|
)))
|
||||||
|
assert.False(t, listed, "answering yes does not delete the target")
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"syscall"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -25,6 +26,11 @@ const (
|
|||||||
// its network namespace even with a published port.
|
// its network namespace even with a published port.
|
||||||
wildcardV4 = "0.0.0.0"
|
wildcardV4 = "0.0.0.0"
|
||||||
|
|
||||||
|
// loopbackV6 and wildcardV6 are the IPv6 counterparts of the two
|
||||||
|
// above.
|
||||||
|
loopbackV6 = "::1"
|
||||||
|
wildcardV6 = "::"
|
||||||
|
|
||||||
// unavailableAddr is a TEST-NET-1 address (RFC 5737). It is a
|
// unavailableAddr is a TEST-NET-1 address (RFC 5737). It is a
|
||||||
// well-formed literal that no host is assigned, so binding it
|
// well-formed literal that no host is assigned, so binding it
|
||||||
// fails with EADDRNOTAVAIL rather than succeeding somewhere
|
// fails with EADDRNOTAVAIL rather than succeeding somewhere
|
||||||
@@ -116,6 +122,24 @@ func otherLocalAddr(t *testing.T) string {
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// skipWithoutIPv6 skips the test when this host cannot bind
|
||||||
|
// loopbackV6. Without IPv6 there is no second family for a bind to
|
||||||
|
// claim or leave alone.
|
||||||
|
func skipWithoutIPv6(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var listenCfg net.ListenConfig
|
||||||
|
|
||||||
|
l, err := listenCfg.Listen(
|
||||||
|
t.Context(), "tcp6", net.JoinHostPort(loopbackV6, "0"),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Skip("host has no IPv6 loopback to bind")
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, l.Close())
|
||||||
|
}
|
||||||
|
|
||||||
// startBoundServer starts the wired app with the given bind address
|
// startBoundServer starts the wired app with the given bind address
|
||||||
// on a free port and returns that port. The app is stopped on
|
// on a free port and returns that port. The app is stopped on
|
||||||
// cleanup.
|
// cleanup.
|
||||||
@@ -292,6 +316,59 @@ func TestBindAddress_WildcardReachesOtherAddresses(t *testing.T) {
|
|||||||
requireDialable(t, net.JoinHostPort(other, strconv.Itoa(port)))
|
requireDialable(t, net.JoinHostPort(other, strconv.Itoa(port)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestBindAddress_WildcardV4IsIPv4Only pins that 0.0.0.0 answers on
|
||||||
|
// IPv4 alone, as the README says, whether it is written as 0.0.0.0 or
|
||||||
|
// as the IPv4-mapped ::ffff:0.0.0.0. Go listens on 0.0.0.0 with a
|
||||||
|
// socket that answers IPv6 too unless told otherwise, which put the
|
||||||
|
// admin UI and the receiver on every IPv6 address of a host whose
|
||||||
|
// operator had chosen an IPv4 bind.
|
||||||
|
func TestBindAddress_WildcardV4IsIPv4Only(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
skipWithoutIPv6(t)
|
||||||
|
|
||||||
|
for _, bindAddress := range []string{wildcardV4, "::ffff:0.0.0.0"} {
|
||||||
|
t.Run(bindAddress, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
port := startBoundServer(t, bindAddress)
|
||||||
|
|
||||||
|
// Positive control: the service really is up, on IPv4.
|
||||||
|
requireDialable(
|
||||||
|
t, net.JoinHostPort(loopbackV4, strconv.Itoa(port)),
|
||||||
|
)
|
||||||
|
|
||||||
|
dialer := net.Dialer{Timeout: dialTimeout}
|
||||||
|
|
||||||
|
conn, err := dialer.DialContext(
|
||||||
|
t.Context(), "tcp",
|
||||||
|
net.JoinHostPort(loopbackV6, strconv.Itoa(port)),
|
||||||
|
)
|
||||||
|
if err == nil {
|
||||||
|
_ = conn.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
require.ErrorIs(
|
||||||
|
t, err, syscall.ECONNREFUSED,
|
||||||
|
"a %s bind answered on %s", bindAddress, loopbackV6,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestBindAddress_WildcardV6ReachesBothFamilies is the counterpart:
|
||||||
|
// :: is the value that binds IPv6, and it answers IPv4 as well.
|
||||||
|
func TestBindAddress_WildcardV6ReachesBothFamilies(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
skipWithoutIPv6(t)
|
||||||
|
|
||||||
|
port := startBoundServer(t, wildcardV6)
|
||||||
|
|
||||||
|
requireDialable(t, net.JoinHostPort(loopbackV6, strconv.Itoa(port)))
|
||||||
|
requireDialable(t, net.JoinHostPort(loopbackV4, strconv.Itoa(port)))
|
||||||
|
}
|
||||||
|
|
||||||
// TestBindAddress_ServesRequestsOnConfiguredAddress proves the bound
|
// TestBindAddress_ServesRequestsOnConfiguredAddress proves the bound
|
||||||
// listener serves the application rather than merely accepting TCP,
|
// listener serves the application rather than merely accepting TCP,
|
||||||
// so a bind address that is honoured cannot be mistaken for one that
|
// so a bind address that is honoured cannot be mistaken for one that
|
||||||
|
|||||||
@@ -91,6 +91,24 @@ func NewRouterForTest(
|
|||||||
return s.router
|
return s.router
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewRouterWithSentryForTest is NewRouterForTest with the sentryhttp
|
||||||
|
// handler registered, as a set SENTRY_DSN does in production.
|
||||||
|
func NewRouterWithSentryForTest(
|
||||||
|
t *testing.T,
|
||||||
|
log *logger.Logger,
|
||||||
|
cfg *config.Config,
|
||||||
|
mw *middleware.Middleware,
|
||||||
|
h *handlers.Handlers,
|
||||||
|
) http.Handler {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
s := newServerForTest(t, log, cfg, mw, h)
|
||||||
|
s.sentryEnabled.Store(true)
|
||||||
|
s.SetupRoutes()
|
||||||
|
|
||||||
|
return s.router
|
||||||
|
}
|
||||||
|
|
||||||
// ListenAddrForTest exposes the address the HTTP listener binds for
|
// ListenAddrForTest exposes the address the HTTP listener binds for
|
||||||
// a given Config, so the rendering of host and port — IPv6
|
// a given Config, so the rendering of host and port — IPv6
|
||||||
// bracketing above all — can be pinned without standing up a
|
// bracketing above all — can be pinned without standing up a
|
||||||
|
|||||||
+30
-1
@@ -1,9 +1,11 @@
|
|||||||
package server
|
package server
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/netip"
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -39,6 +41,23 @@ func (s *Server) listenAddr() string {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// listenNetwork returns the network the HTTP listener binds on, "tcp4"
|
||||||
|
// for an IPv4 BindAddress and "tcp" for any other.
|
||||||
|
//
|
||||||
|
// On "tcp", Go binds 0.0.0.0 with a socket that answers IPv6 as well,
|
||||||
|
// exactly as it binds ::. "tcp4" keeps 0.0.0.0 to IPv4, which is what
|
||||||
|
// the README says it means. :: stays on "tcp" and answers both. An
|
||||||
|
// IPv4-mapped address such as ::ffff:0.0.0.0 is the IPv4 address it
|
||||||
|
// carries to Go's listener, so it gets "tcp4" too.
|
||||||
|
func (s *Server) listenNetwork() string {
|
||||||
|
addr, err := netip.ParseAddr(s.params.Config.BindAddress)
|
||||||
|
if err == nil && addr.Unmap().Is4() {
|
||||||
|
return "tcp4"
|
||||||
|
}
|
||||||
|
|
||||||
|
return "tcp"
|
||||||
|
}
|
||||||
|
|
||||||
// newHTTPServer builds the HTTP server for this Server's
|
// newHTTPServer builds the HTTP server for this Server's
|
||||||
// configuration.
|
// configuration.
|
||||||
//
|
//
|
||||||
@@ -67,7 +86,17 @@ func (s *Server) serveUntilShutdown() {
|
|||||||
|
|
||||||
s.log.Info("http begin listen", "listenaddr", s.httpServer.Addr)
|
s.log.Info("http begin listen", "listenaddr", s.httpServer.Addr)
|
||||||
|
|
||||||
err := s.httpServer.ListenAndServe()
|
// ListenAndServe always listens on "tcp", which would undo
|
||||||
|
// listenNetwork.
|
||||||
|
var listenCfg net.ListenConfig
|
||||||
|
|
||||||
|
listener, err := listenCfg.Listen(
|
||||||
|
context.Background(), s.listenNetwork(), s.httpServer.Addr,
|
||||||
|
)
|
||||||
|
if err == nil {
|
||||||
|
err = s.httpServer.Serve(listener)
|
||||||
|
}
|
||||||
|
|
||||||
if err != nil && !errors.Is(err, http.ErrServerClosed) {
|
if err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||||
s.log.Error("listen error", "error", err)
|
s.log.Error("listen error", "error", err)
|
||||||
s.shutdownWithFailure()
|
s.shutdownWithFailure()
|
||||||
|
|||||||
@@ -6,7 +6,8 @@ import (
|
|||||||
|
|
||||||
sentryhttp "github.com/getsentry/sentry-go/http"
|
sentryhttp "github.com/getsentry/sentry-go/http"
|
||||||
"github.com/go-chi/chi"
|
"github.com/go-chi/chi"
|
||||||
"github.com/go-chi/chi/middleware"
|
chimw "github.com/go-chi/chi/middleware"
|
||||||
|
"sneak.berlin/go/webhooker/internal/middleware"
|
||||||
"sneak.berlin/go/webhooker/static"
|
"sneak.berlin/go/webhooker/static"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -60,7 +61,7 @@ func (s *Server) SetupRoutes() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) setupGlobalMiddleware() {
|
func (s *Server) setupGlobalMiddleware() {
|
||||||
s.router.Use(middleware.RequestID)
|
s.router.Use(chimw.RequestID)
|
||||||
s.router.Use(s.mw.SecurityHeaders())
|
s.router.Use(s.mw.SecurityHeaders())
|
||||||
s.router.Use(s.mw.Logging())
|
s.router.Use(s.mw.Logging())
|
||||||
|
|
||||||
@@ -90,8 +91,8 @@ func (s *Server) setupGlobalMiddleware() {
|
|||||||
|
|
||||||
// recoverPanics installs on r the recoverer, answering a panic with
|
// recoverPanics installs on r the recoverer, answering a panic with
|
||||||
// page (a plain 500 when page is nil), and inside it the Sentry error
|
// page (a plain 500 when page is nil), and inside it the Sentry error
|
||||||
// reporting (if SENTRY_DSN is set). Repanic is true so panics still
|
// reporting and transaction naming (if SENTRY_DSN is set). Repanic is
|
||||||
// bubble up to the recoverer.
|
// true so panics still bubble up to the recoverer.
|
||||||
//
|
//
|
||||||
// Each admin page route group installs its own, with the error page,
|
// Each admin page route group installs its own, with the error page,
|
||||||
// as its first middleware. A panic there is logged, reported and
|
// as its first middleware. A panic there is logged, reported and
|
||||||
@@ -105,6 +106,11 @@ func (s *Server) recoverPanics(r chi.Router, page http.Handler) {
|
|||||||
Repanic: true,
|
Repanic: true,
|
||||||
})
|
})
|
||||||
r.Use(sentryHandler.Handle)
|
r.Use(sentryHandler.Handle)
|
||||||
|
// Installed with every sentryhttp handler rather than once
|
||||||
|
// globally: in an admin page route group, the group's own
|
||||||
|
// handler finishes the transaction as the group returns, so
|
||||||
|
// a rename further out would come too late.
|
||||||
|
r.Use(nameSentryTransaction)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -340,7 +346,7 @@ func (s *Server) setupWebhookRoutes() {
|
|||||||
// unauthenticated endpoint's body; TestReceiver_OversizeBodyRefused
|
// unauthenticated endpoint's body; TestReceiver_OversizeBodyRefused
|
||||||
// pins it.
|
// pins it.
|
||||||
s.router.With(s.mw.ReceiverRateLimit()).HandleFunc(
|
s.router.With(s.mw.ReceiverRateLimit()).HandleFunc(
|
||||||
"/h/{uuid}",
|
middleware.ReceiverRoute,
|
||||||
s.h.HandleWebhook(),
|
s.h.HandleWebhook(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package server
|
package server
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -71,10 +72,15 @@ func scrubSentryRequest(
|
|||||||
|
|
||||||
pattern := sentryRoutePattern(hint)
|
pattern := sentryRoutePattern(hint)
|
||||||
|
|
||||||
// Only transaction events carry a Transaction name, and the SDK
|
// Only transaction events carry a Transaction name. sentryhttp
|
||||||
// builds it from the concrete path too (sentryhttp.go:105 via
|
// builds it from the concrete path too (sentryhttp.go:105 via
|
||||||
// tracing.go:553). Rewritten on the same terms.
|
// tracing.go:553) and marks it SourceURL, so it is rewritten on
|
||||||
if event.Transaction != "" {
|
// the same terms. Only a name nameSentryTransaction set from the
|
||||||
|
// route pattern, which it marks SourceRoute, is kept.
|
||||||
|
namedByRoute := event.TransactionInfo != nil &&
|
||||||
|
event.TransactionInfo.Source == sentry.SourceRoute
|
||||||
|
|
||||||
|
if event.Transaction != "" && !namedByRoute {
|
||||||
event.Transaction = sentryTransactionName(
|
event.Transaction = sentryTransactionName(
|
||||||
event.Transaction, pattern,
|
event.Transaction, pattern,
|
||||||
)
|
)
|
||||||
@@ -120,7 +126,8 @@ func scrubSentryRequest(
|
|||||||
// The transaction dispatch has no such request: Span.doFinish calls
|
// The transaction dispatch has no such request: Span.doFinish calls
|
||||||
// hub.CaptureEvent (tracing.go:356), which passes a nil hint that the
|
// hub.CaptureEvent (tracing.go:356), which passes a nil hint that the
|
||||||
// client replaces with an empty one (client.go:620-622). The pattern
|
// client replaces with an empty one (client.go:620-622). The pattern
|
||||||
// is therefore always "" there, and the callers fall back.
|
// is therefore always "" there, and the callers fall back. A routed
|
||||||
|
// transaction is named by nameSentryTransaction instead.
|
||||||
func sentryRoutePattern(hint *sentry.EventHint) string {
|
func sentryRoutePattern(hint *sentry.EventHint) string {
|
||||||
if hint == nil || hint.Context == nil {
|
if hint == nil || hint.Context == nil {
|
||||||
return ""
|
return ""
|
||||||
@@ -142,6 +149,52 @@ func sentryRoutePattern(hint *sentry.EventHint) string {
|
|||||||
return rctx.RoutePattern()
|
return rctx.RoutePattern()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// nameSentryTransaction renames the request's Sentry transaction to
|
||||||
|
// "METHOD <route pattern>", such as "POST /h/{uuid}". sentryhttp names
|
||||||
|
// it after the concrete path, which scrubSentryRequest withholds;
|
||||||
|
// since Sentry groups transactions by name, every transaction in the
|
||||||
|
// service would otherwise land in one group per method, such as
|
||||||
|
// "POST /(redacted)".
|
||||||
|
//
|
||||||
|
// It has to run inside the sentryhttp handler, where the transaction
|
||||||
|
// is on the request context. It renames on the way out, once chi has
|
||||||
|
// routed the request, and does so in a defer so that a request whose
|
||||||
|
// handler panics is named too. The context is taken when the defer is
|
||||||
|
// set up, before routing, but the chi routing context on it is a
|
||||||
|
// pointer that chi fills in as it routes.
|
||||||
|
func nameSentryTransaction(next http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(
|
||||||
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
defer setSentryTransactionName(r.Context(), r.Method)
|
||||||
|
|
||||||
|
next.ServeHTTP(w, r)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// setSentryTransactionName does the rename for nameSentryTransaction.
|
||||||
|
// A path that matches nothing at the top level has no pattern; its
|
||||||
|
// name is left as the SDK set it, for scrubSentryRequest to withhold.
|
||||||
|
// Inside a route group, a path that matches no route in the group, or
|
||||||
|
// a request the group's own middleware turns away before routing, has
|
||||||
|
// the group's pattern, such as "/pages/*", and is named after it.
|
||||||
|
func setSentryTransactionName(ctx context.Context, method string) {
|
||||||
|
transaction := sentry.TransactionFromContext(ctx)
|
||||||
|
rctx := chi.RouteContext(ctx)
|
||||||
|
|
||||||
|
if transaction == nil || rctx == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
pattern := rctx.RoutePattern()
|
||||||
|
if pattern == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
transaction.Name = method + " " + pattern
|
||||||
|
transaction.Source = sentry.SourceRoute
|
||||||
|
}
|
||||||
|
|
||||||
// sentryRouteURL rebuilds an event's request URL with the route
|
// sentryRouteURL rebuilds an event's request URL with the route
|
||||||
// pattern in place of the concrete path.
|
// pattern in place of the concrete path.
|
||||||
//
|
//
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import (
|
|||||||
"github.com/go-chi/chi"
|
"github.com/go-chi/chi"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/server"
|
"sneak.berlin/go/webhooker/internal/server"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -130,11 +131,15 @@ func (c sentryCase) capture(t *testing.T) []*sentry.Event {
|
|||||||
// router mirrors the one ordering these tests depend on, over the two
|
// router mirrors the one ordering these tests depend on, over the two
|
||||||
// route patterns they need: a recovering middleware outside, then the
|
// route patterns they need: a recovering middleware outside, then the
|
||||||
// sentryhttp handler registered with Use and Repanic set, exactly as
|
// sentryhttp handler registered with Use and Repanic set, exactly as
|
||||||
// routes.go orders the two. The bare recover stands in for
|
// routes.go orders the two. It leaves out nameSentryTransaction,
|
||||||
// Middleware.Recoverer, which holds that outer slot in production; it
|
// which routes.go installs inside the sentryhttp handler, so a
|
||||||
// is here only to keep panic stacks out of the test output. That the
|
// transaction captured here keeps the SDK's name and shows what the
|
||||||
// production one really does catch what sentryhttp re-raises is
|
// hook does with a transaction that missed the rename. The bare
|
||||||
// pinned separately, by TestSentryStillSeesAPanic.
|
// recover stands in for Middleware.Recoverer, which holds that outer
|
||||||
|
// slot in production; it is here only to keep panic stacks out of the
|
||||||
|
// test output. That the production one really does catch what
|
||||||
|
// sentryhttp re-raises is pinned separately, by
|
||||||
|
// TestSentryStillSeesAPanic.
|
||||||
func (c sentryCase) router() http.Handler {
|
func (c sentryCase) router() http.Handler {
|
||||||
handler := func(_ http.ResponseWriter, r *http.Request) {
|
handler := func(_ http.ResponseWriter, r *http.Request) {
|
||||||
// This call is what drains the body tee and fills the
|
// This call is what drains the body tee and fills the
|
||||||
@@ -353,8 +358,9 @@ func TestSentryScrub_KeepsTheRoutingContext(t *testing.T) {
|
|||||||
// Span.doFinish captures with a nil hint, so BeforeSendTransaction
|
// Span.doFinish captures with a nil hint, so BeforeSendTransaction
|
||||||
// gets one with no context and no request: the route pattern is out of
|
// gets one with no context and no request: the route pattern is out of
|
||||||
// reach and both the URL and the SDK-built transaction name have to
|
// reach and both the URL and the SDK-built transaction name have to
|
||||||
// fall back. Tracing is off in this service, so no transaction event
|
// fall back. This router has no nameSentryTransaction, so this is the
|
||||||
// is produced today; the hook is a floor against that changing.
|
// floor under a transaction that missed the rename;
|
||||||
|
// TestSentryTransaction_NamedByRoute covers the routes as they ship.
|
||||||
func TestSentryScrub_RedactsTheTransactionDispatch(t *testing.T) {
|
func TestSentryScrub_RedactsTheTransactionDispatch(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -376,6 +382,83 @@ func TestSentryScrub_RedactsTheTransactionDispatch(t *testing.T) {
|
|||||||
assert.Equal(t, "POST /(redacted)", event.Transaction)
|
assert.Equal(t, "POST /(redacted)", event.Transaction)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestSentryTransaction_NamedByRoute serves requests through the real
|
||||||
|
// route tree with tracing on, which the service leaves off. A routed
|
||||||
|
// request's transaction is named after its route pattern, so Sentry
|
||||||
|
// can group transactions by route, and the entrypoint UUID in the
|
||||||
|
// receiver path appears nowhere in the event. A path that matches
|
||||||
|
// nothing at the top level has no pattern, and its name is still
|
||||||
|
// withheld; one that matches no route inside a route group is named
|
||||||
|
// after the group's pattern.
|
||||||
|
func TestSentryTransaction_NamedByRoute(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
// newTestEnv leaves the receiver rate limit at zero, which
|
||||||
|
// refuses every request before it reaches the receiver.
|
||||||
|
env := newTestEnvWithConfig(t, &config.Config{
|
||||||
|
DataDir: t.TempDir(),
|
||||||
|
Environment: config.EnvironmentDev,
|
||||||
|
ReceiverRateLimit: 10,
|
||||||
|
})
|
||||||
|
|
||||||
|
router := server.NewRouterWithSentryForTest(
|
||||||
|
t, env.log, env.cfg, env.mw, env.hnd,
|
||||||
|
)
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
method, path, want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
http.MethodPost,
|
||||||
|
"/h/" + sentryReceiverUUID,
|
||||||
|
"POST /h/{uuid}",
|
||||||
|
},
|
||||||
|
// An admin page route group, which installs its own
|
||||||
|
// sentryhttp handler.
|
||||||
|
{http.MethodGet, "/pages/login", "GET /pages/login"},
|
||||||
|
{http.MethodGet, "/pages/unrouted", "GET /pages/*"},
|
||||||
|
{
|
||||||
|
http.MethodPost,
|
||||||
|
"/h/" + sentryReceiverUUID + "/unrouted",
|
||||||
|
"POST /(redacted)",
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.want, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
transport := &captureTransport{}
|
||||||
|
|
||||||
|
opts := server.SentryClientOptionsForTest(
|
||||||
|
"https://public@sentry.invalid/1",
|
||||||
|
"webhooker-test",
|
||||||
|
)
|
||||||
|
opts.Transport = transport
|
||||||
|
opts.EnableTracing = true
|
||||||
|
opts.TracesSampleRate = 1.0
|
||||||
|
|
||||||
|
client, err := sentry.NewClient(opts)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
req := httptest.NewRequestWithContext(
|
||||||
|
sentry.SetHubOnContext(
|
||||||
|
context.Background(),
|
||||||
|
sentry.NewHub(client, sentry.NewScope()),
|
||||||
|
),
|
||||||
|
tc.method, tc.path, nil,
|
||||||
|
)
|
||||||
|
router.ServeHTTP(httptest.NewRecorder(), req)
|
||||||
|
|
||||||
|
event := onlyEvent(t, transport.events)
|
||||||
|
require.Equal(t, "transaction", event.Type)
|
||||||
|
|
||||||
|
assert.Equal(t, tc.want, event.Transaction)
|
||||||
|
assert.NotContains(
|
||||||
|
t, marshalEvent(t, event), sentryReceiverUUID,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestSentryScrub_TransactionDispatchIsUnscrubbedWithoutTheHook pins
|
// TestSentryScrub_TransactionDispatchIsUnscrubbedWithoutTheHook pins
|
||||||
// that dispatch's premise the same way, since it is the one the
|
// that dispatch's premise the same way, since it is the one the
|
||||||
// service does not exercise today.
|
// service does not exercise today.
|
||||||
|
|||||||
@@ -135,6 +135,8 @@ func New(lc fx.Lifecycle, params ServerParams) (*Server, error) {
|
|||||||
s.httpServer = s.newHTTPServer()
|
s.httpServer = s.newHTTPServer()
|
||||||
|
|
||||||
lc.Append(fx.Hook{
|
lc.Append(fx.Hook{
|
||||||
|
//nolint:contextcheck // The listener outlives this hook, whose
|
||||||
|
// context fx cancels once the app has started.
|
||||||
OnStart: func(_ context.Context) error {
|
OnStart: func(_ context.Context) error {
|
||||||
s.startupTime = time.Now()
|
s.startupTime = time.Now()
|
||||||
go s.Run()
|
go s.Run()
|
||||||
|
|||||||
@@ -441,10 +441,9 @@ func (s *Session) Regenerate(
|
|||||||
//
|
//
|
||||||
// Whichever comes first ends the session.
|
// Whichever comes first ends the session.
|
||||||
//
|
//
|
||||||
// A session that claims to be authenticated but carries no
|
// A session that claims to be authenticated but lacks the timestamp
|
||||||
// timestamps predates this check; it is treated as expired so the
|
// a deadline is measured from is treated as expired, so the user
|
||||||
// user re-authenticates rather than being granted an unbounded
|
// re-authenticates rather than being granted an unbounded session.
|
||||||
// session.
|
|
||||||
func (s *Session) expired(sess *sessions.Session) bool {
|
func (s *Session) expired(sess *sessions.Session) bool {
|
||||||
now := s.now()
|
now := s.now()
|
||||||
|
|
||||||
|
|||||||
@@ -153,17 +153,15 @@ func TestVersion_IsStableAcrossInvocations(t *testing.T) {
|
|||||||
require.Equal(t, first, second)
|
require.Equal(t, first, second)
|
||||||
}
|
}
|
||||||
|
|
||||||
// The build target is the only place composing linker flags. If a
|
// The build target is the only place that stamps the version. If a
|
||||||
// future edit drops either half, the binary silently reports "dev"
|
// future edit drops the -X flag, the binary silently reports "dev"
|
||||||
// again (the defect this package exists for) or fails to link on
|
// again, the defect this package exists for.
|
||||||
// Alpine.
|
func TestMakefile_BuildStampsTheVersion(t *testing.T) {
|
||||||
func TestMakefile_BuildComposesVersionAndExtraFlags(t *testing.T) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
makefile := read(t, makefilePath)
|
makefile := read(t, makefilePath)
|
||||||
|
|
||||||
require.Contains(t, makefile, "-X main.version=$(VERSION)")
|
require.Contains(t, makefile, "-X main.version=$(VERSION)")
|
||||||
require.Contains(t, makefile, "$(GO_LDFLAGS)")
|
|
||||||
require.Contains(t, makefile, "VERSION ?= $(shell script/version)")
|
require.Contains(t, makefile, "VERSION ?= $(shell script/version)")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -222,8 +220,8 @@ func requireMake(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Every compile in the image goes through the build target, so the
|
// The image compiles through the build target, so it carries the
|
||||||
// static relink cannot replace the flags that carry the stamp.
|
// target's version stamp.
|
||||||
func TestDockerfile_BuildsThroughTheMakeTarget(t *testing.T) {
|
func TestDockerfile_BuildsThroughTheMakeTarget(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -232,8 +230,7 @@ func TestDockerfile_BuildsThroughTheMakeTarget(t *testing.T) {
|
|||||||
require.NotContains(t, dockerfile, "go build",
|
require.NotContains(t, dockerfile, "go build",
|
||||||
"a raw go build bypasses the Makefile's -X flag")
|
"a raw go build bypasses the Makefile's -X flag")
|
||||||
require.Contains(t, dockerfile, "ARG VERSION")
|
require.Contains(t, dockerfile, "ARG VERSION")
|
||||||
require.Contains(t, dockerfile,
|
require.Contains(t, dockerfile, `RUN make build VERSION="$VERSION"`)
|
||||||
`make build VERSION="$VERSION" GO_LDFLAGS='-extldflags "-static"'`)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func read(t *testing.T, path string) string {
|
func read(t *testing.T, path string) string {
|
||||||
|
|||||||
+11
-3
@@ -1,7 +1,7 @@
|
|||||||
/*
|
/*
|
||||||
* The two shared styles for the controls a user clicks. Every page loads
|
* The two shared styles for the controls a user clicks, and the rule that
|
||||||
* this file after tailwind.css. It is plain CSS: make css does not build
|
* hides an element marked x-cloak. Every page loads this file after
|
||||||
* it.
|
* tailwind.css. It is plain CSS: make css does not build it.
|
||||||
*
|
*
|
||||||
* A button is btn-primary, btn-secondary or btn-danger, from input.css.
|
* A button is btn-primary, btn-secondary or btn-danger, from input.css.
|
||||||
* A secondary or inline action, such as Copy beside an entrypoint URL or
|
* A secondary or inline action, such as Copy beside an entrypoint URL or
|
||||||
@@ -46,3 +46,11 @@
|
|||||||
outline-offset: 2px;
|
outline-offset: 2px;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Alpine.js removes x-cloak from an element once it has started, so
|
||||||
|
* something Alpine hides is not seen on the page while it loads.
|
||||||
|
*/
|
||||||
|
[x-cloak] {
|
||||||
|
display: none !important;
|
||||||
|
}
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
@@ -58,6 +58,18 @@
|
|||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
|
|
||||||
|
// Asks before a form with a data-confirm attribute is sent, such as a
|
||||||
|
// delete form on the webhook page. The browser's own prompt shows the
|
||||||
|
// attribute's text, and the form is sent only if the user accepts.
|
||||||
|
document.addEventListener("submit", function (event) {
|
||||||
|
"use strict";
|
||||||
|
|
||||||
|
const message = event.target.getAttribute("data-confirm");
|
||||||
|
if (message !== null && !window.confirm(message)) {
|
||||||
|
event.preventDefault();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
// Alpine.js components.
|
// Alpine.js components.
|
||||||
//
|
//
|
||||||
// The pages' Content-Security-Policy forbids eval, so the UI loads
|
// The pages' Content-Security-Policy forbids eval, so the UI loads
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
{{if .Binary}}
|
{{if .Binary}}
|
||||||
<p class="text-xs text-gray-500">This body is binary ({{.Size}} bytes) and is not shown. <a href="{{.EventURL}}/body" class="btn-small">Download the body</a></p>
|
<p class="text-xs text-gray-500">This body is binary ({{.Size}} bytes) and is not shown. <a href="{{.EventURL}}/body" class="btn-small">Download the body</a></p>
|
||||||
{{else if .Text}}
|
{{else if .Text}}
|
||||||
<pre class="rounded-md border border-gray-200 bg-white p-2 text-xs text-gray-700 overflow-x-auto whitespace-pre-wrap break-all"{{if .Scroll}} style="max-height: 32rem; overflow-y: auto"{{end}}>{{.Text}}</pre>
|
<pre class="rounded-md border border-gray-200 bg-white p-2 text-xs text-gray-700 overflow-x-auto whitespace-pre-wrap break-all{{if .Scroll}} max-h-[32rem] overflow-y-auto{{end}}">{{.Text}}</pre>
|
||||||
{{if .Cut}}
|
{{if .Cut}}
|
||||||
<p class="mt-2 text-xs text-gray-500">Showing the first {{.ShownBytes}} of {{.Size}} bytes, unformatted. <a href="{{.EventURL}}" class="btn-small">Show the whole body</a> <a href="{{.EventURL}}/body" class="btn-small">Download the body</a></p>
|
<p class="mt-2 text-xs text-gray-500">Showing the first {{.ShownBytes}} of {{.Size}} bytes, unformatted. <a href="{{.EventURL}}" class="btn-small">Show the whole body</a> <a href="{{.EventURL}}/body" class="btn-small">Download the body</a></p>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|||||||
@@ -4,6 +4,5 @@
|
|||||||
<title>{{block "title" .}}Webhooker{{end}}</title>
|
<title>{{block "title" .}}Webhooker{{end}}</title>
|
||||||
<link rel="stylesheet" href="/s/css/tailwind.css">
|
<link rel="stylesheet" href="/s/css/tailwind.css">
|
||||||
<link rel="stylesheet" href="/s/css/style.css">
|
<link rel="stylesheet" href="/s/css/style.css">
|
||||||
<style>[x-cloak] { display: none !important; }</style>
|
|
||||||
{{block "head" .}}{{end}}
|
{{block "head" .}}{{end}}
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|||||||
@@ -13,13 +13,11 @@
|
|||||||
<div class="p-4">
|
<div class="p-4">
|
||||||
<!-- A value too wide to sit beside its name moves to the
|
<!-- A value too wide to sit beside its name moves to the
|
||||||
next line, where a list breaks only at the spaces
|
next line, where a list breaks only at the spaces
|
||||||
between its entries. overflow-wrap: anywhere breaks
|
between its entries. wrap-anywhere breaks inside a
|
||||||
inside a value only when it alone is wider than the
|
value only when it alone is wider than the line. -->
|
||||||
line; an inline style, because the committed
|
|
||||||
tailwind.css has no class for it. -->
|
|
||||||
<div class="flex flex-wrap justify-between items-start gap-4">
|
<div class="flex flex-wrap justify-between items-start gap-4">
|
||||||
<code class="text-sm font-medium text-gray-900">{{.Name}}</code>
|
<code class="text-sm font-medium text-gray-900">{{.Name}}</code>
|
||||||
<code class="text-sm text-gray-900" style="overflow-wrap: anywhere">{{.Value}}</code>
|
<code class="text-sm text-gray-900 wrap-anywhere">{{.Value}}</code>
|
||||||
</div>
|
</div>
|
||||||
<p class="text-sm text-gray-500 mt-1">{{.Description}}</p>
|
<p class="text-sm text-gray-500 mt-1">{{.Description}}</p>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -5,12 +5,10 @@
|
|||||||
{{define "content"}}
|
{{define "content"}}
|
||||||
<!-- 108rem, half again the 72rem (max-w-6xl) of the webhook list, the
|
<!-- 108rem, half again the 72rem (max-w-6xl) of the webhook list, the
|
||||||
event log, the navbar and the footer, so an entrypoint URL fits on
|
event log, the navbar and the footer, so an entrypoint URL fits on
|
||||||
one line. An inline style, because the committed tailwind.css has
|
one line. wrap-anywhere goes only on names and descriptions: a
|
||||||
no class this wide. wrap-anywhere goes only on names and
|
row too wide for a phone must still run past the edge, where the
|
||||||
descriptions: a row too wide for a phone must still run past the
|
browser test sees it, rather than break its controls mid-word. -->
|
||||||
edge, where the browser test sees it, rather than break its
|
<div class="max-w-[108rem] mx-auto px-6 py-8">
|
||||||
controls mid-word. -->
|
|
||||||
<div class="mx-auto px-6 py-8" style="max-width: 108rem">
|
|
||||||
<div class="mb-6">
|
<div class="mb-6">
|
||||||
<a href="/hooks" class="btn-small">← Back to webhooks</a>
|
<a href="/hooks" class="btn-small">← Back to webhooks</a>
|
||||||
<div class="flex flex-wrap justify-between items-center gap-2 mt-2">
|
<div class="flex flex-wrap justify-between items-center gap-2 mt-2">
|
||||||
@@ -23,11 +21,12 @@
|
|||||||
<div class="flex gap-2">
|
<div class="flex gap-2">
|
||||||
<a href="/hook/{{.Webhook.ID}}/events" class="btn-secondary">Full Event Log</a>
|
<a href="/hook/{{.Webhook.ID}}/events" class="btn-secondary">Full Event Log</a>
|
||||||
<a href="/hook/{{.Webhook.ID}}/edit" class="btn-secondary">Edit</a>
|
<a href="/hook/{{.Webhook.ID}}/edit" class="btn-secondary">Edit</a>
|
||||||
<!-- The delete prompts are the browser's own, so they
|
<!-- Before a delete form is sent, static/js/app.js
|
||||||
work without the page's scripts. The template
|
asks in the browser's own prompt, with the form's
|
||||||
escapes each name for the script, so a quote or a
|
data-confirm text; is a line break. The
|
||||||
backslash in it shows as typed. -->
|
template escapes each name for the attribute, so a
|
||||||
<form method="POST" action="/hook/{{.Webhook.ID}}/delete" onsubmit="return confirm('Delete webhook "{{.Webhook.Name}}"?\n\nThis deletes its stored events{{with .Stats}} ({{.WithinRetention.Events}}){{end}} and their deliveries. Any archive files it wrote are kept.')">
|
quote in it shows as typed. -->
|
||||||
|
<form method="POST" action="/hook/{{.Webhook.ID}}/delete" data-confirm="Delete webhook "{{.Webhook.Name}}"? This deletes its stored events{{with .Stats}} ({{.WithinRetention.Events}}){{end}} and their deliveries. Any archive files it wrote are kept.">
|
||||||
<input type="hidden" name="csrf_token" value="{{.CSRFToken}}">
|
<input type="hidden" name="csrf_token" value="{{.CSRFToken}}">
|
||||||
<button type="submit" class="btn-danger">Delete</button>
|
<button type="submit" class="btn-danger">Delete</button>
|
||||||
</form>
|
</form>
|
||||||
@@ -90,7 +89,7 @@
|
|||||||
{{if .Active}}Deactivate{{else}}Activate{{end}}
|
{{if .Active}}Deactivate{{else}}Activate{{end}}
|
||||||
</button>
|
</button>
|
||||||
</form>
|
</form>
|
||||||
<form method="POST" action="/hook/{{$.Webhook.ID}}/entrypoints/{{.ID}}/delete" onsubmit="return confirm('Delete entrypoint "{{if .Description}}{{.Description}}{{else}}{{$.BaseURL}}/h/{{.Path}}{{end}}"?\n\nSenders using its URL get an error from now on, and the URL cannot be restored.')" class="inline">
|
<form method="POST" action="/hook/{{$.Webhook.ID}}/entrypoints/{{.ID}}/delete" data-confirm="Delete entrypoint "{{if .Description}}{{.Description}}{{else}}{{$.BaseURL}}/h/{{.Path}}{{end}}"? Senders using its URL get an error from now on, and the URL cannot be restored." class="inline">
|
||||||
<input type="hidden" name="csrf_token" value="{{$.CSRFToken}}">
|
<input type="hidden" name="csrf_token" value="{{$.CSRFToken}}">
|
||||||
<button type="submit" class="btn-small text-red-600" title="Delete">Delete</button>
|
<button type="submit" class="btn-small text-red-600" title="Delete">Delete</button>
|
||||||
</form>
|
</form>
|
||||||
@@ -278,7 +277,7 @@
|
|||||||
{{if .Active}}Deactivate{{else}}Activate{{end}}
|
{{if .Active}}Deactivate{{else}}Activate{{end}}
|
||||||
</button>
|
</button>
|
||||||
</form>
|
</form>
|
||||||
<form method="POST" action="/hook/{{$.Webhook.ID}}/targets/{{.ID}}/delete" onsubmit="return confirm('Delete target "{{.Name}}"?\n\nNothing more is delivered to it. Its past deliveries stay in the event log.')" class="inline">
|
<form method="POST" action="/hook/{{$.Webhook.ID}}/targets/{{.ID}}/delete" data-confirm="Delete target "{{.Name}}"? Nothing more is delivered to it. Its past deliveries stay in the event log." class="inline">
|
||||||
<input type="hidden" name="csrf_token" value="{{$.CSRFToken}}">
|
<input type="hidden" name="csrf_token" value="{{$.CSRFToken}}">
|
||||||
<button type="submit" class="btn-small text-red-600" title="Delete">Delete</button>
|
<button type="submit" class="btn-small text-red-600" title="Delete">Delete</button>
|
||||||
</form>
|
</form>
|
||||||
|
|||||||
Reference in New Issue
Block a user