diff --git a/.dockerignore b/.dockerignore index a01a41c..5565f6c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,13 @@ +# .ci-fingerprint is deliberately NOT excluded: it is the CI cache barrier +# that keeps the check stages from replaying a cached pass. See the lint +# stage of the Dockerfile. .git/ bin/ +# Third-party browser assets are fetched and hash-verified inside the build by +# script/fetch-assets. Excluding any host copy keeps a developer's working tree +# from supplying the bytes that get shipped. The script and its +# static/vendor.sha256 manifest stay in the context. +static/js/alpine.min.js *.md LICENSE .editorconfig diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 5d9892e..562ec31 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -11,5 +11,33 @@ jobs: steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 2024-10-23 + with: + # The fingerprint step below needs history to find the last commit + # that touched the Docker build context, and the superseded-status + # step needs it to walk ancestors (it aborts on a shallow clone). + fetch-depth: 0 + + - name: Mark superseded run statuses + # Gitea cancels the in-flight run when another commit is pushed to the + # same branch and records the cancellation as `failure`, so a commit + # that was never tested reads as a test result. The script rewrites + # those statuses to say what happened. See its header for why the + # state stays `failure` and not `skipped`. + env: + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + run: script/ci-mark-superseded + + - name: Fingerprint the build context + # `.dockerignore` keeps docs out of the build context, so a docs-only + # commit legitimately replays the whole image from cache and stays + # cheap. Every other commit writes a new fingerprint into the context, + # which invalidates the `COPY . .` layer of both check stages: a + # commit that was never linted, formatted-checked, tested and built + # cannot report success from cache. + run: | + set -eu + fp="$(git log -1 --format=%H -- . ':!*.md' ':!LICENSE' ':!.editorconfig')" + printf '%s\n' "${fp:-$GITHUB_SHA}" > .ci-fingerprint + - name: Build Docker image (runs make check) run: script/cibuild diff --git a/.gitignore b/.gitignore index d615704..50cd133 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,12 @@ data/ # Temporary files tmp/ -temp/ \ No newline at end of file +temp/ + +# CI cache barrier, written into the build context by the check workflow +.ci-fingerprint + +# Third-party browser assets, fetched and hash-verified by +# script/fetch-assets against static/vendor.sha256. Not committed: +# REPO_POLICIES.md forbids minified bundles in version control. +/static/js/alpine.min.js \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 8d62594..bb47620 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,12 +12,21 @@ WORKDIR /src COPY go.mod go.sum ./ RUN go mod download -# Copy source code +# Copy source code. In CI the context also carries .ci-fingerprint, whose +# value changes with every commit that touches the build context (see +# .gitea/workflows/check.yml). That invalidates this layer, so the checks +# below cannot report success by replaying a cached pass. Do not add it to +# .dockerignore. COPY . . -# Run formatting check and linter +# Run formatting check and linter. golangci-lint is invoked directly rather +# than through `make lint`: this stage is already the pinned linter image, and +# script/lint is a wrapper that builds Dockerfile.lint, so calling it here +# would need a docker daemon inside the build. Keep these steps in step with +# Dockerfile.lint, including --network=none (see its header for why). RUN make fmt-check -RUN make lint +RUN --network=none golangci-lint config verify --config .golangci.yml +RUN --network=none golangci-lint run --config .golangci.yml ./... # Build stage # golang:1.26.1-bookworm (Debian-based), 2026-03-17 @@ -28,7 +37,9 @@ FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a349228 # Depend on lint stage passing COPY --from=lint /src/go.sum /dev/null -RUN apt-get update && apt-get install -y --no-install-recommends make && rm -rf /var/lib/apt/lists/* +# jq is a runtime dependency of script/ci-mark-superseded, which the test +# suite executes. +RUN apt-get update && apt-get install -y --no-install-recommends make curl ca-certificates jq && rm -rf /var/lib/apt/lists/* WORKDIR /build @@ -36,19 +47,42 @@ WORKDIR /build COPY go.mod go.sum ./ RUN go mod download -# Copy source code +# Copy source code, including the .ci-fingerprint cache barrier described in +# the lint stage above. COPY . . +# Fetch the third-party browser assets the UI serves. They are not committed +# (REPO_POLICIES.md forbids minified bundles in version control) and +# .dockerignore keeps any host copy out of the build context, so this step is +# the only way they enter the image. Each download is checked against a +# hardcoded sha256 and the build fails on mismatch; make test re-checks the +# hashes against the bytes go:embed actually put in the binary. +RUN script/fetch-assets + # Run tests and build RUN make test -RUN make build + +# Version stamped into the binary. .dockerignore excludes .git/, so +# nothing in this stage can derive it: script/docker resolves it on the +# host and passes it in. The default is what a bare `docker build .` +# with no --build-arg gets, and it names no tag the tree may not be at. +# +# Declared here, below the test and asset steps, so a changed version +# does not invalidate their cached layers. +ARG VERSION=unknown + +RUN make build VERSION="$VERSION" # Rebuild with static linking for Alpine runtime. # make build already verified compilation. # The CGO binary from `make build` is dynamically linked against glibc, # which doesn't exist on Alpine (musl). Rebuild with static linking so # the binary runs on Alpine without glibc. -RUN CGO_ENABLED=1 go build -ldflags '-extldflags "-static"' -o bin/webhooker ./cmd/webhooker +# +# The static flags go in through GO_LDFLAGS rather than a -ldflags of +# their own: the build target composes them with the -X that stamps the +# version, so this relink cannot silently drop the stamp. +RUN CGO_ENABLED=1 make build VERSION="$VERSION" GO_LDFLAGS='-extldflags "-static"' # Runtime stage # alpine:3.21, 2026-03-17 @@ -75,6 +109,18 @@ USER webhooker EXPOSE 8080 +# The binary defaults BIND_ADDRESS to 127.0.0.1, which is right for a +# bare host: the cleartext listener serves the admin UI and the +# unauthenticated receiver, so it must not appear on every interface +# of a machine that configured nothing. A container is the other case. +# Its network namespace is already the isolation boundary, so binding +# every address inside it exposes nothing; what decides exposure is +# the publish flag, and `-p 127.0.0.1:8080:8080` is the operator's +# control there. Shipping the image on loopback would buy no security +# and would make the process unreachable through its own published +# port. +ENV BIND_ADDRESS=0.0.0.0 + HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ CMD wget --no-verbose --tries=1 --spider http://localhost:8080/.well-known/healthcheck || exit 1 diff --git a/Dockerfile.lint b/Dockerfile.lint new file mode 100644 index 0000000..9a9af51 --- /dev/null +++ b/Dockerfile.lint @@ -0,0 +1,37 @@ +# Lint-only image, built by script/lint. golangci-lint is never installed on +# the host: the repo is COPYed into the pinned image and linted as a build +# step, so a successful build IS a clean lint. This works even when the docker +# daemon is remote and bind mounts are impossible. +# +# script/lint passes --no-cache-filter=lint. Without it an unchanged tree +# replays the lint stage from cache and the build succeeds in under a second +# having run no linter at all. Do not drop that flag. +# +# The lint steps run with --network=none. `golangci-lint config verify` is +# documented as fetching its JSON schema over HTTPS, which would make linting +# depend on an unpinned remote artifact; this pinned image resolves the schema +# without any network, and --network=none enforces that rather than trusting +# it. It also proves no linter reaches out at analysis time. If a future image +# bump makes either step need the network, this build fails loudly instead of +# quietly acquiring an unpinned dependency. + +# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-07 +# Using Debian-based image because mattn/go-sqlite3 (CGO) does not +# compile on Alpine musl (off64_t is a glibc type). +FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS deps + +WORKDIR /src + +# Copy go mod files first for better layer caching. This stage is cacheable; +# only the lint stage below is forced to re-execute. +COPY go.mod go.sum ./ +RUN go mod download + +FROM deps AS lint + +COPY . . + +# `run` silently ignores config keys it does not recognize, so a typo would +# disable a setting without a word. `config verify` is what catches that. +RUN --network=none golangci-lint config verify --config .golangci.yml +RUN --network=none golangci-lint run --config .golangci.yml ./... diff --git a/Makefile b/Makefile index 81ee568..ceed419 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,35 @@ -.PHONY: bootstrap setup test lint fmt fmt-check check build run dev deps docker clean hooks css +.PHONY: bootstrap setup assets test lint fmt fmt-check check build run dev deps docker clean hooks css version # Default target .DEFAULT_GOAL := check +# Version stamped into the binary. Derived from git by script/version; +# override it (`make build VERSION=v1.2.3`) where git metadata is +# unavailable, which is how the Dockerfile passes its build arg in. +VERSION ?= $(shell script/version) + +# An empty override (`make build VERSION=`, or a `--build-arg VERSION=` +# landing on the Dockerfile's `make build VERSION="$VERSION"`) means unset, +# exactly as it does in script/version -- stamping "" would leave the binary +# reporting no version and the footer back on its "dev" fallback. `override` +# is required: a plain assignment loses to the command-line definition it +# exists to correct. +override VERSION := $(or $(strip $(VERSION)),$(shell script/version)) + +# Extra linker flags for the build target. The static relink in the +# Dockerfile adds -extldflags here rather than passing its own -ldflags, +# so composing flags cannot drop the version stamp. +GO_LDFLAGS ?= + bootstrap: @script/bootstrap setup: @script/setup +assets: + @script/fetch-assets + test: @script/test @@ -25,7 +46,7 @@ check: @script/check build: - go build -o bin/webhooker ./cmd/webhooker + go build -ldflags '$(strip -X main.version=$(VERSION) $(GO_LDFLAGS))' -o bin/webhooker ./cmd/webhooker run: build ./bin/webhooker @@ -37,6 +58,9 @@ deps: go mod download go mod tidy +version: + @echo $(VERSION) + docker: @script/docker diff --git a/README.md b/README.md index 1514b61..c66e5ca 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,16 @@ with retry support, logging, and observability. Category: infrastructure ### Prerequisites -- Go 1.26+ -- golangci-lint v2.11+ -- Docker (for containerized deployment) +- Go 1.26.1+ (the version in `go.mod`) +- Docker (for linting, for the test stage of the CI gate, and for + containerized deployment) +- `curl`, used by `script/fetch-assets` to download the third-party + browser assets, which are not committed (`make bootstrap` installs + it if missing) + +golangci-lint is not a prerequisite and must not be installed on the +host: `script/bootstrap` does not install it, and `make lint` runs the +digest-pinned linter image via `Dockerfile.lint`. ### Quick Start @@ -22,14 +29,18 @@ with retry support, logging, and observability. Category: infrastructure git clone https://git.eeqj.de/sneak/webhooker.git cd webhooker -# Install Go dependencies -make deps +# Install Go dependencies and the third-party browser assets. +# `make deps` alone is not enough: it only runs go mod download/tidy, +# and the checks below need the fetched assets. +make bootstrap -# Run all checks (format, lint, test, build) +# Run all checks (test, lint, format check) make check -# Run in development mode (uses SQLite in current directory) -make dev +# Run in development mode. DATA_DIR defaults to /var/lib/webhooker in +# every environment, so set it (in .env or the shell) to a writable +# directory when running from a clone. +DATA_DIR=./data make dev # Build Docker image make docker @@ -40,44 +51,75 @@ make docker ```bash make bootstrap # Install all dependencies (idempotent) make setup # Bootstrap + install git pre-commit hook +make assets # Fetch + verify third-party browser assets make fmt # Format code (gofmt + goimports) -make lint # Run golangci-lint +make fmt-check # Fail if gofmt would change anything (writes nothing) +make lint # Run golangci-lint in Docker (Dockerfile.lint) make test # Run tests with race detection make check # test + lint + fmt-check (CI gate) -make build # Build binary to bin/webhooker +make build # Build binary to bin/webhooker (version-stamped) +make version # Print the version this checkout would stamp +make run # build, then run ./bin/webhooker make dev # go run ./cmd/webhooker +make deps # go mod download + go mod tidy make docker # Build Docker image make hooks # Install git pre-commit hook that runs script/precommit +make css # Regenerate static/css/tailwind.css (needs tailwindcss) +make clean # Remove bin/ ``` ### Configuration All configuration is via environment variables. For local development, -you can place variables in a `.env` file in the project root (loaded -automatically via `godotenv/autoload`). +you can place variables in a `.env` file in the process working +directory, read once at startup before anything else looks at the +environment. + +The file is optional and having none is the normal case for a +deployment. A file that is there but cannot be parsed aborts startup +with a message naming it, because a single malformed line makes none +of the file apply: every variable in it silently reverts to its +default, which is exactly the failure [Invalid values abort +startup](#invalid-values-abort-startup) exists to prevent, for all of +them at once. A variable already present in the real environment wins +over the file's value for the same name. The environment is selected by setting `WEBHOOKER_ENVIRONMENT` to `dev` -or `prod` (default: `dev`). The setting controls several behaviors: +or `prod` (default: `dev`). The setting controls exactly one behavior: -| Behavior | `dev` | `prod` | -| --------------------- | -------------------------------- | ------------------------------- | -| CORS | Allows any origin (`*`) | Disabled (no-op) | -| Session cookie Secure | `false` (works over plain HTTP) | `true` (requires HTTPS) | +| Behavior | `dev` | `prod` | +| -------- | ----------------------- | ---------------- | +| CORS | Allows any origin (`*`) | Disabled (no-op) | -The CSRF cookie's `Secure` flag and Origin/Referer validation mode are -determined per-request based on the actual transport protocol, not the -environment setting. The middleware checks `r.TLS` (direct TLS) and the -`X-Forwarded-Proto` header (TLS-terminating reverse proxy) to decide: +The environment setting does **not** control cookie security. Both the +session cookie and the CSRF cookie get their `Secure` flag, and the +CSRF middleware its Origin/Referer validation mode, from the transport +of each individual request, decided by one predicate — +`internal/reqtls.IsTLS`. It reports TLS for a direct TLS connection +(`r.TLS`) or for a TLS-terminating reverse proxy that reports one in +`X-Forwarded-Proto`: - **Direct TLS or `X-Forwarded-Proto: https`**: Secure cookies, strict Origin/Referer validation. - **Plaintext HTTP**: Non-Secure cookies, relaxed Origin/Referer checks (token validation still enforced). -This means CSRF protection works correctly in all deployment scenarios: -behind a TLS-terminating reverse proxy, with direct TLS, or over plain -HTTP during development. When running behind a reverse proxy, ensure it -sets the `X-Forwarded-Proto: https` header. +The `X-Forwarded-Proto` value is matched case-insensitively on its +first comma-separated element, trimmed, so `HTTPS` and the appended +chains a proxy behind another proxy emits (`https, http`) are all read +as TLS. + +This means both cookie security and CSRF protection work correctly in +all deployment scenarios: behind a TLS-terminating reverse proxy, with +direct TLS, or over plain HTTP during development — a plain-HTTP local +run gets non-`Secure` cookies and remains usable, and a proxied +deployment gets `Secure` ones without the operator setting anything. +When running behind a reverse proxy, ensure it sets the +`X-Forwarded-Proto: https` header. Unlike `X-Forwarded-For`, this +header is read from any peer and is **not** gated by +`TRUSTED_PROXIES`; a correctly configured proxy overwrites whatever a +client sent. On a listener exposed directly to clients, any client can +assert it, so do not run one without a proxy in front. All other differences (log format, security headers, etc.) are independent of the environment setting — log format is determined by @@ -87,62 +129,1092 @@ TTY detection, and security headers are always applied. | ----------------------- | ----------------------------------- | -------- | | `WEBHOOKER_ENVIRONMENT` | `dev` or `prod` | `dev` | | `PORT` | HTTP listen port | `8080` | +| `BIND_ADDRESS` | IP address the HTTP listener binds. Loopback by default, so the cleartext listener is not published on every interface. The Docker image ships `0.0.0.0` instead. See [Bind address](#bind-address) | `127.0.0.1` (image: `0.0.0.0`) | | `DATA_DIR` | Directory for all SQLite databases | `/var/lib/webhooker` | | `DEBUG` | Enable debug logging | `false` | -| `METRICS_USERNAME` | Basic auth username for `/metrics` | `""` | -| `METRICS_PASSWORD` | Basic auth password for `/metrics` | `""` | -| `SENTRY_DSN` | Sentry error reporting DSN | `""` | +| `MAINTENANCE_MODE` | Report `maintenanceMode: true` in the healthcheck JSON. It does not change how any request is served — no maintenance page exists | `false` | +| `METRICS_USERNAME` | Basic auth username for `/metrics`. Must be set together with `METRICS_PASSWORD`; one without the other fails startup | `""` | +| `METRICS_PASSWORD` | Basic auth password for `/metrics`. Must be set together with `METRICS_USERNAME`; one without the other fails startup | `""` | +| `SENTRY_DSN` | Sentry error reporting DSN. Unset leaves error reporting off; a value the Sentry SDK cannot parse fails startup rather than serving with reporting silently off | `""` | +| `RETENTION_SWEEP_INTERVAL` | How often the retention reaper and archive sweeper run (Go duration, must be positive) | `1h` | +| `SESSION_IDLE_TIMEOUT` | Idle session timeout (Go duration) | `24h` | +| `RECEIVER_RATE_LIMIT` | Receiver requests/minute per IP per entrypoint (10x that per IP across the route) | `120` | +| `TRUSTED_PROXIES` | CIDRs whose forwarded headers are trusted (unset: all clients behind a proxy share one rate-limit bucket; a correct login password is never throttled either way) | `""` (none) | +| `ALLOWED_EGRESS_CIDRS` | CIDRs that delivery targets may reach despite the SSRF blocklist. Read [Allowing egress to your own network](#allowing-egress-to-your-own-network) before setting it | `""` (none) | + +#### Allowing egress to your own network + +By default every delivery target must resolve to a public address. The +private and reserved ranges — RFC 1918, loopback, CGNAT, link-local and +the rest — are refused, which stops a target from being used to make +webhooker probe the network it sits in. + +That default is also inconvenient for the thing webhooker is mostly +for: taking a public webhook and forwarding it to something on your own +network. A container on the same Docker network, a box on `10.x`, a +service on `127.0.0.1` — all refused, until you name them. + +`ALLOWED_EGRESS_CIDRS` is a comma-separated list of CIDR blocks (a bare +address such as `10.0.0.7` is accepted and treated as a single host), +for example `10.0.0.0/8, 172.17.0.0/16`. Addresses inside those blocks +become valid delivery destinations. Everything outside them keeps the +default answer, so this only ever adds destinations — it never removes +any, and it cannot narrow what was already reachable. + +**The risk, plainly.** Each block you list is a network that anyone who +can create a delivery target can now make this process issue requests +into, and read the response body back out of via the delivery log. That +is server-side request forgery, deliberately enabled and scoped by you. +A webhooker admin account is therefore as trusted as the narrowest +thing on those networks: an unauthenticated admin panel, a database +listening without a password, or an internal API that trusts its +network position is reachable through it. List the smallest blocks that +cover the destinations you actually deliver to — prefer +`10.1.2.3/32` over `10.0.0.0/8` — and never list a block wider than the +network you are willing to expose. + +Listing `0.0.0.0/0` or `::/0` opens **every** other private and +reserved range at once — loopback, RFC 1918, CGNAT, ULA, the lot. It is +a functional off switch for everything except the addresses listed as +unconditionally blocked below, and it makes any delivery target a probe +into your entire network and this host's own loopback services. Do not +list it. + +Two things this setting cannot do: + +- **It cannot turn the guard off.** There is no boolean, and no value + that disables SSRF protection wholesale. The guard is always on and + the list is always an allowlist; an empty list (the default) means + every private and reserved range stays refused. Note that + `0.0.0.0/0` gets you most of the way there anyway, per above. +- **It cannot open link-local, or a cloud metadata endpoint that + discloses credentials or user data.** An address is on the list below + when both of these hold: the provider fixes it, so it cannot collide + with anything you run; and reaching it hands out credentials, user + data or bootstrap material. Those stay blocked no matter what you + list, including when you list them outright or list a supernet such + as `0.0.0.0/0`, `::/0`, `fd00::/8` or `100.64.0.0/10`. Treat this as + best effort rather than a guarantee — it is a hand-maintained list + and the caveat below the table applies: + + | Blocked unconditionally | What it is | + | ----------------------- | ---------- | + | `169.254.0.0/16` | IPv4 link-local, carrying `169.254.169.254` (AWS, Azure, DigitalOcean, Hetzner, OpenStack and others — not Alibaba, which uses `100.100.100.200` below) | + | `fe80::/10` | IPv6 link-local | + | `fd00:ec2::254/128` | AWS IPv6 IMDS | + | `fd00:ec2::23/128` | AWS EKS Pod Identity Agent | + | `fd20:ce::254/128` | GCP metadata for IPv6-only instances | + | `fd00:c1::a9fe:a9fe/128` | Oracle OCI IMDS over IPv6 | + | `fd00:42::42/128` | Scaleway metadata over IPv6 | + | `fd00:a9fe:a9fe::1/128` | Linode/Akamai metadata over IPv6 | + | `100.100.100.200/32` | Alibaba Cloud metadata, inside CGNAT | + | `192.0.0.192/32` | Oracle Cloud Classic metadata | + | `::a9fe:a9fe/128` | `169.254.169.254` as an IPv4-compatible IPv6 address | + | `64:ff9b::a9fe:a9fe/128` | `169.254.169.254` behind the NAT64 well-known prefix | + + The IPv4-mapped form `::ffff:169.254.169.254` is covered by the + `169.254.0.0/16` entry. Reaching any of these is credential or + user-data theft rather than delivery to an internal service. Every + entry outside the two link-local blocks is a single address, so + blocking it costs you nothing else on the network around it. + + The six ULA entries, all inside `fd00::/8`, are why this matters in + practice: `fd00::/8` is an ordinary block to allowlist for your own + IPv6 network, and without those host routes that one line would hand + out cloud credentials on five providers at once. There is only one + `/8` involved — `fd20:ce::254` masks into `fd00::/8` as well — and + the six endpoints are five providers because AWS appears twice, IMDS + and EKS Pod Identity. Several of them are described as "link-local" — + or even "localhost" — in their own vendor's documentation, but they + are ULAs and `fe80::/10` does not cover them. + + Every entry above is reserved space. All but the last two are already + refused with no allowlist set, and listing them here is only what + stops an allowlist from reopening them; the last two are the alternate + encodings, which the default blocklist does not match. A publicly + routable metadata address is not listed here, because nothing on this + list can be reopened and blocking one that way would leave you no + escape hatch at all. + + This list is not exhaustive of every cloud's metadata address — if + yours is not here, do not allowlist the block that contains it. + +The list is applied at one place in the code, which both target +creation and delivery consult, so a URL that the target form accepts is +one that delivery will actually attempt — the two cannot disagree. +Delivery re-resolves and re-checks the destination at dial time, so a +hostname that resolves to an allowed address during validation and a +different one later (DNS rebinding) is still refused unless the new +address is also allowed. + +A set but unparseable value aborts startup. When the list is non-empty +webhooker logs it at startup, blocks and all, so the hole is visible in +the log of any deployment that has one. + +#### Bind address + +`BIND_ADDRESS` is the IP address the HTTP listener binds. The binary +defaults to `127.0.0.1`, so a bare webhooker is reachable only from the +host it runs on. The Docker image ships `ENV BIND_ADDRESS=0.0.0.0` +instead — see below for why the two differ. + +That listener speaks **cleartext**, and it serves both the admin UI and +the unauthenticated webhook receiver. webhooker terminates no TLS +itself; a production deployment puts a reverse proxy in front of it +(see +[Deployment behind a reverse proxy](#deployment-behind-a-reverse-proxy)), +and the proxy reaches it over loopback. A default that bound every +interface would leave that cleartext port answering the internet +alongside the proxy — the admin login form and the receiver, in the +clear, on a port nobody chose to publish. Reaching webhooker from +another host is therefore something you configure, not something you +get by default. + +**In a container the answer is `0.0.0.0`, which is why the image ships +that.** A container's network namespace is already the boundary the +loopback default is reaching for: nothing outside the container gets to +`0.0.0.0:8080` because of the namespace, whatever the process bound. +Exposure is decided at the publish flag instead — `-p +127.0.0.1:8080:8080` rather than `-p 8080:8080` — which is the +operator's to choose and is what +[Running with Docker](#running-with-docker) shows. A loopback bind +inside a container buys nothing and makes the process unreachable +through its own published port. + +The value must be an IP address literal: + +- `127.0.0.1` — loopback only (the binary's default). Use this with a + reverse proxy on the same host. +- `0.0.0.0` — every IPv4 address. The image's default; on a bare host, + only behind a firewall on the port. +- `::` — every address, IPv6 and (on Linux, with the default + `net.ipv6.bindv6only=0`) IPv4 as well. +- A specific address such as `10.0.0.5` — that interface only. + +An **empty** value is treated as unset, as everywhere else here, and +takes the default. In a container that matters: `BIND_ADDRESS=` throws +away the image's `0.0.0.0` and falls back to the binary's +`127.0.0.1`, which is the one quiet failure this setting has — see +[Running with Docker](#running-with-docker). + +Hostnames are **not** accepted. `localhost` aborts startup rather than +being resolved: which of `127.0.0.1` and `::1` it means differs by +host, a name can resolve to several addresses of which only one could +be bound, and the answer can change under a running process. A value +carrying a port (`127.0.0.1:8080`) is likewise rejected — the port is +`PORT`'s business. Any unparseable value aborts startup; see +[Invalid values abort startup](#invalid-values-abort-startup). + +An address that parses but is not assigned to this host — say +`10.0.0.5` on a machine that has no such interface — is a valid +literal, so it reaches the listener and fails there. The process logs +the bind error and exits non-zero rather than staying up with nothing +listening. The effective value is in the `bindAddress` field of the +startup log line, which is the way to check what a running deployment +actually bound. + +#### Metrics credentials + +`METRICS_USERNAME` and `METRICS_PASSWORD` are set together or not at +all. With both set, `/metrics` is served behind basic auth. With +neither set, the route is not registered and returns 404. With one set +and the other empty or unset, the process refuses to start and exits +non-zero with an error naming both variables — mounting the endpoint +on the username alone would publish it behind a password that is the +empty string, and quietly withholding it would deny an endpoint that +was asked for. The `hasMetricsAuth` field in the startup log and the +existence of the route are the same value, so they cannot disagree. + +#### Single-instance lock + +Exactly one webhooker process may use a `DATA_DIR` at a time. Two +processes sharing one open the same databases and each run delivery +recovery over the same rows, so every pending delivery goes out twice — +duplicate delivery to your endpoints, from nothing worse than an +overlapping deploy or a double start. + +At startup, before anything opens a database, the process takes an +exclusive advisory lock (`flock(2)`) on `{DATA_DIR}/webhooker.lock` and +holds it for its lifetime. A second process pointed at the same +directory prints a message naming it and exits non-zero: + +``` +webhooker: data directory is already in use by another instance: /var/lib/webhooker (/var/lib/webhooker/webhooker.lock). Only one webhooker may use a data directory: two both run delivery recovery over the same rows and both deliver +``` + +The lock is the kernel's, not the file's: it is released when the +process exits, including `kill -9`, so a leftover `webhooker.lock` +never blocks a restart and must not be deleted by hand. The file is +also left in place on a clean shutdown, deliberately — unlinking it +would let the next process lock a fresh inode while a third still held +the old one. + +To run two webhookers on one host, give each its own `DATA_DIR`. + +`flock(2)` is host-local and per-inode: it arbitrates between processes +and containers sharing a volume or bind mount on one machine, but not +between hosts on a network filesystem, and a `DATA_DIR` inside a +container's own writable layer is not shared with anything. On a +filesystem that refuses `flock` outright, startup fails closed — the +process reports the error and refuses to start rather than running +unlocked. + +#### Trusted proxies + +`TRUSTED_PROXIES` is a comma-separated list of CIDR blocks (a bare +address such as `192.168.1.7` is accepted and treated as a single +host), for example `192.168.1.7, 2001:db8::5`. It decides whose +`X-Forwarded-For` header the rate limiters believe, so it should name +the addresses of your reverse proxies and nothing else. + +`X-Forwarded-For` is honoured **only** when the connecting peer is +inside one of these blocks; for every other peer the client identity is +the connection's own address and the header is ignored. The default is +the empty list, which trusts nobody — anything else would let any +client pick its own rate limit bucket, minting a fresh one per request +or draining someone else's. Set it to the address of your reverse +proxy, and to nothing wider. A set but unparseable value aborts +startup. + +That default is safe against forged headers, but leaving it unset in +production has a cost you must know about. Production runs behind a +TLS-terminating reverse proxy, so with `TRUSTED_PROXIES` unset every +request keys on the proxy's own address and all clients share a single +bucket per limit. The receiver limits become service-wide ceilings, +and the login endpoint's failure counting collapses onto one key, so a +stranger's wrong passwords throttle every other client's wrong +passwords. + +What it cannot do is lock the operator out. The login endpoint +verifies credentials **before** it consults any limit and charges only +failures, so a correct password is never throttled no matter how full +the bucket is. See [Rate Limiting](#rate-limiting). + +The remedy is to set `TRUSTED_PROXIES` to your reverse proxy's +address, which restores per-client buckets. webhooker logs a warning +at startup whenever `TRUSTED_PROXIES` is empty, in every environment — +not only when `WEBHOOKER_ENVIRONMENT=prod`, because that variable +defaults to `dev` and an operator who never set it is precisely the +one at risk. The warning is informational when nothing proxies to the +process: with no proxy in front, the peer address is the client's own +and the buckets are already per-client. See +[Rate Limiting](#rate-limiting) for what each limit shares. + +`X-Real-IP` and `True-Client-IP` are **never** read, from any peer. +Reverse proxies append to `X-Forwarded-For` but forward other client +headers verbatim, so a single-valued header is client-controlled even +behind a trusted proxy. + +Within a trusted request, `X-Forwarded-For` is read right to left, +because the rightmost entry is the one the nearest proxy appended and +everything left of it may have been written by the client. The first +hop that is not itself a trusted proxy is taken as the client. A hop +that is not a bare IP address — `ip:port`, a bracketed IPv6 literal, +the token `unknown` — ends the walk and the peer address is used +instead, since past such an entry the chain is not the shape assumed +here. The peer address is likewise used when the header is absent or +every hop in it is a trusted proxy. + +Two operator requirements follow: + +- Your proxy must **append** the peer address to `X-Forwarded-For` + (nginx `$proxy_add_x_forwarded_for`, HAProxy `option forwardfor`, + Caddy and AWS ALB by default), and must append a bare address with + no port. +- List proxy hosts **only**. Any address inside `TRUSTED_PROXIES` + chooses its own rate-limit key: its `X-Forwarded-For` is walked, so + it can name a different address on every request to get a fresh + bucket each time, or name another client's address to drain that + client's bucket. Never list a block that also covers clients — a + broad `10.0.0.0/8` on a network where clients live in the same range + makes all three limits, including the unauthenticated webhook + receiver, silently bypassable by every client in the block. + +#### Sessions + +Sessions are bounded by two independent clocks, and end at whichever +one runs out first: + +- **Idle expiry** (`SESSION_IDLE_TIMEOUT`, default `24h`) is a sliding + window. Every authenticated request pushes it forward, so a session + in continuous use never hits it, while an abandoned one expires a day + after its last use. Any non-positive value (`0`, or a negative + duration such as `-1s`) disables idle expiry entirely; the absolute + cap below still applies. A set-but-unparseable value aborts startup + rather than silently falling back to the default. +- **Absolute expiry** is a fixed 7 days from login. Activity does + **not** extend it: after a week, every session ends and the user + authenticates again. + +Only requests that authenticate with the session count as activity, so +an unauthenticated request carrying the cookie cannot keep a session +alive. The idle timestamp is rewritten at most once per tenth of the +idle window rather than on every request, which means a session may +expire up to 10% early relative to the user's true last request, but +never late. + +Both clocks are anchored by timestamps stored in the session cookie. +Sessions issued before this feature existed carry neither, so they are +treated as expired: upgrading to a build that has it logs every +existing session out once, and those users sign in again. + +#### Invalid values abort startup + +The defaults above apply **only** to variables that are unset (or set +to an empty string). A variable that is set but cannot be parsed is a +fatal configuration error: webhooker logs the offending variable and +its value and refuses to start, rather than silently running with a +substituted default. `PORT=eighty`, `DEBUG=ture`, and +`RETENTION_SWEEP_INTERVAL=1 hour` all abort startup. `PORT` must +additionally be a number in the range 1–65535, +`RECEIVER_RATE_LIMIT` must be at least 1, +`RETENTION_SWEEP_INTERVAL` must be greater than zero (it is a ticker +period, so `0s` or a negative value would crash the reaper after +startup), every entry in `TRUSTED_PROXIES` and +`ALLOWED_EGRESS_CIDRS` must be a CIDR block or a bare IP address, and +`BIND_ADDRESS` must be an IP address literal — `localhost`, +`127.0.0.1:8080` and `10.0.0.0/8` are each rejected rather than +resolved, split, or narrowed to something they do not say — and +`SENTRY_DSN` must parse as a Sentry DSN. +`SESSION_IDLE_TIMEOUT` is the exception: a +non-positive value there means idle expiry is disabled, not invalid. + +`SENTRY_DSN` is checked with the Sentry SDK's own parser, the same call +the SDK makes on the DSN it is later handed, so what configuration +accepts is exactly what will initialise. A typo in it is the one +configuration mistake nothing downstream can ever notice — the variable +is still set, so every later signal reports error reporting as on while +no report is being sent — which is why it aborts rather than starting +with reporting off. Leaving it unset is not a mistake and not affected: +error reporting is simply off and startup is normal. + +Boolean variables (`DEBUG`, `MAINTENANCE_MODE`) accept exactly the +spellings Go's `strconv.ParseBool` accepts — `1`, `t`, `T`, `TRUE`, +`true`, `True`, `0`, `f`, `F`, `FALSE`, `false`, `False` — and nothing +else. `yes`, `on`, and `off` are rejected rather than quietly treated +as false. On first startup, webhooker automatically generates a cryptographically secure session encryption key and stores it in the database. This key persists across restarts — no manual key management is needed. -On first startup, webhooker creates an `admin` user -with a randomly generated password and logs it to stdout. This password -is only displayed once. +#### The admin account + +On first startup — a `DATA_DIR` with no accounts in it — webhooker +creates an `admin` user with a randomly generated password and prints +it to standard output as a ruled banner: + +``` +======================================================================== +WEBHOOKER FIRST BOOT: an admin account has been created. + + username: admin + password: 3xamPl3-p4ssw0rd + +Save this password now: it is shown only here, and only once. +If it is lost, run `webhooker resetpw admin` on a stopped deployment. +======================================================================== +``` + +It is a banner rather than a log line because that is the only time it +is ever shown: as one `INFO` record it sat among the roughly 45 fx +`PROVIDE`/`RUN`/`HOOK` lines a boot writes, and under `docker run -d` +it is one line in a log subject to rotation. The database stores only +its Argon2id hash. There is no second account and no forgot-password +flow, so the banner and the reset command below are the only two ways +in. + +#### Recovering a lost admin password + +`webhooker resetpw` sets an existing account's password from the +command line: + +```bash +# Generate a new password and print it. +DATA_DIR=/var/lib/webhooker webhooker resetpw -generate admin + +# Or supply one on standard input (minimum 8 characters). +printf '%s' "$NEW_PASSWORD" | \ + DATA_DIR=/var/lib/webhooker webhooker resetpw admin +``` + +In a container it is the same binary, which the image sets as `CMD` +rather than `ENTRYPOINT`, so the whole command has to be given: + +```bash +docker run --rm -v webhooker-data:/var/lib/webhooker \ + webhooker /app/webhooker resetpw -generate admin +``` + +Stop the service first — with the volume still attached to a running +container, the command refuses. + +The password is never taken as a command-line argument: on Linux argv +is readable through `/proc` by every account on the host for as long as +the process lives. Standard input is echoed when it is a terminal — the +prompt says so — so `-generate` or a pipe is preferable on a shared +machine. + +What it will not do: + +- **Run against a live deployment.** It takes the same exclusive + `DATA_DIR` lock the server does (see + [Single-instance lock](#single-instance-lock)) and refuses while a + running instance holds it, naming the directory and exiting non-zero. + A running process keeps serving every session that authenticated with + the old password, so a reset underneath it would report a change the + service does not honour. +- **Create anything.** A `DATA_DIR` that does not exist, or that holds + no `webhooker.db`, is an error rather than a new empty deployment — + a mistyped path must not be built out and then reported as a success. +- **Create an account.** A username that does not exist is an error. + `resetpw` changes an existing account's password and nothing else. + +`DATA_DIR` selects the deployment exactly as it does for the server. A +password that changes on disk takes effect at the next login; sessions +that are already authenticated are unaffected either way. + +Changing a password you still know needs none of this — use +`POST /user/{username}/password` in the web UI. + +#### What `DEBUG=true` exposes + +`DEBUG=true` lowers the log level to `DEBUG`, which turns on every +statement GORM runs, the two by-design lookup misses on the +unauthenticated routes, and the rate limiter's own rejections. It is +meant to be safe to turn on while diagnosing a live service and safe to +paste the output of into a bug report. + +What it does **not** put in the log: + +- **Values bound to a SQL statement.** Statements are logged with their + placeholders, never with the values substituted into them, at every + level. That is what keeps the session encryption key out of the first + boot's `INSERT INTO settings` and the `admin` account's Argon2id + password hash out of its `INSERT INTO users` — the two statements + that made a debug log worth stealing. It applies to every table and + every statement rather than to a list of tables known to hold a + secret, so a table added later is covered without anyone remembering + to add it. The cost is that a failing statement can no longer be + replayed from the log alone: the statement, the table, the driver + error and the row count are all still there, but its values have to + come from the database. + `internal/gormlog/firstboot_test.go` boots the real graph with + `DEBUG=true` against an empty `DATA_DIR` and asserts that neither + secret appears in what that boot wrote to stdout. + The one exception is `(*gorm.DB).Scan`, which GORM logs through its + own trace recorder rather than through this filter. No production + code path calls it, and `internal/gormlog/scan_guard_test.go` fails + if a non-test file adds one. +- **Session cookies, API keys or target credentials.** None of these is + logged at any level. + +What is in the log regardless of `DEBUG`, and is not a debug-logging +decision: + +- **The initial `admin` password**, in the clear, once, on the first + boot that creates the account — as the banner described under + [The admin account](#the-admin-account), written straight to standard + output rather than through the logger. That banner is the only place + it is ever shown; the database stores the hash. A first boot's output + is not safe to paste anywhere until that account's password has been + changed. The same applies to `webhooker resetpw -generate`, which + prints the password it generated in the same form. +- **An authenticated operator's own configuration**, echoed back + untruncated — webhook names, target hostnames. See the logging + section under Security for the full list and for the per-line size + bound that covers unauthenticated traffic. ### Running with Docker ```bash docker run -d \ - -p 8080:8080 \ + -p 127.0.0.1:8080:8080 \ -v /path/to/data:/var/lib/webhooker \ -e WEBHOOKER_ENVIRONMENT=prod \ + -e BIND_ADDRESS=0.0.0.0 \ webhooker:latest ``` +**The image and the bare binary default `BIND_ADDRESS` differently, on +purpose.** The binary defaults to `127.0.0.1`; the image ships +`ENV BIND_ADDRESS=0.0.0.0`, so the `-e BIND_ADDRESS=0.0.0.0` above is +belt-and-braces and the command works without it. + +The two cases are not the same question. On a bare host, `0.0.0.0` +puts the cleartext admin UI and the unauthenticated receiver on every +interface of the machine, which is what the loopback default exists to +prevent. In a container, the network namespace is already that +boundary: nothing outside reaches `0.0.0.0:8080` because of the +namespace, not because of the bind. What decides exposure there is the +**publish flag**, and that is the line to get right. + +So publish to `127.0.0.1:8080` rather than `8080`. A bare +`-p 8080:8080` opens the port on every interface of the host — through +firewall rules too, since Docker's forwarding rules are inserted ahead +of most host firewalls. Publish to the host address your reverse proxy +connects from, and nothing wider. + +**An empty `BIND_ADDRESS` is treated as unset**, like every other +variable here, so `-e BIND_ADDRESS=` does not mean "keep the image +default" — it discards the image's `0.0.0.0` and falls back to the +_binary's_ `127.0.0.1`. In a container that is the failure below, and +nothing in the logs names the variable. A templated Compose file or a +`.env` line with an empty value is the usual way in. Either set a +literal or leave the variable out entirely. + +Overriding `BIND_ADDRESS` to a loopback address in a container — by +that route or deliberately — makes the container unreachable from +outside its namespace even with `-p`. The published port answers +nothing, and the health check fails too: it requests +`http://localhost:8080`, `localhost` resolves to `::1` first, and a +`127.0.0.1` bind is not listening there. The container then goes +`unhealthy` about **65 seconds** after start — from `HEALTHCHECK +--start-period=5s --interval=30s --retries=3`, so failing probes at +5s, 35s and 65s, and `unhealthy` on the third. (Docker's probe cadence +during the start period has changed between versions; re-derive from +those three values rather than trusting the figure. Measured at 65s on +Docker 29.7.2.) A container `unhealthy` with `connection refused` in +its health log, or a published port that resets connections, is this. + The container runs as a non-root user (`webhooker`, UID 1000), exposes port 8080, and includes a health check against `/.well-known/healthcheck`. The `/var/lib/webhooker` volume holds all -SQLite databases: the main application database (`webhooker.db`) and -the per-webhook event databases (`events-{uuid}.db`). Mount this as a -persistent volume to preserve data across container restarts. +SQLite databases: the main application database (`webhooker.db`), the +per-webhook event databases (`events-{uuid}.db`), and any archive +databases written by `database` targets (`archive-{uuid}.db`). Mount +this as a persistent volume to preserve data across container +restarts. + +**The bind-mounted directory must be owned by UID 1000, or the +container does not start.** Docker creates a `-v` source path that +does not exist yet as `root:root`, and the process runs as UID 1000, +so it cannot take its `DATA_DIR` lock: + +``` +webhooker: locking data directory /var/lib/webhooker: open +/var/lib/webhooker/webhooker.lock: permission denied +``` + +It exits non-zero at that point, before opening any database. Create +the directory ahead of the first `docker run`: + +```bash +mkdir -p /path/to/data +chown 1000:1000 /path/to/data +chmod 750 /path/to/data +``` + +The same `chown` is what a restore needs — see step 4 of +[Restore](#restore). A **named volume** does not have this problem: +Docker copies the image's ownership onto a volume it initializes, and +the image creates `/var/lib/webhooker` owned by `webhooker`. + +**The file modes are not yours to set, and do not depend on the +directory.** `webhooker.db` holds target configuration in plaintext — +bearer tokens, API keys, Slack webhook URLs — along with the session +encryption key, so webhooker creates every SQLite file it owns `0600`: +each database and both of its `-wal` and `-shm` sidecars, across all +three tiers. Files an earlier build left `0644` are tightened when +they are opened. A `DATA_DIR` webhooker creates itself is `0750`, but +a bind mount supplies its own directory and Docker's default for one +it creates is `0755`; the `0600` files hold there regardless. The +`chmod 750` above is defence in depth — it stops other local users +listing the directory and learning your webhook UUIDs from the +`events-{uuid}.db` filenames — not the barrier protecting the +credentials. + +## Deployment behind a reverse proxy + +webhooker terminates no TLS of its own. It serves plaintext HTTP and +expects a reverse proxy in front of it, which is the deployment it is +built for: the proxy holds the certificate, and webhooker binds +loopback where only the proxy can reach it. + +Five things have to be right. Each one is silent when it is wrong — +the service comes up, serves pages, and is broken in a way nothing +reports. + +1. **Bind or firewall the app port.** The binary binds `127.0.0.1` by + default, so the cleartext listener is not published beside the + proxy. The image binds `0.0.0.0` inside its own network namespace + and relies on the publish address instead — + `-p 127.0.0.1:8080:8080`. Either way the port must reach the proxy + and nothing else; widen it only with a firewall or a publish + address in front of it. A cleartext port answering the internet + serves the admin login form and the unauthenticated receiver with + no TLS at all, and the proxy in front of it changes nothing about + that. +2. **Set `WEBHOOKER_ENVIRONMENT=prod`, and make sure the proxy sends + `X-Forwarded-Proto`.** These are two requirements, not one. The + environment setting decides CORS and nothing else: the default + `dev` answers every origin with `Access-Control-Allow-Origin: *` + (without credentials), which a server-rendered production + deployment has no use for. Cookie `Secure` and the strict + Origin/Referer mode are **not** tied to it — they are decided per + request from the transport, which behind a proxy means the + `X-Forwarded-Proto` header. The block below sets it; without it + every request is read as plaintext and cookies ship without + `Secure`. See [Configuration](#configuration). +3. **Set `TRUSTED_PROXIES` to the proxy's address.** Unset, every rate + limiter keys on the connecting peer, which behind a proxy is the + proxy on every request: all clients collapse into one global bucket + per limit and the receiver's per-IP limits become service-wide + ceilings. See [Trusted proxies](#trusted-proxies). List the proxy + and nothing else. +4. **Send `Host` as `$http_host`, not `$host`.** `$host` strips the + port. webhooker's Origin/Referer check compares against the host it + was given, so on any port other than 443 `$host` makes every form + POST — including login — fail with `403 origin invalid`, with + nothing in the error naming the cause. +5. **Keep the proxy's access log.** webhooker's own access log records + the peer address, which behind a proxy is always the proxy. The + proxy's log is the only record of which client sent what. nginx's + default `combined` format already logs `$remote_addr`; do not + replace it with one that drops the client address, and retain those + logs as long as you would want to answer a question about traffic. + +### nginx + +Complete server block. Replace the `server_name` and the two +certificate paths. + +```nginx +server { + listen 443 ssl; + listen [::]:443 ssl; + http2 on; # nginx 1.25.1+; older: listen 443 ssl http2; + + server_name webhooker.example.com; + + ssl_certificate /etc/ssl/certs/webhooker.example.com.crt; + ssl_certificate_key /etc/ssl/private/webhooker.example.com.key; + ssl_protocols TLSv1.2 TLSv1.3; + + # webhooker caps form POST bodies at 1 MB. nginx's default happens + # to match, so leaving this out breaks nothing today — but if you + # ever raise webhooker's cap, this is the limit you will still be + # hitting, and the rejection is nginx's HTML page rather than + # webhooker's message. + client_max_body_size 1m; + + # $remote_addr is the client. webhooker's own log records this + # proxy and nothing else, so this file is the only place the + # client's address is written down. + access_log /var/log/nginx/webhooker.access.log combined; + + location / { + # A literal address, not localhost: with BIND_ADDRESS at its + # 127.0.0.1 default, a localhost that resolves to ::1 first + # gets connection refused. + proxy_pass http://127.0.0.1:8080; + + # $http_host, NOT $host. $host drops the port and every form + # POST fails with 403 origin invalid on any port but 443. + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Above webhooker's own 60s request timeout, so its 503 + # reaches the client instead of nginx cutting the connection + # first and answering 504. + proxy_read_timeout 70s; + } +} + +server { + listen 80; + listen [::]:80; + server_name webhooker.example.com; + return 308 https://$host$request_uri; +} +``` + +`X-Forwarded-For` must be **appended** to, which +`$proxy_add_x_forwarded_for` does. webhooker reads no other forwarded +client header: `X-Real-IP` and `True-Client-IP` are ignored from every +peer, so setting them has no effect. See +[Trusted proxies](#trusted-proxies) for how the chain is walked. + +`X-Forwarded-Proto: https` is what tells webhooker the request arrived +over TLS, which decides the `Secure` flag on both the session and CSRF +cookies and the strict Origin/Referer mode. Without it, requests are +treated as plaintext and the cookies ship without `Secure`. Unlike +`X-Forwarded-For`, this header is read from any peer and is not gated +by `TRUSTED_PROXIES`, so the proxy must overwrite whatever a client +sent — `$scheme` above does. + +With that block, webhooker's environment is: + +```sh +WEBHOOKER_ENVIRONMENT=prod +BIND_ADDRESS=127.0.0.1 # the default; stated here to be explicit +TRUSTED_PROXIES=127.0.0.1 +``` + +If nginx runs on another host, `BIND_ADDRESS` becomes the address it +connects to, `TRUSTED_PROXIES` becomes nginx's address, and the port +must be firewalled to that address — the traffic between them is +cleartext. + +### HSTS is always sent, and is not configurable + +Every response carries +`Strict-Transport-Security: max-age=63072000; includeSubDomains; preload`. +Two years, every subdomain, and a `preload` token. There is no setting +that changes or suppresses it. + +This is worth knowing before the first request reaches a browser: a +client that sees it once will refuse plaintext HTTP to that hostname — +and to every subdomain of it — for two years, whatever else is served +there. Terminate TLS on a hostname you are prepared to keep on HTTPS. + +## Backup, Restore, and Upgrades + +### What to back up + +All persistent state lives in `DATA_DIR` (`/var/lib/webhooker` by +default). Back up that directory in full — the state is spread across +several files whose names depend on your data, so copying the directory +is both the simplest and the only complete rule: + +- `webhooker.db` — one per install. Settings (including the session + encryption key), users, API keys, webhooks, entrypoints, targets. +- `events-{webhook_uuid}.db` — **one per webhook**. Events, deliveries, + delivery results. +- `archive-{webhook_uuid}.db` — **one per webhook that has a `database` + target**. Archived events. Keyed on the webhook UUID, not the target + UUID: a webhook with several `database` targets still has exactly one + archive file. + +`{webhook_uuid}` is the webhook's UUID primary key in its canonical +36-character hyphenated form, so a real filename looks like +`events-3f2a1c9e-....db`. The only other file is `webhooker.lock`, the +always-empty [single-instance lock](#single-instance-lock); it holds no +state and is not part of the backup set — a copied one is stale and +blocks nothing. + +**`-wal` and `-shm` sidecars.** Every database runs in WAL journal mode, +so while the service is running each `{name}.db` has a `{name}.db-wal` +and a `{name}.db-shm` beside it. **`-wal` is part of the database, not a +scratch file**: it holds committed transactions that are not yet in the +`.db`, so a copy of the `.db` without its `-wal` is missing data and may +have no readable schema at all. `-shm` is regenerable, but there is no +reason to separate the two — copy the directory and you have them. + +A clean shutdown closes `webhooker.db` and every `events-*.db`, which +checkpoints and removes their sidecars; a killed or crashed instance +leaves them, and they must be carried with the `.db`. **Archive +databases are different**: their handle is not closed at shutdown, so +`archive-*.db-wal` and `-shm` normally survive a clean stop and the +`-wal` can hold every row the archive has. Measured on a stopped +instance: `archive-….db` 4096 bytes with no table, its `-wal` 157 KB +holding all 8 archived events. Copying `DATA_DIR` in full is what makes +this a non-issue; copying `.db` files out of it by name is not. + +Configuration is **not** in `DATA_DIR` — it comes from the environment +and from a `.env` file read out of the process working directory. Back +that up with your deployment config, separately. + +### A hot copy is not safe + +Every database webhooker opens runs in WAL journal mode. The main and +event databases are also held open for the entire process lifetime — +`WebhookDBManager` caches event database handles and closes them only on +webhook deletion or shutdown — so "it looked idle" is not a guarantee +that nothing was mid-transaction. + +That means `cp`, `rsync`, `tar` or a filesystem snapshot taken against a +running instance can capture a database and its `-wal` at two different +instants and yield a file that is corrupt or missing state. Copying a +`.db` on its own is worse and fails loudly: recently written pages, +including the schema itself on a young database, live in the `-wal`, so +the copy reads back as an empty or table-less database. Use one of the +two procedures below instead. + +**Stop, copy, start.** The simplest, needs no extra tooling, and the +only one that gives a single point in time across every file: + +```bash +docker stop webhooker +cp -a /path/to/data /path/to/backup-$(date -u +%Y%m%dT%H%M%SZ) +docker start webhooker +``` + +**SQLite online backup.** No downtime, one file at a time: + +```bash +for db in /path/to/data/*.db; do + sqlite3 "$db" ".backup '/path/to/backup/$(basename "$db")'" +done +``` + +`.backup` reads through the WAL and writes a single consistent file with +no sidecars of its own, so the destination is complete as it stands. +Two caveats. First, the runtime image is `alpine:3.21` with only +`ca-certificates` added — the `sqlite3` CLI is **not** in it, so run +this on the host against the volume path, or from a throwaway container +that mounts the volume. Second, each file is captured at its own +instant, so a webhook created or an event delivered between two files +being copied lands in one and not the other. If you need the whole set +coherent as of a single moment, stop the service. + +Note that `sqlite3 .dump` is **not** one of these procedures: it is +an export, it holds a read transaction open for as long as it runs, and +it pins the WAL against checkpointing for that whole time. It is safe to +run — it does not block ingestion — but back up with `.backup` or a +stopped copy. + +Archive databases are the one exception the service is built for: the +archive writer closes and reopens its handle around writes (debounced +to at most one reopen per second), so an operator can move +`archive-{uuid}.db` away for offline retention while the service runs, +and it is recreated on the next write. See +[Database Architecture](#database-architecture). That is a +move-the-file-away workflow, not a substitute for the backup procedures +above. + +**Move the sidecars with it.** Under WAL that workflow is no longer a +single file, and the common case is the dangerous one. The reopen +happens on the *next* write after the debounce window elapses, so after +the last write of a burst nothing checkpoints: measured, 20 s after ten +events the `archive-….db` was 4096 bytes — a header, no table — with +all ten rows sitting in a 189 KB `-wal`. Copying the `.db` alone at that +moment yields a file that opens with `no such table: archived_events`. +The file becomes self-contained again when the handle closes, which +happens on the next write past the debounce window, when the connection +pool retires the idle connection (about a minute after the last write), +or at the idle archive sweep — measured, the same file was a complete +20 KB `.db` with no sidecars about a minute after its last write. +Shutdown is **not** on that list: the archive handle is not closed when +the service stops. So either move `archive-{uuid}.db` together with any +`-wal`/`-shm` beside it, or wait until there are none. + +### Restore + +1. Stop the service. + +2. Restore the **whole set together**: `webhooker.db` *and* every + `events-*.db` *and* every `archive-*.db`. A partial restore fails + quietly rather than loudly. Every database is opened `mode=rwc`, so a + missing `events-{uuid}.db` is **created empty** on first access + instead of erroring — the webhook comes back with its configuration + intact and its entire event history silently gone. Event databases + restored without `webhooker.db` are simply orphaned; nothing + references their UUIDs. + +3. Carry any `*.db-wal` and `*.db-shm` files that are in the backup. + They are part of the database, and dropping a `-wal` silently + discards every transaction it still holds. An `.backup` set will not + contain any: it writes a single consolidated file per database. A + stop-and-copy set has none for `webhooker.db` or the `events-*.db`, + because a clean stop closes those and checkpoints their sidecars + away — but it will normally have them for `archive-*.db`, whose + handle stays open across shutdown, and those carry the archive's + rows. A copy salvaged from a crashed instance has them for + everything, and needs all of them. + +4. **Fix ownership.** The container runs as the non-root `webhooker` + user, UID 1000 / GID 1000. Restored files must be owned by (or + writable by) that UID, and so must the directory itself — SQLite + creates the `-wal` and `-shm` sidecars beside the database, so a + writable file inside a directory it cannot write is not enough: + + ```bash + chown -R 1000:1000 /path/to/data + ``` + + Restoring as `root` on the host and forgetting this step is the + usual way a restore fails. + +5. Start the service. `AutoMigrate` runs against each restored database + as it is opened. + +### Upgrades + +**Back up before every upgrade.** Every start runs GORM `AutoMigrate` +unconditionally against whatever files it finds: + +- the main database on connect — `Setting`, `User`, `APIKey`, `Webhook`, + `Entrypoint`, `Target` +- each event database when it is lazily opened — `Event`, `Delivery`, + `DeliveryResult` +- each archive database on every open and reopen + +There is no schema version table, no migration ledger, and no down +migrations. Nothing in the files records which version wrote them, and +no code path undoes a migration. + +Upgrade procedure: + +1. Stop the service. +2. Back up `DATA_DIR` using one of the procedures above. +3. Pull the new image and start it. +4. Confirm `database migrations completed` in the logs before putting + traffic back on it. +5. Confirm the new build is the one running: + `curl -s http://host:8080/.well-known/healthcheck` reports the + version it was stamped with (see [Version stamping](#version-stamping)). + +**Upgrading past the introduction of `BIND_ADDRESS`:** earlier versions +always bound every interface. **Container deployments are unaffected** +— the image ships `ENV BIND_ADDRESS=0.0.0.0`, so a `docker run` or +Compose service that worked before still works with nothing changed. + +A **bare binary** is the case that changes: the listener now binds +`127.0.0.1` unless `BIND_ADDRESS` says otherwise, so a deployment that +relied on reaching it from another host becomes unreachable until it +sets the address the proxy connects to. Check the `bindAddress` field +of the startup log to see what a running process bound. See +[Bind address](#bind-address). + +**Downgrade is unsupported.** Once a newer binary has migrated the files +there is no way to move them back. `AutoMigrate` is additive — it adds +tables, columns and indexes and never drops or rewrites them — so an +older binary will generally open migrated files and appear to work while +writing against a schema it does not know about. The failure mode is +silent divergence, not a startup error. The only supported way back to +an older version is restoring the pre-upgrade backup, which discards +everything received since that backup was taken. + +### Version stamping + +The binary reports its version at `/.well-known/healthcheck` (the +`version` field), in the UI footer, and in the startup log line +(`msg=starting`, `version=...`). It is also the Sentry release name, +as `webhooker-{version}`. The value is stamped in at build time by the +linker; it is not read from a file at runtime, so it identifies the +build itself. + +`script/version` produces the value and both build paths use it: + +| Build | What it reports | +| --- | --- | +| Clean checkout at a tag | exactly that tag, e.g. `v1.0.0` | +| Commits past a tag | `v1.0.0-3-g1a2b3c4` — tag, commits since, short SHA | +| No tag reachable | the short SHA, e.g. `1a2b3c4` | +| Uncommitted changes | the above with a `-dirty` suffix | +| No git metadata | `unknown` | + +`unknown` is what a source tarball or a `docker build .` with no +`--build-arg VERSION=...` reports. `.dockerignore` excludes `.git/`, so +the build context carries no git metadata and the image cannot derive +the version itself: `script/docker` (and so `make docker`) resolves it +on the host and passes it in as the `VERSION` build arg. A build that +reports `unknown` is a build nobody told what it was; it is not a +failure, but it cannot be traced back to a commit. + +`make version` prints what the current checkout would stamp, and +`make build VERSION=v1.2.3` overrides it. An empty override — from +`make build VERSION=` or from `--build-arg VERSION=` — means unset +rather than `""`, and resolves the way an absent one does. + +Nothing that varies between two builds of the same commit is stamped — +no timestamp, no hostname, no builder identity — so two builds of one +commit still produce a byte-identical binary. + +### Backups contain secrets + +Treat a backup with the same care as the credentials inside it. Encrypt +backups at rest and restrict who can read them. + +- `events-{uuid}.db` and `archive-{uuid}.db` hold the **full payload + body and headers** of every event as received, including whatever the + sending service put in them — tokens, signatures, personal data. +- Event databases written before + [issue #206](https://git.eeqj.de/sneak/webhooker/issues/206) was fixed + **also contain target credentials**: a GORM association upsert on the + delivery and retry write path copied `targets` rows, `config` + included, into the per-webhook database. For a Slack target the + `webhookUrl` *is* the bearer credential, and an `http` target's URL + can embed userinfo. This version never writes those rows; the first + time it opens such a file it deletes them and vacuums the file, which + removes the credential bytes rather than only unlinking the rows. + Deleting alone would not: the bytes stay readable in the file's free + pages until it is rewritten. The sweep is recorded in the file's + `user_version` only once the vacuum returns, so a sweep that fails or + is interrupted fails the open and is retried on the next one, and a + file this version has opened without error holds no leaked rows and + no recoverable bytes from them. On upgrade this rewrites each + existing `events-{uuid}.db` once, on its first open. Two cases still + hand over live delivery destinations: a backup taken from an older + build, and a backup of a file this version has not yet opened + successfully. Copies already made stay affected — the sweep only + rewrites the file it opens, and freed blocks may persist in + filesystem snapshots and on the underlying storage. Rotate any target + credential that was in a backup you cannot account for. +- `webhooker.db` stores target config **unencrypted**, tracked at + [issue #212](https://git.eeqj.de/sneak/webhooker/issues/212), next to + the session encryption key and the Argon2id password hashes. + +## The entrypoint URL is the authentication secret + +The receiver verifies nothing about an inbound request. The UUID in an +entrypoint's URL is its credential: anyone who holds that URL can +submit events to it, and the receiver checks nothing else about the +sender. Treat an entrypoint URL the way you would treat an API token. + +There is no way to rotate the UUID in place. To retire one, delete the +entrypoint (or deactivate it, which answers `410`) and create a new +one, then point the sender at the new URL. ## Entrypoints This repository adheres to the [Scripts to Rule Them All](https://github.com/github/scripts-to-rule-them-all) standard: normalized scripts in `script/` are the entrypoints for the -development workflow, and the Makefile targets are thin shims that call -them. We provide: +development workflow. Ten of the Makefile's seventeen targets are thin +shims that call them; `build`, `run`, `dev`, `deps`, `clean`, `css` and +`version` are inline commands with no script behind them, though +`build` and `version` both take their value from `script/version`. We +provide: - `script/bootstrap` — install all dependencies (idempotent) - `script/setup` — make a fresh clone ready for development (bootstrap, then install-precommit) - `script/projectname` — output the project name ("webhooker") +- `script/fetch-assets` — download the third-party browser assets into + `static/`, verifying each against its pinned sha256 - `script/test` — run the test suite -- `script/lint` — run golangci-lint +- `script/lint` — run golangci-lint in Docker (see Linting below) - `script/fmt` — format all code (writes) - `script/fmt-check` — check formatting (read-only) - `script/check` — run test, lint, and fmt-check -- `script/docker` — build the Docker image tagged via `script/projectname` +- `script/version` — output the version to stamp into the binary (see + [Version stamping](#version-stamping)) +- `script/docker` — build the Docker image tagged via + `script/projectname`, passing `script/version`'s output in as the + `VERSION` build arg - `script/cibuild` — CI entrypoint: `docker build .` (the Dockerfile runs the checks, so a green build implies a green repo) +- `script/ci-mark-superseded` — CI helper: mark the commits whose run a + newer push cancelled (see [CI gate honesty](#ci-gate-honesty)) - `script/precommit` — pre-commit checks (`go mod tidy` guard, then `script/check`) - `script/install-precommit` — install the git pre-commit hook that runs `script/precommit` +## Third-party browser assets + +The web UI serves one third-party script, Alpine.js. It is **not** committed: +a minified bundle in the tree is unreviewable, and `REPO_POLICIES.md` bars +both committed build artifacts and unpinned external references. + +Instead `script/fetch-assets` downloads it from a pinned URL, checks the +download against a hardcoded sha256, and installs it under `static/`. The +sha256 of every installed asset is recorded in `static/vendor.sha256`, and +`static/vendor_test.go` re-hashes the bytes `go:embed` put in the binary +against that manifest — so the pin is enforced on what actually ships, not +merely written down. Any mismatch fails the build. + +`make bootstrap` runs the fetch for local development, and the Dockerfile +runs it in the build stage; `.gitignore` and `.dockerignore` keep the +artifact out of both the repo and the build context. + +To move to a new version: update the version, URL, and tarball sha256 in +`script/fetch-assets` and the asset sha256 in `static/vendor.sha256`, then +run `make assets && make check`. + ## Rationale Webhook integrations between services are inherently fragile. The @@ -162,6 +1234,16 @@ webhooker solves this by acting as a durable intermediary: backoff. Every delivery attempt is logged with status codes, response bodies, and timing. + **That guarantee is at-least-once, not exactly-once.** When a send + reaches its target but the write recording that outcome fails, the + delivery is deliberately left in a recoverable state rather than + marked done — losing a delivery is the worse failure — so the + pending sweep picks it up about fifteen minutes later, or the next + restart does, and the target receives a payload it already got. + webhooker adds no delivery identifier of its own to an outbound + request, so **make your receiver idempotent** against whatever the + payload itself carries. + 3. **Observability** — Full request/response logging for every webhook received and every delivery attempted. Prometheus metrics expose volume, latency, and error rates. The web UI provides real-time @@ -171,9 +1253,13 @@ webhooker solves this by acting as a durable intermediary: targets simultaneously. This enables patterns like forwarding a GitHub webhook to both a deployment service and a Slack channel. -5. **Replay** — Stored events can be manually redelivered for debugging - or testing, without requiring the original sender to fire the webhook - again. +5. **Replay and resubmit** — Every received event is stored in full, + and the event log offers two redelivery actions built on that. + **Replay** re-sends one finished delivery to its own target, for + recovering a delivery that failed. **Resubmit** re-injects the stored + event as a new undelivered event and fans it out to every currently + active target, for firing captured traffic at a backend under + development. Both are web UI actions; there is no API for either. ### Use Cases @@ -182,7 +1268,8 @@ webhooker solves this by acting as a durable intermediary: - **Observability** via Prometheus metrics on webhook frequency, payload size, and delivery performance - **Debugging** and introspection of webhook payloads in the web UI -- **Replay** of webhook events for application testing and development +- **Resubmit** of captured webhook events for application testing and + development, and **replay** of a single failed delivery for recovery - **Fan-out** delivery of a single webhook to multiple downstream targets - **High-availability ingestion** for delivery to less reliable backend @@ -211,7 +1298,12 @@ It uses: - **[gorilla/csrf](https://github.com/gorilla/csrf)** for CSRF protection (cookie-based double-submit tokens) - **[go-chi/httprate](https://github.com/go-chi/httprate)** for - per-IP login rate limiting (sliding window counter) + sliding-window rate limiting of the password-change and webhook + receiver endpoints. The bucket is per client IP only when + `TRUSTED_PROXIES` names the reverse proxy; unset, every client + behind that proxy shares one bucket per limit. The login endpoint + counts failed attempts itself instead, so that a correct password is + never throttled (see [Rate Limiting](#rate-limiting)) - **[Prometheus](https://prometheus.io)** for metrics, served at `/metrics` behind basic auth - **[Sentry](https://sentry.io)** for optional error reporting @@ -229,7 +1321,7 @@ The codebase uses consistent naming throughout (rename completed in ### Data Model -webhooker's data model has eight entities organized into two tiers: the +webhooker's data model has nine entities organized into two tiers: the **application tier** (user and webhook configuration) and the **event tier** (event ingestion, delivery, and logging). @@ -292,7 +1384,10 @@ A registered user of the webhooker service. Passwords are hashed with Argon2id using secure defaults (64 MB memory, 1 iteration, 4 threads, 32-byte key, 16-byte salt). On first startup, an `admin` user is created with a randomly generated 16-character -password logged to stdout. +password printed once to stdout; `webhooker resetpw` sets it again if +it is lost (see [The admin account](#the-admin-account)). Every one of +those paths hashes through the same `internal/database` code, so the +parameters cannot drift between them. #### Webhook @@ -307,13 +1402,45 @@ event routing. | `user_id` | UUID | Foreign key → User | | `name` | string | Human-readable name | | `description` | string | Optional description | -| `retention_days` | integer | Days to retain events (default: 30) | +| `retention_days` | integer | Days to retain events (default: 30; 0 means retain forever) | **Relations:** Belongs to User. Has many Entrypoints. Has many Targets. The `retention_days` field controls how long event data is kept in the webhook's dedicated database before automatic cleanup. +Setting `retention_days` to `0` means "retain events forever". Because +the column carries a default of 30, a literal zero cannot survive an +insert, so a zero is rewritten on save to a sentinel of `365 * 1000` +days (`database.RetentionForeverDays`). The retention reaper recognises +that sentinel and skips the webhook entirely, and the web UI displays +such a webhook's retention as "forever" rather than as a day count. + +Submitted `retention_days` values therefore fall into three bands, not +two: + +- `1` up to `database.MaxFiniteRetentionDays` (106751 days, about 292 + years) is accepted as a finite retention. +- Above that ceiling but below the retain-forever sentinel of 365000 + (`database.RetentionForeverDays`) is rejected with a 400. This is the + band the cap exists for. +- `0`, and `365000` or above, are accepted and mean retain forever, + collapsing to the sentinel — `0` in `Webhook.BeforeSave`, the large + values in `parseRetentionDays`. The large values are not out of + range: the edit form pre-fills the sentinel for a retain-forever + webhook, so submitting that form back unchanged has to keep meaning + "forever". + +A negative value is in none of the three: `parseRetentionDays` rejects +it with a 400 before `BeforeSave` ever sees it. + +The cap is not arbitrary: the reaper computes its cutoff as a +`time.Duration`, an int64 nanosecond count, and a longer period +overflows it. An overflowed cutoff lands in the future, where it +matches every row, so the sweep would delete every event the webhook +has instead of none. The reaper also clamps the value it is given, so a +row written by an older version cannot trigger that either. + #### Entrypoint A receiver URL where external services POST webhook events. Each @@ -325,7 +1452,7 @@ the full request and creates an Event. | -------------- | ------- | ----------- | | `id` | UUID | Primary key | | `webhook_id` | UUID | Foreign key → Webhook | -| `path` | string | Unique URL path (UUID-based, e.g. `/webhook/{uuid}`) | +| `path` | string | Unique bare UUID, generated at creation. The `/webhook/` prefix is route only and is not stored: the receiver matches this column against the raw `{uuid}` path segment. It is also the entrypoint's credential; see [The entrypoint URL is the authentication secret](#the-entrypoint-url-is-the-authentication-secret) | | `description` | string | Optional description | | `active` | boolean | Whether this entrypoint accepts events (default: true) | @@ -349,8 +1476,8 @@ events should be forwarded. | `type` | TargetType | One of: `http`, `slack`, `database`, `log` | | `active` | boolean | Whether deliveries are enabled (default: true) | | `config` | JSON text | Type-specific configuration | -| `max_retries` | integer | Maximum retry attempts for HTTP targets (0 = fire-and-forget, >0 = retries with backoff) | -| `max_queue_size` | integer | Maximum queued deliveries (for HTTP targets with retries) | +| `max_retries` | integer | Maximum retry attempts for `http` and `slack` targets (0 = fire-and-forget, >0 = retries with backoff and a circuit breaker). Ignored by `database` and `log` targets | +| `max_queue_size` | integer | Stored and shown on the target's detail view, but not enforced anywhere yet: nothing in the delivery engine consults it. Queue depth is set by the two fixed 10,000-entry channels | **Relations:** Belongs to Webhook. Has many Deliveries. @@ -363,6 +1490,11 @@ events should be forwarded. greater than 0, failed deliveries are retried with exponential backoff up to `max_retries` attempts, protected by a per-target circuit breaker. +- **`slack`** — Post the event as a formatted message to a + Slack-compatible incoming webhook URL (`webhookUrl` in `config`). It + is built on the same HTTP core as `http` and honours `max_retries` + identically, circuit breaker included. See the Slack target section + under "Per-Webhook Event Databases" for the message format. - **`database`** — Archive the full event as a row into a separate per-webhook archive database (`archive-{webhookID}.db`) for long-term retention, with an optional creation-validated expiry (default: keep @@ -375,6 +1507,53 @@ events should be forwarded. The `config` field stores type-specific configuration as JSON (e.g., destination URL, custom headers, timeout settings). +**`http` target configuration:** + +| Key | Type | Description | +| --------- | ------------- | ----------- | +| `url` | string | Destination the event is POSTed to | +| `headers` | object | Extra request headers, applied last so they win over the event's own forwarded headers | +| `timeout` | integer (sec) | Per-target request timeout; unset (or 0) uses the shared 30-second client timeout | + +`timeout` is capped at **300 seconds**, and the form rejects anything +above it rather than substituting the cap. A delivery attempt holds one +of the bounded pool's workers for its whole duration, so an unbounded +timeout would let a single unresponsive destination stall the queue. + +`headers` rejects the names the delivery path or `net/http` writes +regardless of what is configured: `Host`, `Content-Length`, +`Transfer-Encoding`, `Connection`, `Trailer` and `User-Agent`. These are +refused at the form rather than accepted and ignored, because a stored +header that provably never reaches the wire tells the operator their +configuration took effect when it did not. `Content-Type` is _not_ +reserved: a configured one deliberately overrides the event's. + +**Redirects.** A redirect from an `http` target's destination is +followed, up to ten hops, and the delivery's recorded status and body +come from the final hop. One rule governs every header the delivery +carries for someone else — the configured `headers` and the inbound +event headers forwarded from the sender alike: **a hop that leaves the +origin the target names carries none of them.** Leaving the origin +means a different host, a different port, or a step down from `https` +to `http`. Both classes routinely carry a secret — a configured +`X-Api-Key` or `PRIVATE-TOKEN`, an inbound `X-Hub-Signature` — and an +open redirect at the destination would otherwise hand it to a host the +operator never chose. `net/http` already does this for `Authorization` +and `Cookie`. The delivery path's own headers (`Content-Type`, +`User-Agent`) are not origin-scoped and always travel, so a body +preserved across a `307` is still typed. A `301`, `302` or `303` is a +different matter, and this is `net/http`'s behaviour rather than +webhooker's: the POST becomes a GET and the event body and its +`Content-Type` are dropped, so the destination the chain ends at +receives no event at all — and the delivery is still recorded +`Delivered` on that hop's `2xx`. Redirects within the target's own +origin keep everything, so a destination that redirects its own paths +is unaffected; the drop is per hop rather than permanent, so a chain +that returns to the configured origin carries the headers again, +exactly as `net/http` treats `Authorization`. Each hop is dialled +through the same SSRF guard as the first, so a redirect aimed at a +private or reserved address is refused at connect time. + #### APIKey A programmatic access credential for API authentication. @@ -392,17 +1571,18 @@ A programmatic access credential for API authentication. #### Event A captured incoming webhook request. Stores the complete HTTP request -data for replay and auditing. +data for auditing, for replay, and for resubmission. | Field | Type | Description | | -------------- | ------ | ----------- | | `id` | UUID | Primary key | | `webhook_id` | UUID | Foreign key → Webhook | | `entrypoint_id` | UUID | Foreign key → Entrypoint | -| `method` | string | HTTP method (POST, PUT, etc.) | +| `method` | string | HTTP method of the captured request. Always `POST`: the receiver answers every other method with 405 before an Event is created | | `headers` | JSON | Complete request headers | | `body` | text | Raw request body | | `content_type` | string | Content-Type header value | +| `resubmitted_from_id` | UUID | The event this one was copied from by a resubmit (nullable; empty for an event that arrived on the receiver). Not a foreign key: the source event can be reaped by retention while its copies remain | **Relations:** Belongs to Webhook. Belongs to Entrypoint. Has many Deliveries. @@ -434,6 +1614,43 @@ DeliveryResults. succeeded). - **`failed`** — All retry attempts exhausted without success. +**Replay.** A `delivered` or `failed` delivery is finished as far as +the engine is concerned, but the event is still stored, so the event +log offers a per-delivery **Replay** action for it. Replay creates a +NEW `pending` delivery for the same event and target and hands it to +the engine on the ordinary path — same retries, same SSRF guard, same +circuit breaker as a first attempt. It never touches the delivery it +repeats: that row's status, timestamps and recorded attempts stand as +the record of what happened. + +What is re-sent is the stored event body, against the target's +configuration **as it stands now** — the point of a replay is to +deliver where the destination has since been fixed. A target that has +been deleted or deactivated therefore refuses the replay with a +message on the event log rather than delivering from stale +configuration, and a replay is refused while an earlier one for the +same event and target is still pending or retrying. + +**Resubmit.** Replay recovers one delivery; **resubmit** re-injects one +EVENT. The event log offers a per-event **Resubmit** action that stores +a NEW event copying the stored one's `method`, `headers`, `body` and +`content_type` verbatim, then fans it out to the webhook's currently +**active** targets — resolved fresh by the same query the receiver +uses, so a target created long after the original event arrived +receives it. That is the difference that matters: a target added to +test a backend under development has no prior delivery, so there is +nothing to replay to it, while a resubmit reaches it like any other +active target. Inactive targets are skipped, exactly as the receiver +skips them. + +The new event is a first-class event in the log with its own +deliveries, not a marker on the one it came from, and the original's +deliveries are left untouched. It records `resubmitted_from_id`, and +the event log shows the relationship both ways, so a captured event +fired twenty times at a backend stays traceable. Resubmitting the same +event repeatedly is supported and is the point of the action — there is +no in-flight refusal; the route's own rate limit is what bounds it. + #### DeliveryResult The result of a single delivery attempt. Every attempt (including @@ -454,7 +1671,9 @@ retries) is individually logged for full observability. #### Common Fields -All entities include these fields from `BaseModel`: +Every entity except `Setting` includes these fields from `BaseModel`. +`Setting` is a bare key-value row with no `id`, no timestamps and no +soft delete: | Field | Type | Description | | ------------ | --------- | ----------- | @@ -471,6 +1690,11 @@ webhooker uses **separate SQLite database files**: a main application database for configuration data and per-webhook databases for event storage. All database files live in the `DATA_DIR` directory. +Every one of them is created `0600`, and so is each `-wal` and `-shm` +sidecar. See +[Running with Docker](#running-with-docker) for what that does and +does not protect. + **Main Application Database** (`{DATA_DIR}/webhooker.db`) — stores configuration and application state: @@ -500,7 +1724,7 @@ handles connection pooling, lazy opening, migrations, and cleanup. This separation provides: - **Isolation** — a high-volume webhook won't cause lock contention or - WAL bloat affecting the main application or other webhooks. + journal growth affecting the main application or other webhooks. - **Independent lifecycle** — event databases can be independently backed up, archived, rotated, or size-limited without impacting the application. @@ -509,10 +1733,15 @@ This separation provides: DB; the event database file is hard-deleted (permanently removed). - **Per-webhook retention** — the `retention_days` field on each webhook controls automatic cleanup of old events in that webhook's database - only. -- **Performance** — each webhook's database has its own WAL, its own - page cache, and its own lock, so concurrent event ingestion across - webhooks won't contend. + only, or disables cleanup entirely when set to `0` (retain forever). +- **Performance** — each webhook's database has its own page cache and + its own lock, so concurrent event ingestion across webhooks won't + contend. Every database — main, per-webhook, and archive — is opened + through one code path (`internal/database/sqlite_open.go`) in WAL + journal mode, with a 10-second busy timeout, `BEGIN IMMEDIATE` + transactions, and a bounded connection pool. Under WAL a reader never + blocks a writer, so an operator reading a database does not stall + event ingestion into it. The **database target type** builds on this architecture to provide long-term archiving, separate from the per-webhook event database (which @@ -531,6 +1760,36 @@ older than the expiry are pruned each time the archive is (re)opened. An archive write failure is never silent success: the delivery records a failed attempt with the error and is marked failed. +Because reopens only happen on writes, an archive belonging to a webhook +that has stopped receiving events would never be pruned. A background +**archive sweeper** closes that gap: on the same interval as the event +retention reaper (`RETENTION_SWEEP_INTERVAL`) it prunes every archive +whose database target declares a positive expiry, whether or not the +webhook is still receiving traffic. The sweep never creates an archive — +a webhook whose archive file does not yet exist is skipped, not +initialised — it takes the same per-webhook lock the write path uses, so +it can never interleave with a write, and it leaves the archive closed +afterwards so the move-the-file-away workflow keeps working. Archives +with no expiry, or the expiry `never`, are not touched by the sweep at +all. + +Note that a webhook has one archive file but may carry more than one +`database` target, each with its own `expiry`. The shortest expiry +configured on any of them therefore governs the whole archive, and the +sweep applies it whether or not the webhook is still receiving events. +Configure a single `database` target per webhook unless you intend that. + +Deleting a webhook releases its archive: the delivery engine's cached +archive writer is dropped and its file handle closed, so nothing lingers +after the webhook is gone. The archive **file itself is deliberately +left on disk**. Unlike the event database — per-webhook working storage +that is hard-deleted with the webhook — an archive is long-term storage +an operator may still want to keep or move away for offline retention, +and destroying it as a side effect of deleting a webhook would be +unrecoverable. Removing `archive-{webhookID}.db` is the operator's call. +Deleting a webhook's last `database` target releases the writer the same +way, and for the same reason leaves the file alone. + The **Slack target type** sends webhook events as formatted messages to any Slack-compatible incoming webhook URL (works with Slack, Mattermost, and other compatible services). Each message includes event metadata @@ -538,8 +1797,9 @@ and other compatible services). Each message includes event metadata pretty-printed in a code block. JSON payloads are automatically formatted with indentation for readability; non-JSON payloads are shown as raw text. Large payloads are truncated to keep messages reasonable. -Config stores `webhook_url` — the Slack/Mattermost incoming webhook -endpoint. +Config stores `webhookUrl` — the Slack/Mattermost incoming webhook +endpoint. That is the JSON key; the error text for a missing one reads +`webhook_url is required`, which is the message, not the key. The database uses the [modernc.org/sqlite](https://pkg.go.dev/modernc.org/sqlite) driver at @@ -558,12 +1818,15 @@ External Service │ │ │ Stack │ │ Handler │ └─────────────┘ └──────────────┘ └──────┬───────┘ │ - 1. Look up Entrypoint by UUID - 2. Capture full request as Event - 3. Create Delivery records for each active Target - 4. Build self-contained DeliveryTask structs - (target config + event data inline for ≤16KB) - 5. Notify Engine via channel (no DB read needed) + 1. Look up Entrypoint by UUID — 404 if unknown, + 410 if inactive + 2. Read the body under the 1 MB cap + 3. Capture full request as Event + 4. Create Delivery records for each active Target + 5. Build self-contained delivery.Task structs + (target config + event data inline for + bodies < 16 KiB) + 6. Notify Engine via channel (no DB read needed) │ ▼ ┌──────────────┐ @@ -597,7 +1860,7 @@ at any time, preventing goroutine explosions regardless of queue depth. a delivery channel (new tasks from the webhook handler) and a retry channel (tasks from backoff timers). Both are buffered to 10,000. - **Fan-out via channel, not goroutines:** When an event arrives with - multiple targets, each `DeliveryTask` is sent to the delivery channel. + multiple targets, each `delivery.Task` is sent to the delivery channel. Workers pick them up and process them — no goroutine-per-target. - **Worker goroutines:** A fixed number of worker goroutines select from both channels. Each worker processes one task at a time, then picks up @@ -621,7 +1884,12 @@ This means: - **Independent results** — each worker records its own delivery result in the per-webhook database without coordination. - **Graceful shutdown** — cancel the context, workers finish their - current task and exit. `WaitGroup.Wait()` ensures clean shutdown. + current task and exit. The stop hook waits for the pool via + `lifecycle.WaitForShutdown`, which bounds that wait by fx's stop + timeout rather than blocking forever on a wedged worker. On timeout + it logs at `ERROR` and returns an error, and the goroutines that + did not finish are still running — an unclean shutdown is reported + rather than hidden. **Recovery paths:** @@ -636,12 +1904,27 @@ This means: durable fallback that ensures no retry is permanently lost, even under extreme backpressure. -### Circuit Breaker (HTTP Targets with Retries) +**Changing a target's type does not migrate in-flight deliveries.** Only +`http` and `slack` targets own durable retries; `database` and `log` +targets are fire-and-forget and never produce a `retrying` delivery. If a +target's `type` is edited from a retrying type to a non-retrying (or +unknown) one while one of its deliveries is still `retrying`, both +recovery paths above terminally mark that delivery `failed` and record a +`DeliveryResult` naming the current target type as the reason, logging it +at warn level. The delivery is not re-dispatched under the new type — the +operator never asked for that delivery — and the event itself remains +stored in the per-webhook event database. Nothing is lost: the delivery +is `failed`, which is terminal, so the event log offers **Replay** on +it to re-send it to that target under the new type, and **Resubmit** on +the event to re-inject it to every currently active target. -HTTP targets with `max_retries` > 0 are protected by a **per-target circuit breaker** that -prevents hammering a down target with repeated failed delivery attempts. -The circuit breaker is in-memory only and resets on restart (which is -fine — startup recovery rescans the database anyway). +### Circuit Breaker (HTTP and Slack Targets with Retries) + +`http` and `slack` targets with `max_retries` > 0 are protected by a +**per-target circuit breaker** that prevents hammering a down target +with repeated failed delivery attempts. The circuit breaker is +in-memory only and resets on restart (which is fine — startup recovery +rescans the database anyway). **States:** @@ -677,29 +1960,638 @@ fine — startup recovery rescans the database anyway). - **Failure threshold:** 5 consecutive failures before opening - **Cooldown:** 30 seconds in open state before probing -**Scope:** Circuit breakers only apply to **HTTP targets with -`max_retries` > 0**. Fire-and-forget HTTP targets (`max_retries` == 0), -Slack targets, database targets (local operations), and log -targets (stdout) do not use circuit breakers. +**Scope:** Circuit breakers apply to **`http` and `slack` targets with +`max_retries` > 0**. The Slack target is built on the same HTTP core +and hands its own `max_retries` to the same retry path, so it gets a +breaker with the same 5-failure / 30-second defaults. Fire-and-forget +targets of either type (`max_retries` == 0), database targets (local +operations), and log targets (stdout) do not use circuit breakers. When a circuit is open and a new delivery arrives, the engine marks the delivery as `retrying` and schedules a retry timer for after the remaining cooldown period. This ensures no deliveries are lost — they're just delayed until the target is healthy again. +### Metrics + +`/metrics` serves one Prometheus registry behind basic auth (see +[Infrastructure Endpoints](#infrastructure-endpoints)). Alongside the +inbound HTTP metrics recorded by the middleware, it exposes the +delivery pipeline — the part of the service that can be failing while +the receive side looks perfectly healthy, because it is: events are +arriving and being stored, they are just not getting anywhere. + +| Metric | Type | Meaning | +| ------ | ---- | ------- | +| `webhooker_events_received_total` | counter | Events received and durably stored. Compare against the delivery counters on one dashboard | +| `webhooker_delivery_attempts_total` | counter | Delivery attempts actually dispatched to a target. A delivery an open circuit breaker refused is not one: it is counted as a retry instead | +| `webhooker_deliveries_succeeded_total` | counter | Deliveries that reached `delivered` | +| `webhooker_deliveries_failed_total` | counter | Deliveries that failed terminally and will not be retried | +| `webhooker_delivery_retries_total` | counter | Deliveries put back into `retrying` | +| `webhooker_delivery_replays_total` | counter | Deliveries an operator replayed from the event log. A replay runs the ordinary engine path, so it also moves the attempt, outcome and duration series; this is the only one that separates it from ordinary traffic | +| `webhooker_events_resubmitted_total` | counter | Stored events an operator re-injected from the event log. The new event also moves `webhooker_events_received_total`, since it is a stored event the delivery side is compared against; this counter is what separates the two. Unlabelled: the target types it fans out to belong to the delivery series | +| `webhooker_delivery_duration_seconds` | histogram | Wall time of a single dispatched delivery attempt, the same duration the attempt's `DeliveryResult` records | +| `webhooker_deliveries_pending` | gauge | Deliveries currently in `pending` | +| `webhooker_deliveries_retrying` | gauge | Deliveries currently in `retrying` | +| `webhooker_circuit_breakers_open` | gauge | Circuit breakers currently open | + +Every delivery metric carries exactly one label, `target_type`, and +cardinality is the whole reason for that restriction. The two +event-level counters carry no label at all — an event is not the +property of any one target type. A target type is +one of four compile-time constants, so the label domain is bounded by +construction; a value outside that set collapses to `unknown` rather +than minting a series of its own. Target ids, event ids and entrypoint +ids are deliberately not labels: they are UUIDs minted per operator +action or per inbound request, a series is never reclaimed once it +exists, and labelling by any of them would make `/metrics` a memory +leak that grows with traffic. + +The two queue-depth gauges are counted out of the databases by a +sampler that runs every 30 seconds for as long as the delivery engine +does, rather than tracked as deltas alongside the status transitions: a +delta would have to be seeded at startup from rows a previous process +wrote, and would drift permanently on any transition that failed to +persist. + +Those two gauges also publish an `unknown` series, from startup rather +than on first occurrence. Deliveries queued against a target that has +since been deleted are counted there: that backlog is the one nobody is +watching, so it is the one that must not silently vanish from the +gauge. The outcome counters move only after the status change has been +written, so a transition the database rejected is never reported as an +outcome that happened. + +#### Inbound HTTP metrics + +The middleware records three more on the same registry: + +| Metric | Type | Labels | +| ------ | ---- | ------ | +| `http_request_duration_seconds` | histogram | `service`, `handler`, `method`, `code` | +| `http_response_size_bytes` | histogram | `service`, `handler`, `method`, `code` | +| `http_requests_inflight` | gauge | `service`, `handler` | + +Two of those labels are written once per request from bytes the client +chose, so both are bounded to something this service registers: + +- `handler` is the chi route pattern — `/webhook/{uuid}`, never the + concrete path. A request matching no route carries `(unmatched)`, + and no entrypoint UUID ever reaches a label. +- `method` is the request method when the router can route it, and + `(unmatched)` otherwise. `net/http` accepts any RFC 9110 token as a + method, so the raw value bounds the label at nothing; the nine chi + matches routes for stay distinguishable, and a token that could only + ever have produced a 405 does not get a series of its own. + +The other two are not request-controlled: `code` is the status one of +this service's own handlers wrote, and `service` is a fixed empty +string. + +`http_requests_inflight` is deliberately aggregate — its `handler` is +always `(all)`, one series counting the requests in flight across the +whole service. The gauge is incremented before routing and decremented +after the handler returns, and the route pattern exists only between +those two moments, so labelling it by pattern would increment one +series and decrement another, leaving every pattern permanently off by +the number of requests it served. + ### Rate Limiting -Global rate limiting middleware (e.g., per-IP throttling applied at the -router level) **must not** apply to webhook receiver endpoints. Webhook -endpoints receive automated traffic from external services at -unpredictable rates, and blanket rate limits would cause legitimate -deliveries to be dropped. +Global blanket rate limiting middleware (e.g., a per-IP throttle shared +with the web UI) **must not** apply to webhook receiver endpoints. +Webhook endpoints receive automated traffic from external services at +unpredictable rates, and blanket limits shared with other routes would +cause legitimate deliveries to be dropped. -Instead, each webhook has its own individually configurable rate limit, -applied within the webhook handler itself. By default, no rate limit is -applied — webhook endpoints accept traffic as fast as it arrives. Rate -limits can be configured per-webhook when needed (e.g., to protect -against a misbehaving sender). +The receiver instead has its own dedicated abuse limit, scoped to the +`/webhook/{uuid}` route only and keyed per client IP per request path +(`httprate.KeyByEndpoint`): one misbehaving sender is throttled without +affecting other senders of the same entrypoint or the same sender's +other entrypoints. Keying on the path rather than on the entrypoint +matters — see the aggregate limit below. The limit is +`RECEIVER_RATE_LIMIT` requests per minute (default 120, generous for +legitimate webhook senders). Requests over the limit receive HTTP 429 +with a `Retry-After` header. A set-but-invalid `RECEIVER_RATE_LIMIT` +value aborts startup rather than silently falling back to the default. + +A second limit sits in front of that one, keyed on the client IP alone +and covering the whole route at ten times `RECEIVER_RATE_LIMIT` requests +per minute (default 1200). The per-entrypoint limit needs it: the route +pattern matches any single path segment, so a client that invents a +fresh path per request gets a fresh per-entrypoint bucket every time and +would otherwise have no aggregate limit at all — while each of those +requests still costs an entrypoint lookup before it 404s. The aggregate +limit leaves room for one address to drive several entrypoints at their +full rate, and it is not configurable separately. + +What that aggregate limit bounds is the database work an invented path +costs; log volume it caps rather than eliminates. A path that names no +entrypoint is recorded by the handler at `DEBUG`, and the aggregate +limiter logs its own rejections at `DEBUG` and without the path, so +neither appears at all under the default level. The per-entrypoint +limiter is the loud one: it logs every rejection at `WARN` with the +request path, which on this route is attacker-controlled text. A client +hammering a single invented path is served `RECEIVER_RATE_LIMIT` +requests and has the rest of its aggregate budget rejected there, so +the aggregate limit is what bounds the _number_ of those `WARN` lines — +to under ten times `RECEIVER_RATE_LIMIT` per minute per client IP, 1080 +at the defaults, where before it there was no bound at all. Their +_width_ is bounded by the field budgets below, the same ones the access +log spends. The access log is bounded by neither limit: every request +is recorded once at `INFO`, served or rejected alike. + +What the access log does bound is the _content_ of those lines. A 3xx +or 4xx response logs the chi route pattern — `/webhook/{uuid}`, +`/user/{username}//`, or the literal `(unmatched)` when the request hit +no route at all — in place of the concrete URL. Those are the outcomes +an unauthenticated client can drive for free: 404 and 429 on any +invented receiver path, a login redirect on any invented profile path. +Logging the URL there would let a flood write text of its own choosing, +at a length of its own choosing, into the log. 2xx and 5xx responses +keep the concrete path — a success resolved against a static route or +against the operator's own data (on the receiver, against a stored +entrypoint UUID), and a 5xx is a bug in this service, where the exact +path is the evidence and no client can provoke one at will. + +The query string is never logged; it is replaced by the fixed marker +`?(redacted)`. It is client-chosen on every route, and +`/.well-known/healthcheck` and `/s/*` answer 200 to anyone with no rate +limiter in front of them, so a query on a fixed 200 URL would otherwise +buy the same amplification as an invented path. Nothing debuggable is +lost: `page`, on the authenticated pagination links, is the only query +parameter this service reads. + +Client-supplied request content does not leave the host by the other +route either. The Sentry SDK attaches the request to every event it +captures, independently of the access log, and `SendDefaultPII=false` +does not cover all of what it copies: the raw query string and the +first 10 KiB of the request body are both taken unconditionally, the +body precisely because these handlers call `ParseForm`. A `BeforeSend` +hook therefore replaces the query string and the body with +`(redacted)`, drops cookies and the remote-address environment, and +reduces the headers to a fixed allowlist — `Accept`, `Content-Length`, +`Content-Type`, `Host`, `Origin`, `Referer`, `User-Agent` and +`X-Request-Id`. + +The same hook rewrites the request URL. The SDK builds it as +`scheme://host/path` from the concrete path, which on the receiver +route is `/webhook/` in full — and that UUID is a write +capability, not an identifier: anyone holding it can post events this +service accepts and its targets then deliver. A tracker has its own +retention, access control and deletion policy, so the rule the access +log follows above does not carry across that boundary. What is sent is +the chi route pattern instead: `http://host/webhook/{uuid}`. + +The scheme and the host are kept, and everything else in the URL is +discarded rather than edited, so a future SDK version that starts +appending a query string cannot widen this. The scheme has to survive +for the reason given below. The host is whatever the request's `Host` +header carried — this service validates no hostname, so on a directly +exposed deployment a client sets it — and that same header is on the +allowlist above, so scrubbing the host out of the URL would withhold +nothing that is not sent anyway. + +The body, the query string and the URL are all handled on every route +rather than filtered by route. For the URL that is also what keeps the +event locatable: an error event is grouped by its exception and stack +trace, not by its URL, so replacing the path with the pattern costs no +grouping and the pattern still names the route in the UI. And an +unconditional rule cannot leak on a route somebody forgets to add to +it, which a route-conditional one can. For the body there is a second +reason: nothing debuggable is lost, because every handler reads its +fields with `PostFormValue`, so the body is exactly where the +credentials are — the target destination URL, the login password, both +password-change fields — and the one route whose body is genuine +signal is the receiver, whose body is already stored on the event and +served from the UI, so a tracker is not where anyone reads it. + +The route is reachable from the hook only on the error dispatch. +`sentryhttp`'s recover path puts the request on the context it hands +to `RecoverWithContext`, and the SDK carries that context through to +`BeforeSend` as `hint.Context`, so +`hint.Context.Value(sentry.RequestContextKey)` yields the live request +and chi's `RoutePattern()` yields the matched pattern off it. The +transaction dispatch has no such request: a finished span captures +with a nil hint, which the client replaces with an empty one, so +`BeforeSendTransaction` sees no context at all. Tracing is off in this +service, so no transaction event is produced today, but the hook is +installed on both dispatches as a floor. + +Where the pattern is out of reach — the transaction dispatch, an event +captured outside the router, or a request that matched no route — the +fallback is never the concrete path. The path becomes the literal +`/(redacted)`, so the URL reads `http://host/(redacted)`; a URL the +rewrite cannot parse into a scheme is withheld whole. A transaction +event additionally carries the SDK's own `METHOD /path` name, built +from the concrete path as well; it is rewritten on the same terms, to +`POST /webhook/{uuid}` where the pattern is known and `POST +/(redacted)` where it is not. + +The headers are an allowlist for the same reason the rules above are +unconditional: the SDK's own filter removes four names and passes +everything else, which would ship `X-CSRF-Token` and the shared +secrets senders put on the receiver route. What survives still names +the failing route — scheme, host, route pattern, method — and +`X-Request-Id` ties the event to the local access log line that holds +the rest. Nothing dropped is needed for the likeliest use, debugging a +CSRF rejection. Its three inputs are the TLS decision, `Origin` and +`Referer`; the latter two are kept, and the first is the scheme of the +retained URL, because the SDK derives that scheme from +`r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https"`. That +predicate is the SDK's own and is stricter than `internal/reqtls.IsTLS`, +which this service now uses everywhere it decides transport: the SDK +reports `http` for the `HTTPS` and `https, http` spellings `reqtls` +accepts. Only a reported scheme is affected, no decision is, so it is +left to the SDK rather than reimplemented. That is what the rewrite +above preserves it for, and it is why dropping `X-Forwarded-Proto` +costs nothing. The dropped provider headers (`X-GitHub-Event`, +`X-Gitlab-Event` and the like) are real signal but are recorded +locally on the event, and +`Sentry-Trace`/`Baggage` are already reflected in the event's trace +context. + +The remaining client-supplied fields are truncated rather than dropped, +each to a fixed budget: 512 bytes for `url`, `useragent` and `referer`, +128 for `request_id` (chi passes an inbound `X-Request-Id` header +through), and 32 for `method`. A truncated `User-Agent` is still worth +reading; an absent one is not. A cut value ends in `[truncated]`, which +is charged on top of the budget rather than inside it. + +Each budget is spent in _encoded_ bytes, not in the bytes the client +sent. Every rune is charged what the wider of the two log handlers +emits for it: two bytes for a quotation mark, a backslash or a tab; six +for a non-printable rune below U+10000; ten for one at or above it, +which the text handler spells `\UXXXXXXXX`. Go's header parser accepts +all of them in a header value, so a budget counted raw would buy a +field several times its nominal size — and the line, not the header, is +what an operator has to store. Plain ASCII encodes one byte for one, so +a real browser's `User-Agent` still fits whole; a value built out of +escapes keeps a proportionally shorter prefix, which is the right +trade. + +Net: **one `INFO` line per request, of at most 2,560 bytes.** That +ceiling is arithmetic, not an observation: 3 × (512 + 11) for `url`, +`useragent` and `referer`, plus 128 + 11 for `request_id`, plus 32 + 11 +for `method`, plus a 336-byte fixed portion (the field names, the +punctuation, both timestamps at their longest, an IPv6 `remoteIP` with +a zone, the status and the latency) — 2,087 bytes, stated at 2,560 so +the figure has headroom. `internal/middleware/accesslog_test.go` +asserts it against 8 KB of client-chosen text in the path, in the +query, and in each of `User-Agent`, `Referer` and `X-Request-Id`, +including cases built from the characters the handlers escape, and +against the widest access log line the service can be made to write: a +5xx that keeps its concrete path while all three header fields are also +at their budget. Every case runs through both handlers +`internal/logger` can select — the JSON one and the text one it installs +on a tty — since the two do not escape alike and the ceiling is quoted +unqualified. Measured over a real connection, the widest access log line +is 1,972 bytes. + +Multiply that ceiling by the request rate to size log storage. Note +that the rate is not bounded by the limits above on every route: +`/.well-known/healthcheck` and `/s/*` sit behind no limiter, so there +the multiplier is whatever the deployment will serve. + +**The same ceiling covers every other line the service writes through +`slog` that carries text an unauthenticated client supplies**, with one +exception stated below it: the recovered-panic record, which carries a +whole goroutine stack alongside its client-supplied fields and so has +its own wider ceiling. The access log is not the only line a client can +put its own text into, and a budget that held for one line and not the +others would be worse than no stated budget at all. Every `slog` call an +unauthenticated request can reach spends the same per-field budget +through `internal/logfield`, and each carries strictly fewer +client-supplied fields than the access log does, so none of them can be +wider than it: + +| Log line | Level | Client-chosen value | Reachable unauthenticated | +| ------------------------------------------ | ------- | ------------------- | ----------------------------------------- | +| `request body exceeds limit` (413) | `WARN` | path, method | yes — `MaxBodySize` precedes `RequireAuth` | +| `csrf: token validation failed` (403) | `WARN` | path, method | yes — `CSRF` precedes `RequireAuth` | +| `... rate limit exceeded` (429) | `WARN` | path | yes, on the receiver | +| `auth middleware: unauthenticated request` | `DEBUG` | path, method | yes, by definition | +| `entrypoint not found` | `DEBUG` | entrypoint UUID | yes, on the receiver | +| `user not found` / `invalid password` | `DEBUG` | username | yes, on the login form | +| `login failure limit exceeded` (429) | `WARN` | path | yes, on the login form | +| `password verification capacity exhausted` | `WARN` | path | yes, on the login form | + +`DEBUG` being off by default is not a bound. An operator turning it on +to diagnose a flood must not thereby hand the flood an unbounded write, +so those lines are capped too. + +The last two rows are capped defensively rather than against a +demonstrated width: chi routes `POST /pages/login` on a static pattern, +so `r.URL.Path` there is the 12-byte constant `/pages/login` and each +line lands near 120 bytes. `RecordLoginFailure` is nonetheless an +exported method taking any `*http.Request`, and a future caller on a +route with a URL parameter would widen the line. Since no request +through the mux can, both caps are pinned by tests that call those two +entry points directly with the path such a caller would supply. +Removing either cap fails 14 subtests. + +`internal/middleware/logbound_test.go` and +`internal/handlers/logbound_test.go` drive 8 KB of client-chosen text +at each of these — 1 KB at `invalid password`, whose accounts are +shared with the successful-login line, where a username past 4 KB +overflows the session cookie and answers 500 before that line is +written — through both handlers, and through seven fills: plain text +as the baseline, and then the quotation mark, backslash, tab, newline, +C0 control and astral non-printable, six characters the wider of the +two handlers spends more on than the client spent sending them. Every +case holds each line to the 2,560-byte ceiling. That per-line ceiling +is what the figure above states, and every row establishes it. + +Three of the sites go further and bound the whole flood's output — the +total bytes a run of distinct invented values wrote, which is the +shape an operator sizing storage cares about. They are +`request body exceeds limit` +(`TestMaxBodySize_FloodOfOversizePathsDoesNotGrowTheLog`), +`entrypoint not found` and `user not found` (the last two through +`assertBoundedFlood`). The other rows carry no aggregate assertion; +the per-line ceiling is what they establish. + +`internal/logfield/logfield_test.go` measures the per-rune charge +against what the handlers really emit, over roughly 3,000 code points on +each, so an undercharged rune fails a test rather than quietly +falsifying the ceiling. + +**It covers GORM's statement logging as well.** GORM's own default +logger printed the fully interpolated SQL — parameters and all — to +standard output on every statement that returned an error, including a +plain record-not-found, at a level no operator setting reached. Two of +this service's lookups miss by design on unauthenticated routes: the +entrypoint lookup behind `/webhook/{uuid}` and the user lookup behind +the login form, whose path segment and submitted username the client +picks outright. Every +`gorm.Open` in the service now installs the adapter in +`internal/gormlog` instead. It writes through the same `slog` logger as +everything else, so its lines take the level the operator set and the +handler `internal/logger` selected, and every value it emits is spent +through the same `internal/logfield` budget. A record-not-found is not +logged as an error: it is the expected outcome on both of those paths, +and each handler already records its own miss at `DEBUG` — bounded, per +the table above — without the SQL. Slow statements are kept, at `WARN`, +above the same 200 ms threshold GORM used and with the statement +bounded, because that report is the one thing GORM's logger gave an +operator that nothing else here does. The adapter orders its cases +exactly as GORM's own `Trace` orders them, so a statement that both +missed and ran slow is still reported as slow, and dropping the miss +costs an operator no report `IgnoreRecordNotFoundError` would have +kept. A GORM line spends at most two of those budgets — the statement +and the driver error — against a smaller fixed portion than the access +log's, and `internal/gormlog/gormlog_test.go` asserts each line against +`MaxAccessLogLineBytes` directly rather than leaving it as arithmetic. + +The adapter also logs no bound value at all: it implements +`gorm.ParamsFilter` and discards the parameters, so GORM renders the +statement with its placeholders intact instead of substituting the +values into it. That is a separate property from the size bound and it +is what a bound is no substitute for — the session encryption key is 44 +base64 characters and an Argon2id hash under 100, so both fit inside +every budget above and a truncated secret is still a secret. It holds +on all three arms of `Trace`, including the routine one an operator +reaches at `DEBUG`, which is the only level at which a successful +`INSERT` is written at all. One GORM path does not consult the filter — +`(*gorm.DB).Scan`, which records the statement through GORM's own trace +recorder. No production code path calls it; its one caller is +`internal/database/database_test.go:91`, whose `SELECT 1` binds +nothing, and `internal/gormlog/scan_guard_test.go` fails if a non-test +file calls it. `Pluck`, `Row` and `Raw` all run through the normal +callback processor and are filtered. +See `#### What DEBUG=true exposes` under Configuration. + +What that ceiling does **not** cover, stated here so the figure is not +read as more than it is: + +- **Lines carrying an authenticated operator's own input**, which are + not truncated at all. `webhook created` logs the submitted `name` + verbatim and `target URL blocked by SSRF protection` logs the target + host (both `internal/handlers/source_management.go`), as do the + `target_name` lines in `internal/delivery/engine.go` and + `internal/delivery/target_http.go`. The only bound on any of them is + the 1 MB form body cap, so a 100 KB `name` writes a single line of + roughly 600 KB — measured. This is deliberate: every one of these + requires an authenticated operator on a service with no + self-registration, and truncating the operator's own configuration + echoed back would cost debuggability against no adversary. It does + mean the 2,560-byte figure sizes unauthenticated traffic, not the + operator's own administrative requests. +- The **`log` delivery target**, which writes the whole inbound event — + headers and body — to the log. This one is deliberate: capping it + would defeat the target, since emitting the payload is the delivery. + It costs nothing unless an authenticated operator creates a target of + that type on a specific webhook, and each line it writes is bounded + per event by the 1 MB receiver body cap. Adding one is a decision to + spend log volume on that webhook's payloads. +- **Two writers that do not go through `internal/logger` at all**, both + on standard error. `fx` prints the dependency graph and the lifecycle + hooks through its default console logger at startup and shutdown — + nothing calls `fx.WithLogger`, and `fx.New` builds that logger over + `os.Stderr`. The Go runtime writes a panic or a fatal error itself; a + panic in a background worker rather than in a request handler is the + case that reaches it, since nothing recovers those. Neither carries a + client-chosen value at a client-chosen length: the five `panic` calls + in this service are invariant guards over constants and over + `crypto/rand`. +- **`net/http`'s own faults**, which are _not_ a separate writer. + `internal/server/http.go` builds its server with a nil `ErrorLog`, so + `net/http` falls back to the `log` package's default logger — and + `internal/logger` calls `slog.SetDefault`, which redirects that logger + into whichever handler it installed. Those lines therefore arrive on + standard output, shaped like every other line, at `INFO`. They are not + truncated. A handler panic is no longer one of them: the recover + middleware below answers it and writes it as the bounded record + described there instead, and `internal/server/recoverer_test.go` + requires that `http: panic serving` appear in neither of the process's + two streams when a panic is driven through the production router. The + one panic still handed back to `net/http` is `http.ErrAbortHandler`, + which it special-cases and does not log at all. What is left on this + path is `net/http`'s own diagnostics, whose values are the runtime's, + not a client's. + +Wider than that 2,560-byte ceiling, and stated separately rather than +carved out of it: the record a recovered panic produces. The recover +middleware in `internal/middleware` answers `500` and writes one `ERROR` +record through `internal/logger` carrying the panic value, the stack and +the request id — the same `request_id` the access log line for that +request carries, which is how the two are joined. It replaced chi's +`middleware.Recoverer`, which on a current Go release crashed inside its +own stack pretty-printer: the connection was dropped rather than +answered, and what reached the operator described that crash rather than +the fault behind it. + +That record is bounded the same way, in the same encoded bytes and +through the same `internal/logfield` budget: 512 for the panic value, +because a handler is free to build one out of the request, 128 for the +request id, which a client supplies outright through `X-Request-Id`, +and 8,192 for the stack, cut at its far end so that the panic site +survives a cut and `net/http`'s accept frames are what is lost. Net: +**at most 10,240 bytes, once per recovered panic** — 9,121 by the +arithmetic (523 + 8,203 + 139 + a 256-byte fixed portion), stated at +10,240 for headroom. + +Those two numbers are the claim; the measurements below only +illustrate it. `internal/middleware/recoverer_test.go` drives all +three growable fields past their budgets on one record, over both +handlers, and measured 9,009 bytes on the JSON handler and +8,982–8,983 on the text one in one checkout. Neither is an invariant: +the stack's own content decides where its cut lands, so the figures +move by a byte or so between runs. The real case is far below both — +through the shipped middleware chain the whole record measures +roughly 3,960 bytes over a roughly 3,690-byte stack, taken by +`internal/server/recoverer_test.go` from the process's own file +descriptors while driving a panic through the production router over +a real server in a subprocess. That pair moves further still, since +`debug.Stack()` embeds absolute source paths and so depends on where +the tree is checked out: four checkouts have reported 3,959, 3,961, +3,984 and 4,026. What the tests assert is the ceiling, that every +client-supplied field was cut, and that the shipped chain's stack +arrived uncut — never the numbers. + +Every limiter here — receiver, login, and password change — identifies +the client the same way, through one shared key function: the +connection's own address, unless the peer is listed in +`TRUSTED_PROXIES`, in which case the forwarded client address is used +instead. That address becomes a bucket by family: IPv4 keys on the full +address, IPv6 on its `/64` prefix. A routed `/64` is the normal +residential and mobile IPv6 allocation, so keying IPv6 per address would +let one subscriber rotate source addresses and mint a fresh bucket per +request, evading these limits at the network layer without spoofing +anything; the cost is that distinct clients inside one `/64` share a +bucket. IPv4-mapped addresses (`::ffff:1.2.3.4`) key as the IPv4 address +they carry. See [Trusted proxies](#trusted-proxies). Deployed without that +variable set, a client behind a reverse proxy shares one bucket with +every other client behind the same proxy. Set `TRUSTED_PROXIES` to the +proxy's address to get per-client limits back. What the shared bucket +costs is not the same for every limiter, and the two cases pull in +opposite directions: + +- For the **receiver** limits it costs throughput, which is the safe + direction to be wrong in: sharing can only make a limit bind sooner, + never let a sender past it. It matters more for the aggregate limit + than for the per-entrypoint one: with `TRUSTED_PROXIES` unset behind + the reverse proxy a production deployment is required to run behind, + every request keys on the proxy, so the aggregate limit becomes a + service-wide ceiling of 1200 requests per minute across all senders + and all entrypoints, where the per-entrypoint limit's capacity still + grows with the number of entrypoints. Any deployment with more than a + handful of busy entrypoints must set `TRUSTED_PROXIES`. +- For the **login and password-change** limits it costs precision, not + availability. Login failures from every client land in one counter, + so a stranger's wrong passwords make the operator's own wrong + passwords answer `429` sooner; the operator's _correct_ password is + never affected, because it is never counted. Production deployments + should still set `TRUSTED_PROXIES`; webhooker warns at startup + whenever it is empty, in any environment. + +#### The login endpoint + +The login `POST` is the one endpoint with no pre-emptive limiter in +front of it, and that is deliberate. A limiter that spends budget on +arrival is a lockout in this deployment shape: sharing one bucket, a +stranger sending five POSTs a minute — about 0.08 requests per second, +from anywhere — keeps it permanently full, and the operator has no +second administrative path. So the handler inverts the order: + +1. **Credentials are verified first, and only a failed attempt spends + budget.** A correct password is never rate-limited, whatever the + counters hold. This is what guarantees the admin UI stays + reachable. +2. **Failures are counted per (client bucket, submitted username)**, + five per minute, after which further _failures_ from that pair are + answered `429` with a `Retry-After`. That `429` is a label on the + response, not a gate in front of the work: the credential check has + already run by the time the counter is consulted, so a throttled + client's guess is still evaluated. See the guessing rate below. A + successful login clears the counter, so mistyping a few times and + then getting it right leaves you unthrottled. Because the submitted + username is attacker-controlled, at most 1024 username counters and + 1024 fallback address counters are tracked; past the first cap + failures fall back to the address counter, and past both they are + answered as throttled without being recorded. Total tracked state + is under half a megabyte and does not grow with the number of + usernames an attacker invents. +3. **Concurrent password verifications are capped at two, and the + queue for them at 16.** Verifying before counting means every login + request costs an Argon2id hash, and Argon2id here is 64 MB per + hash — two slots is a 128 MB ceiling on password hashing. Every + endpoint that hashes a password takes a slot, including the + password-change endpoint, which holds one across both the + verification and the new hash. A request that waits five seconds + without getting a slot is answered `503 Service Unavailable` and no + hash is computed for it. The wait alone does not bound memory, only + how long one request holds some, so the number of waiters is capped + as well. Size the queue from what a parked waiter actually retains, + not from the 1 MB body cap: that caps the raw body read, while the + body-cap, CSRF and form-parsing middleware all run before the + guard, so a waiter holds its parsed form plus its request header + block for the whole wait. Measured on the pinned Go 1.26.1 + toolchain, as the heap delta with 64 waiters parked in the handler, + an ordinary two-field login form retains ~0 MB, a 1 MB urlencoded + body at Go's 10,000-parameter parse cap retains 2.82 MB (3.09 MB + with `%41` escapes), and the ~0.9 MB of headers the 1 MB header cap + allows takes it to **4.18 MB** — the retained parse and the headers + dominate, not the raw body. So the cap is 16 waiters: 16 x 4.18 MB + is about 67 MB of committed queue memory, and two slots drain a + full 16-deep queue in roughly 0.6 s, far inside the five-second + deadline. A request arriving past the cap is shed with `503` + immediately instead of joining the queue. **Peak commitment for the + endpoint is therefore about 203 MB**: 128 MB of Argon2id, plus the + 18 requests holding a parsed form — 16 queued and the 2 being + hashed — at about 75 MB. That 203 MB is _live_ commitment, not + resident size: the Go collector lets the heap reach roughly twice + the live set before collecting, with transient parse garbage on top + of it. The independent review of this endpoint fired 18 adversarial + requests at an idle guard and measured a peak `HeapAlloc` of + 392 MB. **Provision on the order of 400 MB**, not for the 203 MB + itemised here and not for the hashing budget alone. + +An unknown username is verified against a dummy hash rather than +rejected early, so a nonexistent account costs the same time as a real +one and the response cannot be used to enumerate usernames. + +**This raises online guessing throughput by about 300x, and that is +the trade.** Because the credential check always precedes the counter, +what bounds online brute force is the semaphore, not the failure +counter. Two slots at the cost of one Argon2id verification is on the +order of **27 guesses per second, about 2.3 million per day**, against +5 per minute under the pre-emptive limiter this replaced. Treat that +figure as a lower bound rather than a ceiling: it was measured with +Go's race detector enabled, so real hardware verifies faster and +guesses faster. Choose the admin password to survive millions of +online guesses per day — a long random passphrase, not a memorable +one. Rate-limiting `POST /pages/login` at the reverse proxy, where the +real client address is visible, is the way to put a cheaper bound back +on top. + +The residual exposure is a bounded, self-clearing loss of login +**availability** — not merely of latency. A flood can keep both +verification slots busy, and a request that neither gets a slot within +five seconds nor finds room in the queue is answered `503`. Above +roughly 27 requests per second the operator is not served slowly, it +is shed: its chance per attempt is about the ratio of service rate to +flood rate, so at 400 requests per second it is roughly one attempt in +fourteen. A sufficiently determined flood still denies login for as +long as it runs. + +What changed is the price and the aftermath. Denying login used to +cost an attacker 0.08 requests per second from anywhere; it now costs +30 or more sustained, about 400 times as much. Nothing accumulates +while the flood runs, nothing needs resetting when it stops, and the +operator's correct password succeeds on the first attempt afterwards. +Restarting the service is **not** a remedy: a restart clears the +failure counters, which are not what is saturated, and the flood +re-fills both verification slots on its first two requests. The +remedies are to block the source at the reverse proxy, or to +rate-limit `POST /pages/login` there — the one place a limit can be +applied without reintroducing the lockout, because the proxy sees the +real client address. Setting `TRUSTED_PROXIES` does not stop the +saturation, but it makes the source visible in the failure logs. + +Finer-grained per-webhook rate limits (configured in the web UI and +enforced in the webhook handler) can layer on top of this env-level +abuse limit later; they are tracked as future work. ### API Endpoints @@ -707,17 +2599,17 @@ against a misbehaving sender). | Method | Path | Description | | ------ | --------------------------- | ----------- | -| `GET` | `/` | Root redirect (authenticated → `/sources`, unauthenticated → `/pages/login`) | -| `GET` | `/.well-known/healthcheck` | Health check (JSON: status, uptime, version) | -| `GET` | `/s/*` | Static file serving (embedded CSS, JS) | -| `ANY` | `/webhook/{uuid}` | Webhook receiver endpoint (accepts all methods) | +| `GET` | `/` | Root redirect, 303 (authenticated → `/sources`, unauthenticated → `/pages/login`) | +| `GET` | `/.well-known/healthcheck` | Health check (JSON: `status`, `now`, `uptimeSeconds`, `uptimeHuman`, `version`, `appname`, `maintenanceMode`) | +| any | `/s/*` | Static file serving (embedded CSS, JS). Mounted for every method, not just `GET`/`HEAD`: chi's `Mount` registers all methods and `http.FileServer` special-cases only `HEAD` (by omitting the body), so a `POST` or `DELETE` to an asset is answered `200` with the file. Pinned by `TestStaticServesEveryMethod` | +| `POST` | `/webhook/{uuid}` | Webhook receiver endpoint. `POST` only — every other method is answered `405 Method Not Allowed` with `Allow: POST`. Rate limited (see [Rate Limiting](#rate-limiting)) | #### Authentication Endpoints | Method | Path | Description | | ------ | --------------- | ----------- | -| `GET` | `/pages/login` | Login page | -| `POST` | `/pages/login` | Login form submission | +| `GET` | `/pages/login` | Login page (not rate limited) | +| `POST` | `/pages/login` | Login form submission. Credentials are verified before any limit is consulted, so a correct password is never throttled; 5 FAILED attempts per minute per bucket per submitted username, then `429`. `503` if no verification slot frees up within 5s, or immediately if 16 requests are already queued for one (see [Rate Limiting](#rate-limiting)) | | `POST` | `/pages/logout` | Logout (destroys session) | #### Authenticated Endpoints @@ -725,6 +2617,7 @@ against a misbehaving sender). | Method | Path | Description | | ------ | ------------------------ | ----------- | | `GET` | `/user/{username}` | User profile page | +| `POST` | `/user/{username}/password` | Change the user's password (5 per minute per bucket, then `429`; `503` if no verification slot frees up within 5s, or immediately if 16 requests are already queued for one) | | `GET` | `/sources` | List user's webhooks | | `GET` | `/sources/new` | Create webhook form | | `POST` | `/sources/new` | Create webhook submission | @@ -733,14 +2626,23 @@ against a misbehaving sender). | `POST` | `/source/{id}/edit` | Edit webhook submission | | `POST` | `/source/{id}/delete` | Delete webhook | | `GET` | `/source/{id}/logs` | Webhook event logs | +| `GET` | `/source/{id}/logs/{eventID}/body` | Download an event's full stored body. The log page renders each body only up to its cap, so this is the only route that serves a whole one; it is offered wherever a body is shown truncated | +| `POST` | `/source/{id}/deliveries/{deliveryID}/replay` | Replay a finished delivery: creates a new delivery for the same event against the target's current configuration (30 per minute per bucket, then `429`) | +| `POST` | `/source/{id}/events/{eventID}/resubmit` | Resubmit a stored event: creates a new event copying it and fans that out to every currently active target (30 per minute per bucket, then `429`) | | `POST` | `/source/{id}/entrypoints` | Add entrypoint to webhook | +| `POST` | `/source/{id}/entrypoints/{entrypointID}/delete` | Delete an entrypoint | +| `POST` | `/source/{id}/entrypoints/{entrypointID}/toggle` | Enable or disable an entrypoint | | `POST` | `/source/{id}/targets` | Add target to webhook | +| `GET` | `/source/{id}/targets/{targetID}/edit` | Edit target form. The one page that renders a target's destination URL and header values in full, rather than masked | +| `POST` | `/source/{id}/targets/{targetID}/edit` | Edit target submission | +| `POST` | `/source/{id}/targets/{targetID}/delete` | Delete a target | +| `POST` | `/source/{id}/targets/{targetID}/toggle` | Enable or disable a target | #### Infrastructure Endpoints | Method | Path | Description | | ------ | ---------- | ----------- | -| `GET` | `/metrics` | Prometheus metrics (requires basic auth) | +| `GET` | `/metrics` | Prometheus metrics, behind basic auth. The route is registered only when `METRICS_USERNAME` and `METRICS_PASSWORD` are both set; with neither set it does not exist and returns 404, and with only one set the process refuses to start | #### API (Planned) @@ -754,8 +2656,10 @@ against a misbehaving sender). | `GET` | `/api/v1/webhooks/{id}/events` | List events for webhook | | `POST` | `/api/v1/events/{id}/redeliver`| Redeliver an event | -API authentication will use API keys passed via `Authorization: Bearer -` header. +None of these exist yet. `/api/v1` is mounted with no routes, so every +path under it returns 404 today. API authentication will use API keys +passed via `Authorization: Bearer ` header; no Bearer middleware +is implemented either. ### Package Layout @@ -765,8 +2669,14 @@ imports. The entry point is `cmd/webhooker/main.go`. ``` webhooker/ ├── cmd/webhooker/ -│ └── main.go # Entry point: sets globals, wires fx +│ └── main.go # Entry point: subcommand dispatch; no args locks DATA_DIR and wires fx ├── internal/ +│ ├── banner/ +│ │ └── banner.go # Ruled block for the one credential shown in the clear +│ ├── ciscript/ +│ │ └── doc.go # Tests for the CI shell scripts in script/; no runtime code +│ ├── resetpw/ +│ │ └── resetpw.go # `webhooker resetpw`: set an account's password, stopped deployments only │ ├── config/ │ │ └── config.go # Configuration loading from environment variables │ ├── database/ @@ -783,16 +2693,38 @@ webhooker/ │ │ ├── model_delivery_result.go # DeliveryResult entity (per-webhook DB) │ │ ├── model_apikey.go # APIKey entity │ │ ├── password.go # Argon2id hashing and verification +│ │ ├── retention.go # Retention reaper (per-webhook event expiry) +│ │ ├── testing.go # NewTestDatabase: wrapper for tests, no fx lifecycle │ │ └── webhook_db_manager.go # Per-webhook DB lifecycle manager +│ ├── datadir/ +│ │ └── lock.go # Exclusive advisory lock on DATA_DIR (one instance) │ ├── globals/ │ │ └── globals.go # Build-time variables (appname, version, arch) +│ ├── gormlog/ +│ │ └── gormlog.go # GORM's logger.Interface on top of slog, bounded +│ ├── logfield/ +│ │ └── logfield.go # Encoded-byte budget for client-supplied log values │ ├── delivery/ │ │ ├── engine.go # Event-driven delivery engine (channel + timer based) -│ │ ├── circuit_breaker.go # Per-target circuit breaker for HTTP targets with retries +│ │ ├── circuit_breaker.go # Per-target circuit breaker for http/slack targets with retries +│ │ ├── target.go # Target interface, Task, Scheduler +│ │ ├── target_http.go # HTTP target (retries, circuit breaker) +│ │ ├── target_slack.go # Slack/Mattermost incoming-webhook target +│ │ ├── target_database.go # Database archive target +│ │ ├── target_database_archive.go # Archive file lifecycle and pruning +│ │ ├── target_log.go # Log target (stdout) +│ │ ├── target_config_view.go # Masked target config for templates +│ │ ├── archive_sweeper.go # Periodic pruning of idle archives +│ │ ├── queue_depth.go # Periodic sampler behind the queue-depth gauges +│ │ ├── url_mask.go # Strips credentials from *url.Error │ │ └── ssrf.go # SSRF prevention (IP validation, safe HTTP transport) │ ├── handlers/ │ │ ├── handlers.go # Base handler struct, JSON helpers, template rendering │ │ ├── auth.go # Login, logout handlers +│ │ ├── delivery_replay.go # Per-delivery replay: new delivery, current target config +│ │ ├── event_resubmit.go # Event resubmit: new event, all currently active targets +│ │ ├── entrypoint_view.go # Masked entrypoint view for templates +│ │ ├── event_log_view.go # Event log projection, byte-capped in SQL │ │ ├── healthcheck.go # Health check handler │ │ ├── index.go # Index page handler │ │ ├── profile.go # User profile handler @@ -800,25 +2732,42 @@ webhooker/ │ │ └── webhook.go # Webhook receiver handler │ ├── healthcheck/ │ │ └── healthcheck.go # Health check service (uptime, version) +│ ├── lifecycle/ +│ │ └── lifecycle.go # Shared stop-hook waiter, bounded by the stop context │ ├── logger/ │ │ └── logger.go # slog setup with TTY detection +│ ├── metrics/ +│ │ └── metrics.go # Delivery Prometheus collectors, labelled by target type │ ├── middleware/ │ │ ├── middleware.go # Logging, CORS, Auth, Metrics, MetricsAuth, SecurityHeaders, MaxBodySize │ │ ├── csrf.go # CSRF protection middleware (gorilla/csrf) -│ │ └── ratelimit.go # Per-IP rate limiting middleware (go-chi/httprate) +│ │ ├── ratelimit.go # Per-IP rate limiting middleware (go-chi/httprate) +│ │ ├── loginguard.go # Login failure counters and the Argon2id verification semaphore +│ │ └── testing.go # NewForTest: Middleware without the fx lifecycle +│ ├── reqtls/ +│ │ └── reqtls.go # IsTLS: the one TLS predicate, r.TLS or X-Forwarded-Proto │ ├── server/ │ │ ├── server.go # Server struct, fx lifecycle, signal handling │ │ ├── http.go # HTTP server setup with timeouts │ │ └── routes.go # All route definitions -│ └── session/ -│ └── session.go # Cookie-based session management +│ ├── session/ +│ │ ├── session.go # Cookie-based session management +│ │ └── testing.go # NewForTest: Session without the fx lifecycle +│ └── versionscript/ +│ └── doc.go # Tests for script/version and the build files that use it ├── static/ │ ├── static.go # //go:embed directive -│ ├── css/style.css # Custom stylesheet (system font stack, card effects, layout) -│ └── js/app.js # Client-side JavaScript (minimal bootstrap) -├── templates/ # Go HTML templates (base, index, login, etc.) -├── Dockerfile # Multi-stage: lint, build+test, then Alpine runtime -├── Makefile # fmt, lint, test, check, build, docker targets +│ ├── css/input.css # Tailwind input, source for tailwind.css (make css) +│ ├── css/tailwind.css # Generated stylesheet the pages load +│ ├── css/style.css # Older hand-written stylesheet, no longer loaded +│ ├── js/app.js # Progressive-enhancement copy-to-clipboard +│ ├── js/alpine.min.js # Alpine.js, fetched by script/fetch-assets, not committed +│ └── vendor.sha256 # Pinned hashes the fetched assets are verified against +├── templates/ # Go HTML templates (base, login, sources, etc.) +├── script/ # Scripts to Rule Them All entrypoints +├── Dockerfile # Three stages: lint, test+build, Alpine runtime +├── Dockerfile.lint # Lint-only image built by script/lint +├── Makefile # 10 of 17 targets shim script/; 7 are inline ├── go.mod / go.sum └── .golangci.yml # Linter configuration ``` @@ -834,102 +2783,365 @@ Components are wired via Uber fx in this order: user seed 5. `database.NewWebhookDBManager` — Per-webhook event database lifecycle manager -6. `healthcheck.New` — Health check service -7. `session.New` — Cookie-based session manager (key from database) -8. `handlers.New` — HTTP handlers -9. `middleware.New` — HTTP middleware -10. `delivery.New` — Event-driven delivery engine -11. `delivery.Engine` → `handlers.DeliveryNotifier` — interface bridge -12. `server.New` — HTTP server and router +6. `database.NewRetentionReaper` — Per-webhook event retention sweep +7. `healthcheck.New` — Health check service +8. `session.New` — Cookie-based session manager (key from database) +9. `handlers.New` — HTTP handlers +10. `middleware.New` — HTTP middleware +11. `delivery.New` — Event-driven delivery engine +12. `delivery.NewArchiveSweeper` — Periodic pruning of idle archives +13. `delivery.Engine` → `delivery.Notifier` — interface bridge +14. `delivery.Engine` → `delivery.WebhookEvictor` — interface bridge so + deleting a webhook releases its archive writer +15. `server.New` — HTTP server and router -The server starts via `fx.Invoke(func(*server.Server, *delivery.Engine) -{})` which triggers the fx lifecycle hooks in dependency order. The -`DeliveryNotifier` interface allows the webhook handler to send -self-contained `DeliveryTask` slices to the engine without a direct +The server starts via `fx.Invoke(func(*server.Server, *delivery.Engine, +*database.RetentionReaper, *delivery.ArchiveSweeper) {})`, which +triggers the fx lifecycle hooks in dependency order. The +`delivery.Notifier` interface allows the webhook handler to send +self-contained `delivery.Task` slices to the engine without a direct package dependency. Each task carries all target config and event data -inline (for bodies ≤16KB), so the engine can deliver without reading -from any database — it only writes to record results. +inline (for bodies under 16 KiB, `delivery.MaxInlineBodySize`), so the +engine can deliver without reading from any database — it only writes +to record results. ### Middleware Stack Applied to all routes in this order: -1. **Recoverer** — Panic recovery (chi built-in) -2. **RequestID** — Generate unique request IDs (chi built-in) -3. **SecurityHeaders** — Production security headers on every response +1. **RequestID** — Generate unique request IDs (chi built-in) +2. **SecurityHeaders** — Production security headers on every response (HSTS, X-Content-Type-Options, X-Frame-Options, CSP, Referrer-Policy, Permissions-Policy) -4. **Logging** — Structured request logging (method, URL, status, +3. **Logging** — Structured request logging (method, URL, status, latency, remote IP, user agent, request ID) -5. **Metrics** — Prometheus HTTP metrics (if `METRICS_USERNAME` is set) -6. **CORS** — Cross-origin resource sharing headers -7. **Timeout** — 60-second request timeout +4. **Metrics** — Prometheus HTTP metrics (if `METRICS_USERNAME` and + `METRICS_PASSWORD` are both set) +5. **CORS** — Cross-origin resource sharing headers +6. **Timeout** — 60-second request timeout +7. **Recoverer** — Panic recovery: one `ERROR` record through + `internal/logger` and a `500` 8. **Sentry** — Error reporting to Sentry (if `SENTRY_DSN` is set; configured with `Repanic: true` so panics still reach Recoverer) -Additionally, form endpoints (`/pages`, `/sources`, `/source/*`) apply a -**MaxBodySize** middleware that limits POST/PUT/PATCH request bodies to -1 MB using `http.MaxBytesReader`, preventing oversized form submissions. +Recoverer sits seventh rather than first, and both neighbours are the +reason. It runs **inside** everything that observes the response, so +the `500` it writes for a panicking handler is the status the access +log records and the metrics count; registered first, as chi's own +`middleware.Recoverer` was, the same request was logged as a `200` that +the client never received. It runs **outside** the Sentry handler, so +`Repanic: true` has something to re-raise into: an operator with +`SENTRY_DSN` set keeps the report, and one without it now gets the +local record instead of nothing. What that placement gives up is +recovery of a panic in the six entries above it, none of which does +more than set a header or start a timer. + +Additionally, form endpoints (`/pages`, `/user/*`, `/sources`, +`/source/*`) apply a **MaxBodySize** middleware that limits +POST/PUT/PATCH request bodies to 1 MB. It is registered ahead of the +CSRF middleware in every one of those route groups, because +gorilla/csrf parses the form; if the cap were installed after it, form +parsing would run under net/http's 10 MB default and the 1 MB limit +would never apply. A request that declares a `Content-Length` over the +limit is answered with `413 Request Entity Too Large` without its body +being read and without reaching CSRF, the route group's remaining +middleware, or the handler. It is not rejected before *any* other +middleware, though: the global entries listed above all run first, so +such a request is still logged and given the security headers — and +counted in the metrics, on a deployment where the `/metrics` +credentials are set and the Metrics middleware is therefore registered +at all. The +rejection itself is logged at `WARN` with the method, path and +declared length. A chunked request, or +one that lies about its length, is hard-capped by +`http.MaxBytesReader` and fails downstream at form-parse time. + +Those same four route groups then apply **CSRF** and **NoCache** +(`Cache-Control: no-store`, `Pragma: no-cache`), and every group except +`/pages` applies **RequireAuth**. The rate limiters are per-route +rather than global: **PasswordChangeRateLimit** on +`/user/{username}/password` and **ReceiverRateLimit** on +`/webhook/{uuid}`. There is deliberately none on `/pages/login` — that +endpoint counts failures inside the handler, after the credential +check, see [The login endpoint](#the-login-endpoint). ### Authentication - **Web UI:** Cookie-based sessions using gorilla/sessions with encrypted cookies. Sessions are configured with HttpOnly, SameSite - Lax, and Secure (in production). Session lifetime is 7 days. + Lax, and Secure whenever the request is on TLS — the flag follows the + request's transport, not the environment. Absolute session lifetime + is 7 days, with a sliding idle timeout on top of it (see + [Sessions](#sessions)). - **API (planned):** API key authentication via `Authorization: Bearer` header. API keys are stored per-user with usage tracking (`last_used_at`). - **Metrics:** Basic authentication protecting the `/metrics` endpoint. +- **Recovery:** `webhooker resetpw ` on a stopped deployment + is the only way back into an account whose password was lost (see + [Recovering a lost admin password](#recovering-a-lost-admin-password)). ### Security - Passwords hashed with Argon2id (64 MB memory cost) -- Session cookies are HttpOnly, SameSite Lax, Secure (prod only) +- Session cookies are HttpOnly, SameSite Lax, and Secure on any request + that arrived over TLS (directly or through a reverse proxy reporting + it), decided per-request by `internal/reqtls.IsTLS` rather than by the + configured environment - Session regeneration on login to prevent session fixation attacks - Session key is a 32-byte value auto-generated on first startup and stored in the database - Production security headers on all responses: HSTS, X-Content-Type-Options (`nosniff`), X-Frame-Options (`DENY`), Content-Security-Policy, Referrer-Policy, and Permissions-Policy -- Request body size limits (1 MB) on all form POST endpoints +- Request body size limits (1 MB) on all form POST endpoints, enforced + by middleware that runs before CSRF parses the form - **CSRF protection** via [gorilla/csrf](https://github.com/gorilla/csrf) on all state-changing forms (cookie-based double-submit tokens with HMAC authentication). Applied to `/pages`, `/sources`, `/source`, and `/user` routes. Excluded from `/webhook` (inbound webhook POSTs) and - `/api` (stateless API). The middleware auto-detects TLS status - per-request (via `r.TLS` and `X-Forwarded-Proto`) to set appropriate - cookie security flags and Origin/Referer validation mode + `/api` (stateless API). The middleware detects TLS per-request through + `internal/reqtls.IsTLS` — the same predicate the session cookie uses — + to set appropriate cookie security flags and Origin/Referer validation + mode +- **The entrypoint URL is the receiver's only credential.** Nothing + about an inbound request is verified; possession of the UUID + authorises submission (see + [The entrypoint URL is the authentication secret](#the-entrypoint-url-is-the-authentication-secret)) - **SSRF prevention** for HTTP delivery targets: private/reserved IP ranges (RFC 1918, loopback, link-local, cloud metadata) are blocked both at target creation time (URL validation) and at delivery time (custom HTTP transport with SSRF-safe dialer that validates resolved - IPs before connecting, preventing DNS rebinding attacks) -- **Login rate limiting** via [go-chi/httprate](https://github.com/go-chi/httprate): - per-IP sliding-window rate limiter on the login endpoint (5 POST - attempts per minute per IP) to prevent brute-force attacks + IPs before connecting, preventing DNS rebinding attacks). Both paths + route through a single decision function, so they cannot disagree + about a destination. An operator can permit specific blocks with + [`ALLOWED_EGRESS_CIDRS`](#allowing-egress-to-your-own-network); the + guard cannot be switched off, and link-local plus a + [pinned set](#allowing-egress-to-your-own-network) of known cloud + metadata endpoints — several of which are ULAs outside link-local — + stay blocked whatever is listed, though listing `0.0.0.0/0` or + `::/0` does open every other private range +- **Login limiting is inverted, deliberately.** The login `POST` has + no pre-emptive rate limiter in front of it. Credentials are + verified first and only a _failed_ attempt spends budget, so a + correct password is never throttled and no flood of wrong ones can + deny the operator the only administrative path. Failures are + counted per (bucket, submitted username), five per minute, after + which further failures are answered `429` with a `Retry-After`. + What bounds brute force is not that counter but the cap of two + concurrent Argon2id verifications: a throttled client's guess is + still evaluated, so roughly 27 guesses a second get through and the + admin password has to carry that load (see + [The login endpoint](#the-login-endpoint)). `GET` requests to the + login page are not limited +- **Password-change rate limiting** via [go-chi/httprate](https://github.com/go-chi/httprate): + sliding-window rate limiter, 5 POST attempts per minute per bucket. + It runs behind session auth, so only a client already holding a + valid session reaches it, and an operator throttled out of changing + a password can still log in. The bucket is per client IP only when + `TRUSTED_PROXIES` names the reverse proxy; unset, every client + shares one bucket, which costs precision rather than availability + (see [Rate Limiting](#rate-limiting)). webhooker warns at startup + whenever `TRUSTED_PROXIES` is empty - Prometheus metrics behind basic auth - Static assets embedded in binary (no filesystem access needed at runtime) - Container runs as non-root user (UID 1000) -- GORM soft deletes on all entities (data preserved for audit) +- GORM soft deletes on every entity that carries `BaseModel`, which is + all of them but `Setting` (data preserved for audit) + +### Shutdown + +On SIGINT or SIGTERM, fx runs the registered stop hooks in reverse +dependency order under a **5 second budget** (`fx.StopTimeout` in +`cmd/webhooker/main.go`). That budget covers the whole sequence, not +each hook. The order, read off the fx stop-hook log: + +1. `ArchiveSweeper` +2. `RetentionReaper` +3. `server` — the HTTP drain, bounded separately by + `server.ShutdownTimeout` (**3 seconds**), then a Sentry flush if + `SENTRY_DSN` is set +4. `delivery.Engine` +5. `healthcheck` +6. `WebhookDBManager` +7. the database close + +The two components that can realistically hold the budget run +first: a retention sweep or an archive prune caught mid-tick each +waits on its `WaitGroup` bounded by the stop context, so a wedge +there consumes the 5 seconds before the HTTP server hook is ever +entered. The hooks after the server are microsecond-scale in normal +operation. + +The HTTP drain budget is deliberately **shorter** than the sequence +budget. Were the two equal, a drain that used its whole budget would +exhaust the sequence budget at the instant it finished, and every +later hook — the delivery engine, the healthcheck, the webhook DB +manager and the database close — would be skipped in exactly the +case where the drain mattered. 3 seconds leaves 2 seconds +(`server.TailHookReserve`) for the tail, which is far more than the +microseconds it needs. + +That reserve belongs to the tail hooks, not to the server hook, and +the Sentry flush is what could take it: it runs after the drain +**inside the same hook**, and `sentry.Flush` takes a bare duration +and honours no context, so an unreachable Sentry endpoint would add +its own timeout on top of a full-length drain and consume the whole +sequence budget by itself. It is therefore clamped to whatever is +left on the stop context minus the reserve, and skipped when that +leaves too little to be worth attempting — so a full-length drain +means Sentry events are dropped rather than the database close being +skipped. + +This does not make the database close unconditional: a wedged +`ArchiveSweeper` or `RetentionReaper` still runs first and can +consume the whole budget on its own. + +The value is chosen to sit inside the container stop grace period. +Docker's default `docker stop` grace is 10 seconds and the Dockerfile +sets no `STOPSIGNAL` or grace override, so the process must be gone +before that. fx's own default is 15 seconds, which is past the grace: +the container would be SIGKILLed (exit 137) before the bound could +fire, and nothing that depends on it — including the +`shutdown timed out, goroutines still running` error log that tells +an operator a component is wedged — would ever be reached. + +Two operational consequences follow from bounding the sequence: + +- **A wedged component aborts the rest of the shutdown.** fx checks + the stop context before each remaining hook and returns outright + once it has expired, skipping the hooks it has not reached. If the + first-stopped component consumes the whole budget, the later hooks + never run — **the database close among them**. SQLite is crash-safe, + so this is not corruption, but it is not a clean close either. +- **Lowering the grace below 5 seconds reintroduces the silent + truncation.** `docker stop --time`, Compose's `stop_grace_period`, + or Kubernetes' `terminationGracePeriodSeconds` set under 5 seconds + put SIGKILL back in front of the bound, and the process dies with + no shutdown diagnostics at all. Keep the deployment's grace above + the stop timeout. + +### Linting + +golangci-lint never runs on the host. `script/lint` builds +`Dockerfile.lint`, which copies the repo into the digest-pinned +golangci-lint image and lints as a build step, so a successful build is +a clean lint. A host binary would share one cache and one lock with +every other checkout on the machine, which has produced both invented +findings attributed to other worktrees and unearned passes. + +Three properties are load-bearing: + +- `script/lint` passes `--no-cache-filter=lint`. Without it an unchanged + tree replays the lint layer from cache and the build exits 0 in under + a second having linted nothing. The `deps` stage stays cacheable, so + module downloads are not repeated. Invalidation is scoped to the one + stage; never prune the shared build cache. +- `script/lint` does not trust that flag. Docker silently ignores + `--no-cache-filter` for a stage name that does not match, so a stage + rename or a one-character typo would restore the cached false green + with no warning and a fast exit 0. The script therefore tees the + build output and treats a run as a pass only if golangci-lint's own + summary line (`N issues.` / `N issues:`) appears in it: no summary, + no lint, whatever the exit code says. +- Both lint steps use `RUN --network=none`. `golangci-lint config + verify` is documented as fetching its JSON schema over HTTPS, which + would be an unpinned remote dependency; the pinned image resolves the + schema without network access, and `--network=none` enforces that + instead of trusting it. Verify is worth keeping because + `golangci-lint run` silently ignores config keys it does not + recognize, so a typo would disable a setting with no warning. ### Docker -The Dockerfile uses a multi-stage build: +The Dockerfile uses a three-stage build. Each stage is pinned by +digest, and the two check stages are separate images so the linter's +version is fixed independently of the compiler's: -1. **Builder stage** (Debian-based `golang:1.24`) — installs - golangci-lint, downloads dependencies, copies source, runs `make - check` (format verification, linting, tests, compilation). -2. **Runtime stage** (`alpine:3.21`) — copies the binary, creates the - `/var/lib/webhooker` directory for all SQLite databases, runs as - non-root user, exposes port 8080, includes a health check. +1. **Lint stage** (`golangci/golangci-lint:v2.12.2`, Debian-based) — + installs `make`, downloads dependencies, copies the source, and runs + `make fmt-check`, then `golangci-lint config verify` and + `golangci-lint run`, both with `--network=none`. +2. **Builder stage** (`golang:1.26.1-bookworm`) — depends on the lint + stage passing (it copies a file from it), runs `script/fetch-assets` + to download and verify the third-party browser assets, then runs + `make test` and `make build`, and finally rebuilds the binary with + `CGO_ENABLED=1` and static linking so it runs on musl. Both builds + go through `make build`, the relink adding its `-extldflags` via + `GO_LDFLAGS`, so neither can drop the `-X` that stamps the version. + The version arrives as the `VERSION` build arg, since the context + has no `.git` (see [Version stamping](#version-stamping)). +3. **Runtime stage** (`alpine:3.21`) — copies the static binary, + creates the `/var/lib/webhooker` directory for all SQLite databases, + runs as the non-root `webhooker` user (UID 1000), exposes port 8080, + and includes a health check against `/.well-known/healthcheck`. -The builder uses Debian rather than Alpine because GORM's SQLite -dialect pulls in CGO-dependent headers at compile time. The runtime -binary is statically linked and runs on Alpine. +The lint stage invokes `golangci-lint` directly rather than `make lint`: +it is already the pinned linter image, and `make lint` builds +`Dockerfile.lint`, which would need a docker daemon inside this build. -`docker build .` is the CI gate — if it passes, the code is formatted, -linted, tested, and compiled. +Both check stages use Debian rather than Alpine because +`gorm.io/driver/sqlite` pulls in `mattn/go-sqlite3`, which needs CGO +and does not compile against musl. Only the final binary is statically +linked, which is what lets it run on the Alpine runtime image. + +`script/cibuild` — `docker build .` — is the CI gate: the checks run +inside the image, so a build that succeeds is a repo that is formatted, +linted, tested and compiled. `script/lint` also uses Docker +(`Dockerfile.lint`, see Linting above), so `make lint` and `make check` +run the same pinned linter version the gate does; only `script/test` +and `script/fmt-check` run on the host. + +#### CI gate honesty + +A layer cache lets `docker build .` exit 0 in seconds with the lint and +test stages replayed rather than executed, which would make a green +check meaningless. The `check` workflow therefore writes +`.ci-fingerprint` into the build context before building. Its value is +the hash of the last commit that touched the build context, so: + +- Any commit that changes code (including a squash merge whose tree + matches an already-built branch) gets a new fingerprint, invalidates + the `COPY . .` layer of both check stages, and really runs + `make fmt-check`, `golangci-lint`, `make test`, and `make build`. A + run that reports success ran them. +- A docs-only commit leaves the fingerprint unchanged — `.dockerignore` + excludes `*.md`, `LICENSE` and `.editorconfig` from the context + anyway — so the image replays from cache and costs seconds. + +The module download layer sits above `COPY . .` and stays cached either +way. + +A separate workflow step, run before the fingerprint is written, covers +a second way the gate lied: Gitea cancels an in-flight run when a newer +commit lands on the same branch and records that cancellation as a +`failure` status, so a commit nothing ever tested reads as a test +result. Cancellation is unconditional server-side for push events, so +the superseding run calls `script/ci-mark-superseded`, which rewrites +that exact status to `failure` / +`Superseded by a newer commit; never tested`. + +The state stays `failure` on purpose: Gitea's combined status folds +`skipped` into `success`, so marking a never-tested commit `skipped` +made the status API report green for it, indistinguishable from a commit +that passed. Reading a commit's status on this repo therefore goes: + +- `success` / `Successful in ...` — the checks ran and passed. +- `failure` / `Failing after ...` — the checks ran and failed. +- `failure` / `Superseded by a newer commit; never tested` — the run was + cancelled, by a newer push or by hand, and nothing was verified about + this commit. Test the commit itself before concluding anything about + it. + +Genuine failures and successes are never touched, and no status is left +`pending`, which would block the commit indefinitely. The step derives +its context string from the workflow name, the job **id** and the event. +That is deliberately not byte-identical to Gitea's own rule, which uses +the job's display `name:` where the runner exports the id, so giving the +job a `name:` — or renaming the workflow — makes the derived context +stop matching. The step fails loudly when no status on the commit +carries that context, so no rename can silently disable the rewrite. ## TODO diff --git a/REPO_POLICIES.md b/REPO_POLICIES.md index bc2f161..9aba6b0 100644 --- a/REPO_POLICIES.md +++ b/REPO_POLICIES.md @@ -1,6 +1,6 @@ --- title: Repository Policies -last_modified: 2026-07-06 +last_modified: 2026-08-07 --- This document covers repository structure, tooling, and workflow standards. Code @@ -189,8 +189,13 @@ style conventions are in separate documents: module under test to verify it compiles/parses. There is no excuse for `make test` to be a no-op. -- `make test` must complete in under 20 seconds. Add a 30-second timeout in the - Makefile. +- `make test` must complete in under 60 seconds. That is the hard cap, and a + suite that exceeds it fails. Under 20 seconds is the target. A suite between + 20 and 60 seconds is still green, but the overage must be filed as an + improvement bug against that repo. Add a 90-second timeout to the test + invocation in the Makefile (`go test -timeout 90s`). The backstop deliberately + sits above the hard cap so that it catches a genuinely hung test rather than a + merely slow one. - **`make test` should use the conditional verbose rerun pattern.** Run tests without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to @@ -209,9 +214,9 @@ style conventions are in separate documents: ```makefile test: - @go test -timeout 30s -race -cover ./... || \ + @go test -timeout 90s -race -cover ./... || \ { echo "--- Rerunning with -v for details ---"; \ - go test -timeout 30s -race -v ./...; exit 1; } + go test -timeout 90s -race -v ./...; exit 1; } ``` Python example: @@ -260,7 +265,10 @@ style conventions are in separate documents: - `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only manually by the user. Fetch from - `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`. + `https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`. The + canonical golangci-lint version is v2.12.2 (released 2026-05-06), installed + commit-pinned via + `go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@c0d3ddc9cf3faa61a4e378e879ece580256d76e5`. - When pinning images or packages by hash, add a comment above the reference with the version and date (YYYY-MM-DD). diff --git a/TODO.md b/TODO.md index c869649..3da085d 100644 --- a/TODO.md +++ b/TODO.md @@ -1,33 +1,373 @@ # Workflow -* branch (from `main`) -* do the work in Next Step -* move Next Step to the top of Completed Steps -* move the top item of Future Steps into Next Step -* commit (`TODO.md` changes in the same commit as the work) -* merge to `main` if the branch is not protected, otherwise open a PR -* push +One issue per unit of work, one branch and one PR per issue: + +* ensure a tracked issue exists with a definition of done +* branch from `next` (never from `main`) +* do the work; open a PR based on `next` (never on `main`) +* pass an independent review, then the manager squash-merges into `next` +* push; nothing stays local-only + +`next` is the branch for the next milestone and must stay green and +mergeable to `main` without notice. One `next` -> `main` PR accumulates +the milestone; releases are cut from `main` separately. + +Issue branches do NOT touch this file — the manager maintains it on +`next`. Every branch editing `TODO.md` conflicts with every other +(#112). # Status -pre-1.0. No git tags exist. main (afe88c6) is a working webhook proxy -with auth, CSRF/SSRF protections, login rate limiting, Slack target, -policy compliance (#6), and pinned lint tooling (#55). Note: TODO.md was -deliberately deleted from this repo in f9a9569 (2026-03-01, #6); its -content was folded into the README TODO section, which this draft -reconstructs as of 2026-07-06. +The milestone (https://git.eeqj.de/sneak/webhooker/milestone/9) is the +authoritative list, and the only place to read a count or a state of +play from. This file records where the project is, not what is in +flight: a sentence whose truth depends on a branch being unmerged is +wrong the moment it merges, and this file has been wrong that way +before. + +The durability defect that held the tag has landed +(https://git.eeqj.de/sneak/webhooker/issues/256, commit `8d64259`). +Every SQLite handle opens with WAL journaling and a busy timeout, a +bookkeeping write that fails leaves its delivery in a recoverable +state rather than a lying one, and recovery skips a delivery that +already has a successful result row. Final pre-tag verification +exercised it and confirmed it holds. Whatever the milestone still +shows open is what remains before `v1.0.0`. + +Delivery is at-least-once by design, not by accident: a send whose +result row does not land is attempted again, so a receiver can see a +duplicate. That is deliberate — the alternative is a silent lost +delivery — and the README says so under Rationale. It is not a defect +to re-file. + +One caveat on reading a green check: a docs-only commit deliberately +replays from the layer cache +(https://git.eeqj.de/sneak/webhooker/issues/119), so a green status on +such a commit evidences a replay rather than an executed run. A code +commit invalidates the `COPY` layer and genuinely executes. # Next Step -Implement automatic event retention cleanup based on retention_days: a -periodic maintenance job that deletes Events, Deliveries, and -DeliveryResults older than the parent webhook's retention_days from each -per-webhook event database. The field exists on the Webhook model and -the README promises the behavior, but nothing enforces it, so event -databases currently grow without bound. +Clear the rest of the open 1.0.0 milestone +(https://git.eeqj.de/sneak/webhooker/milestone/9) and tag `v1.0.0`. +Merging `next` into `main` is a separate act from tagging and waits on +neither of those: `next` is kept mergeable at all times, which is the +point of the branch. # Completed Steps +- 2026-08-24 Bind the plaintext HTTP listener deliberately, via + `BIND_ADDRESS` defaulting to `127.0.0.1`, and document the + reverse-proxy deployment. A hostname, an empty value or a value + carrying a port is a startup error, and the `Dockerfile` sets + `0.0.0.0` because a loopback bind inside a container is unreachable + (https://git.eeqj.de/sneak/webhooker/issues/268). The same commit + removed the shutdown race: `httpServer` is built in the constructor + rather than assigned from the serving goroutine, which orders the + write before every fx hook and rules out the nil dereference a + SIGTERM arriving first would have caused, and `sentryEnabled` is an + `atomic.Bool` (https://git.eeqj.de/sneak/webhooker/issues/226) +- 2026-08-24 Remove inbound request signature verification. The + entrypoint UUID is the authentication secret, so the per-entrypoint + shared secret, the `internal/signature` package, the receiver check, + the model fields and the forms are all gone. This reverses the + feature that landed earlier in the same milestone + (https://git.eeqj.de/sneak/webhooker/issues/67, + https://git.eeqj.de/sneak/webhooker/issues/279) +- 2026-08-24 Stamp the build version into the binary and render it in + the UI footer. `script/version` is the single source — `$VERSION`, + else `git describe --tags --always --dirty`, else `unknown` — so a + `make build` binary and a `make docker` image from one checkout + report the same thing, and nothing in it varies between two builds + of the same commit, which the release gate's byte-identical + assertion would catch + (https://git.eeqj.de/sneak/webhooker/issues/253) +- 2026-08-24 Derive cookie `Secure` and CSRF strictness from the + request transport rather than from `WEBHOOKER_ENVIRONMENT`. Behind a + real TLS proxy with the environment left at its `dev` default, the + session cookie silently lost `Secure` while the CSRF cookie on the + same response kept it. `X-Forwarded-Proto` is now matched + case-insensitively on its first comma-separated element, so `HTTPS` + and `https, http` no longer fall to the relaxed CSRF path + (https://git.eeqj.de/sneak/webhooker/issues/269) +- 2026-08-24 Roll back a failed webhook deletion instead of committing + it. A failing delete committed whatever had already succeeded, + hard-deleted the per-webhook event database anyway, and redirected as + though it had worked — orphaned config plus permanently destroyed + history, reported as success. All three delete positions now roll + back with the event database intact + (https://git.eeqj.de/sneak/webhooker/issues/262) +- 2026-08-24 Name a deleted target on its historical deliveries, marked + `(deleted)`, rather than leaving the event log unable to say where a + delivery went. A deleted target's credentials stay masked exactly as + a live one's, and it cannot become deliverable again through the + receiver, resubmit, replay, the edit form or the toggle + (https://git.eeqj.de/sneak/webhooker/issues/211) +- 2026-08-24 Bound both request-controlled `/metrics` label dimensions, + so the unauthenticated receiver is no longer a memory-exhaustion + vector: `handler` carries the chi route pattern, and `method` folds + anything chi cannot route onto a single `(unmatched)` sentinel. Both + were reproduced before the fix — 300 random method tokens took the + series count from 106 to 7,631, and path flooding reached 62,532 — + and a label audit across a live scrape found no third unbounded + dimension (https://git.eeqj.de/sneak/webhooker/issues/254, + https://git.eeqj.de/sneak/webhooker/issues/261) +- 2026-08-24 Validate `max_retries` on both target forms. `abc`, `2.7` + and `-5` silently became 0 — fire-and-forget — including on the edit + path, where it destroyed a working value, and `999999999` stored + verbatim. The ceiling of 20 is the `max` both templates already + declared (https://git.eeqj.de/sneak/webhooker/issues/221) +- 2026-08-24 Resubmit a stored event as a new undelivered event, so a + backend under development can be tested against real captured + traffic. Per-delivery replay cannot serve that: it re-sends one + finished delivery to its own original target, and a target created + for a dev backend has no prior delivery to replay. Resubmit + re-injects the stored event at the top of the receiver path and fans + it out to whatever targets are active now + (https://git.eeqj.de/sneak/webhooker/issues/250) +- 2026-08-20 Take an exclusive lock on `DATA_DIR` at startup, so two + instances on one directory cannot both deliver + (https://git.eeqj.de/sneak/webhooker/issues/201) +- 2026-08-20 Shut down the app when the HTTP listener fails. The + `OnStart` hook returned as soon as the serving goroutine was + spawned, so a failed listen left fx reporting RUNNING and a live + process with nothing bound — invisible to systemd and Docker restart + policies (https://git.eeqj.de/sneak/webhooker/issues/200) +- 2026-08-20 Stop target credentials leaking into the per-webhook event + databases (https://git.eeqj.de/sneak/webhooker/issues/206), log SQL + with placeholders rather than bound values + (https://git.eeqj.de/sneak/webhooker/issues/207), and fail loudly on + half-set metrics auth credentials + (https://git.eeqj.de/sneak/webhooker/issues/205) +- 2026-08-20 Read queue depths with `Find`, not `Scan`. `Scan` swaps + GORM's own trace recorder in for the logging adapter, and that + recorder does not implement `gorm.ParamsFilter`, so those statements + logged their bound values interpolated and bypassed the suppression + above. The two units gated green against a `next` that lacked the + other, and `next` went red when both landed + (https://git.eeqj.de/sneak/webhooker/issues/234) +- 2026-08-20 Render per-attempt delivery detail in the event log + (https://git.eeqj.de/sneak/webhooker/issues/202) and add replay of a + terminally failed delivery + (https://git.eeqj.de/sneak/webhooker/issues/203) +- 2026-08-20 Expose delivery metrics on `/metrics` + (https://git.eeqj.de/sneak/webhooker/issues/209) and document the + backup, restore and upgrade procedures + (https://git.eeqj.de/sneak/webhooker/issues/210) +- 2026-08-20 Add a `webhooker resetpw` subcommand and a bootstrap + banner. The admin bootstrap password was printed once among roughly + 45 fx lines, and under `docker run -d` went to container logs subject + to rotation; there was no reset path at all, so recovery meant + hand-deleting the users row, documented nowhere. The password is read + from stdin or generated, never from argv where `/proc` would publish + it (https://git.eeqj.de/sneak/webhooker/issues/208) +- 2026-08-20 Add `ALLOWED_EGRESS_CIDRS`, an allowlist-only escape hatch + for the SSRF guard, so a self-hosted proxy can forward into the + operator's own network. The guard's always-blocked set cannot be + reopened by configuration + (https://git.eeqj.de/sneak/webhooker/issues/204) +- 2026-08-20 Harden operator-set target headers, which were carried + unsafely across a redirect + (https://git.eeqj.de/sneak/webhooker/issues/233) +- 2026-08-20 Add a target edit form with headers and timeout fields + (https://git.eeqj.de/sneak/webhooker/issues/127) +- 2026-08-18 Raise `script/test`'s per-package timeout from 30s to 90s, + matching the org-wide backstop. `go test` applies `-timeout` per + package, and `internal/handlers` had grown past the old budget: a + cache-defeated build failed outright at `GOMAXPROCS=4`, and every run + under deliberate host load breached 30s. The measurement table lives + in the script (#194) +- 2026-08-18 Re-sync `REPO_POLICIES.md` from `prompts`. The local copy + was stale and still mandated a 20s test target with a 30s timeout, + which the org replaced with a 60s cap and a 90s backstop. A synced + copy is not a source; reading it as one nearly produced a PR against + `prompts` proposing a change already merged there (#196) +- 2026-08-18 Report handler panics through the logger and answer 500. + chi v1.5.5's `Recoverer` scans for a `panic(0x` frame the runtime no + longer emits, then indexes `pkg[-1:]`, so it panicked inside its own + stack printer before writing a byte: the recovery never ran, the + client got a dropped connection instead of a 500, and the original + panic was lost. A local middleware replaces it, bounded by + `MaxPanicLogLineBytes` (#187) +- 2026-08-18 Route GORM's logger through `slog` and bound it. Every + `gorm.Open` left `logger.Default` in place at `Warn` with + `IgnoreRecordNotFoundError` false, so **every record-not-found + printed the fully interpolated SQL to stdout** — including the + client-chosen path on `/webhook/{uuid}` and the submitted username on + the login form, at no level the operator set and outside + `internal/logger` entirely. Three call sites, not the two the issue + named (#178) +- 2026-08-18 Bound every `slog` line against client-chosen text. Eight + sites reachable unauthenticated, found by reading every `slog` call in + the tree rather than only the one reported; the budget moved to a + shared `internal/logfield` so no second truncation exists. `DEBUG` + being off by default is not a bound and is not treated as one (#176) +- 2026-08-18 Stop a slow host turning a login-guard test into a + segfault. A non-fatal `assert` on an acquire result was dereferenced + on the next line, so one timing miss killed the whole + `internal/middleware` binary and reddened CI for unrelated PRs. The + fix also removed a real production race — `acquire` could shed a + request with a slot standing free, because Go picks uniformly among + ready `select` cases (#186) +- 2026-08-18 Send the chi route pattern to Sentry rather than the + concrete path. The receiver's path carries the entrypoint capability + token, so every Sentry event from `/webhook/{uuid}` shipped a live + credential to a third party. Request `Data`, `QueryString`, `Cookies` + and `Env` are dropped and headers reduced to an allowlist (#179) +- 2026-08-18 Read form fields from the POST body only. `r.FormValue` + merges the query string, so a login could be driven by URL parameters + — putting the password somewhere that lands in access logs, proxy + logs and browser history (#160) +- 2026-08-18 Verify login credentials before spending rate-limit + budget, so a flood of wrong passwords cannot lock out the account it + is guessing at. The manager took this decision rather than stall the + queue; it is flagged on the issue for reversal (#150) +- 2026-08-18 Run all linting in Docker via `Dockerfile.lint`. Host lint + was wrong in both directions from version skew and shared caches. + `script/lint` asserts the summary line, because `--no-cache-filter` + silently ignores a stage name it does not match — the flag that makes + the gate meaningful fails open (#109) +- 2026-08-18 Serve an event's full stored body over HTTP. The list + query truncates for rendering, and that truncated value was the only + way to read a body, so the full payload was unreachable (#157) +- 2026-08-18 Bound the access log line against client-chosen text. + `internal/logfield` budgets by *encoded* bytes, not runes, so a + handler's JSON escaping cannot multiply a field past its allowance + (#146) +- 2026-08-18 Mark superseded CI commits `failure` rather than + `skipped`. A skipped run rolls up green, so a commit that was never + tested reported success (#152) +- 2026-08-18 Set `fx.StopTimeout` inside the container stop grace, so + shutdown hooks are bounded by a deadline the orchestrator will + actually honour rather than being killed mid-flush (#134) +- 2026-08-17 Bucket IPv6 rate-limit keys by `/64`. A single allocation + hands out 2^64 addresses, so per-address keying let one client mint + unlimited buckets. Manager decision, recorded on the issue (#125) +- 2026-08-17 Correct release-blocking README and startup-warning + inaccuracies, including claims about behaviour the code does not have + (#151) +- 2026-08-17 Fetch and verify Alpine.js at build time against + `static/vendor.sha256` instead of committing the minified blob, so + the dependency is pinned by hash rather than by trust (#145) +- 2026-08-17 Bound the event log's rendered bodies in the query itself, + so a large stored payload cannot be read into memory just to be + truncated for display (#135) +- 2026-08-17 Mask the `http` target's destination URL in the UI: it can + carry a bearer credential in its path or query, and was rendered + verbatim. Manager decision to mask unconditionally (#115) +- 2026-08-14 Bound shutdown hooks by their stop context, so a hook that + hangs cannot hold the process past its grace period (#102) +- 2026-08-14 Render templates via a buffer rather than the + `ResponseWriter`, so a template error part-way through cannot commit + a 200 and then fail — the response is written only once it is whole + (#123) +- 2026-08-14 Align the session codec's max-age with the 7-day absolute + cap. The codec accepted cookies the session layer considered expired, + so the cap was enforced in one place and not the other (#108) +- 2026-08-12 Warn when `TRUSTED_PROXIES` is empty in production, where + the safe default silently discards forwarded headers and every client + rate-limits as the proxy's address (#149) +- 2026-08-12 Bound the receiver rate limit per client IP across the + whole `/webhook/*` route. The existing limiter keyed on the request + path and `/webhook/{uuid}` matches any single segment, so a client + that invented a fresh path per request minted a fresh bucket per + request: the limit on the only unauthenticated endpoint bounded + nothing in aggregate, and every request still cost an entrypoint + lookup before it 404ed. An outer limiter keyed on the client address + alone now bounds that, chained in front of the unchanged + per-entrypoint limiter (#139) +- 2026-08-12 Correct release-blocking documentation inaccuracies: the + README promised manual redelivery in the present tense in three + places when nothing implements it (the same false claim also sat in + the doc comment that was its source text), the env table omitted + `RETENTION_SWEEP_INTERVAL`, and `TODO.md` itself omitted five landed + units (#141) +- 2026-08-12 Make the CI gate execute the checks it reports on. The + workflow now writes a build-context fingerprint before calling + `script/cibuild`, so a code commit invalidates the `COPY` layer of + the lint and builder stages while a docs-only commit still replays + from cache; a superseding run also rewrites the `failure` status + Gitea leaves on commits it cancelled and never tested. Verified by + pushing a deliberately broken test and watching CI go red (#119) +- 2026-08-12 Require a positive `RETENTION_SWEEP_INTERVAL`: a + non-positive value reached `time.NewTicker` in both the retention + reaper and the archive sweeper, panicking two goroutines with no + recover after startup had already reported success (#140) +- 2026-08-12 Bound the `X-Forwarded-For` scan's allocation to the hop + cap: the reverse walk cuts entries with `strings.LastIndexByte` + instead of joining and splitting, so a 1 MB header allocates 16 bytes + rather than 1.6 MB per request on the unauthenticated receiver. + Semantics proven unchanged by differential testing against the + previous implementation (#133) +- 2026-08-12 Cap the `X-Forwarded-For` hop walk at 64 entries, so an + attacker-supplied chain cannot burn unbounded CPU in the rate-limit + key function; running off the end falls back to the peer address + (#124) +- 2026-08-12 Gate forwarded-header trust behind a `TRUSTED_PROXIES` CIDR + list: all three rate limiters key on the connection's own address + unless the direct peer is a configured proxy, in which case + `X-Forwarded-For` is walked right to left for the first non-proxy hop. + Default trusts nothing, and a set-but-unparseable value aborts + startup. Before this, any client could mint a fresh bucket or drain + another's by rotating a spoofed header (#88) +- 2026-08-11 Web UI cleanup: nav terminology unified on Webhooks, the + Profile settings placeholder removed, a progressive-enhancement copy + button for the entrypoint URL, and retention form copy that states the + actual policy (deletion by the reaper, 0 retains forever) (#57) +- 2026-08-11 Mask the webhook credential in delivery errors and logs: + Go embeds the request URL in `*url.Error`, so every transport failure + persisted the full Slack webhook URL into the per-webhook event + database via `DeliveryResult.Error`, a field a future REST API would + have served. `maskURLError` drops path, query and userinfo while + preserving the wrapped cause, so `errors.Is`/`As` and `Timeout()` + still work and DNS, TLS and timeout failures still read differently + (#118) +- 2026-08-11 Rate-limit the public webhook receiver endpoint + (`RECEIVER_RATE_LIMIT`, default 120/min), keyed on client IP plus + entrypoint path so one entrypoint cannot exhaust another's budget; + over-limit requests get 429 with `Retry-After`. It was the one + unauthenticated, internet-facing endpoint with no limit at all (#64) +- 2026-08-11 Enforce the body size limit before CSRF parses the form: + `MaxBodySize` is now first in all four form-parsing route groups, so + an oversized request is rejected with 413 instead of being read in + full by the CSRF middleware before any cap applied (#90) +- 2026-08-11 Mask target config on the source detail page, which + rendered the stored blob verbatim and so exposed the Slack + incoming-webhook URL — a bearer credential that cannot be revoked + per-holder. Config reaches the template only as a `TargetView` of + labelled fields, and header values are rendered as a count (#113) +- 2026-08-11 Allow `retention_days` of 0 to mean retain forever, via a + sentinel written in `BeforeSave` so the GORM column default cannot + win the race. Also bounds the reaper's cutoff arithmetic: day counts + above 106751 overflowed `time.Duration` and wrapped the cutoff into + the future, where every row matched and the sweep deleted everything + (#79) +- 2026-08-09 Inactivity-based session timeout: sliding idle expiry + (`SESSION_IDLE_TIMEOUT`, default `24h`) refreshed on authenticated + requests, with the 7-day absolute cap kept as an independent + backstop that activity never extends (#66) +- 2026-08-09 Restart recovery and the 60s retry sweep terminally fail an + orphaned `retrying` delivery whose target type no longer supports + retries, recording a `DeliveryResult` with the reason instead of + leaving the delivery stuck forever (#82) +- 2026-08-09 Root the delivery engine's worker pool and the retention + reaper's sweep loop at `context.Background()` rather than the fx + `OnStart` hook context (#97), which carries fx's 15s start timeout and + killed both roughly fifteen seconds after boot: the proxy silently + stopped delivering webhooks entirely, and the reaper never ran a + single sweep under its default one-hour interval +- 2026-08-09 Archive writer lifecycle (#89): deleting a webhook (or its + last `database` target) evicts the cached archive writer and closes + its handle while deliberately leaving `archive-{webhookID}.db` on + disk, and a new `ArchiveSweeper` prunes idle archives on the existing + `RETENTION_SWEEP_INTERVAL` without ever creating an archive file +- 2026-08-09 Configuration parsing fails loudly on set-but-unparseable + environment values: `envInt` removed in favour of `envPositiveInt` + plus a `PORT` range check, `envBool` now parses with + `strconv.ParseBool`, and defaults apply only to unset variables (#80) +- 2026-08-07 Automatic event retention cleanup based on + `retention_days`, deleting expired events, deliveries, and delivery + results from each per-webhook event database (#63) - 2026-08-07 Update golangci-lint to v2.12.2 (Docker image digest in `Dockerfile`, release-archive sha256 pins in `script/bootstrap`), adopt the canonical `.golangci.yml` (v2 `linters.settings` layout so @@ -56,13 +396,13 @@ databases currently grow without bound. # Future Steps -- Manual event redelivery from the web UI (replay is a core promised - capability in the README rationale) -- Delivery status and retry management UI +- Delivery status and retry management UI. Replay of a terminally + failed delivery and per-attempt detail already landed + (https://git.eeqj.de/sneak/webhooker/issues/203, + https://git.eeqj.de/sneak/webhooker/issues/202) - Per-webhook rate limiting in the receiver handler (per-webhook config plus handler enforcement; global limits must not apply to receiver endpoints) -- Webhook signature verification for GitHub and Stripe HMAC formats - API key authentication for programmatic access (APIKey model exists; Bearer token middleware does not) - REST API v1 @@ -71,8 +411,11 @@ databases currently grow without bound. - event redelivery endpoint - OpenAPI specification - Analytics dashboard: success rates, response times, volume -- Session expiration tuning and a remember-me option -- Password change and reset flow +- A remember-me option at login +- Password reset flow for a forgotten password over the web. The + authenticated password *change* flow already landed, and a lost + password is recoverable from the console with `webhooker resetpw` + (https://git.eeqj.de/sneak/webhooker/issues/208) - Later, nice to have - email delivery target type - SNS and S3 delivery targets diff --git a/cmd/webhooker/dotenv_test.go b/cmd/webhooker/dotenv_test.go new file mode 100644 index 0000000..7a579c8 --- /dev/null +++ b/cmd/webhooker/dotenv_test.go @@ -0,0 +1,107 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/config" +) + +// dotEnvKey is a throwaway variable name these tests write and read, +// so they cannot disturb real configuration. +const dotEnvKey = "WEBHOOKER_TEST_DISPATCH_VALUE" + +// writeDotEnvInWorkingDir puts contents in a .env file in a fresh +// temporary directory and moves the process there. +// +// The callers are deliberately not parallel and must stay that way: +// t.Chdir moves the whole process. Go releases parallel tests only +// after every sequential test in the package has finished, so nothing +// else runs while these do. +func writeDotEnvInWorkingDir(t *testing.T, contents string) { + t.Helper() + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, config.DotEnvPath), + []byte(contents), 0o600, + )) + t.Chdir(dir) +} + +// TestDispatch_MalformedDotEnvRefuses pins the second half of the +// defect. godotenv applies nothing at all when a file will not parse, +// so one mistyped line used to revert every variable in it to its +// default and start the server anyway, with no log line naming the +// file. The refusal has to arrive before any subcommand runs, which +// is why `help` — the one subcommand that touches nothing — is still +// refused here. +// +//nolint:paralleltest // t.Chdir moves the whole process. +func TestDispatch_MalformedDotEnvRefuses(t *testing.T) { + writeDotEnvInWorkingDir(t, "PORT 19615\n") + + var stdout, stderr bytes.Buffer + + code := dispatch( + []string{helpCommand}, strings.NewReader(""), &stdout, &stderr, + ) + + require.Equal(t, 1, code, "a broken .env must exit non-zero") + assert.Contains( + t, stderr.String(), config.DotEnvPath, + "the refusal must name the file", + ) + assert.Empty( + t, stdout.String(), + "the subcommand must not have run", + ) +} + +// TestDispatch_LoadsDotEnvBeforeSubcommands pins the ordering the +// godotenv/autoload import used to provide for free. It ran in an +// init(), so .env was in the environment before anything read it — +// including config.DataDir, which both the DATA_DIR lock and resetpw +// call outside the fx graph. Loading any later would let a .env that +// sets DATA_DIR lock one directory while the config opened databases +// in another. +func TestDispatch_LoadsDotEnvBeforeSubcommands(t *testing.T) { + t.Setenv(dotEnvKey, "placeholder") + require.NoError(t, os.Unsetenv(dotEnvKey)) + + writeDotEnvInWorkingDir(t, dotEnvKey+"=from-dot-env\n") + + var stdout, stderr bytes.Buffer + + code := dispatch( + []string{helpCommand}, strings.NewReader(""), &stdout, &stderr, + ) + + require.Equal(t, 0, code) + assert.Equal( + t, "from-dot-env", os.Getenv(dotEnvKey), + "the file must be applied before the subcommand runs", + ) +} + +// TestDispatch_MissingDotEnvIsFine pins the case most deployments are +// in: no .env at all, which must stay a normal start. +// +//nolint:paralleltest // t.Chdir moves the whole process. +func TestDispatch_MissingDotEnvIsFine(t *testing.T) { + t.Chdir(t.TempDir()) + + var stdout, stderr bytes.Buffer + + code := dispatch( + []string{helpCommand}, strings.NewReader(""), &stdout, &stderr, + ) + + require.Equal(t, 0, code) + assert.Empty(t, stderr.String()) +} diff --git a/cmd/webhooker/main.go b/cmd/webhooker/main.go index 59fc655..0da8fa0 100644 --- a/cmd/webhooker/main.go +++ b/cmd/webhooker/main.go @@ -2,19 +2,63 @@ package main import ( + "fmt" + "io" + "os" + "time" + "go.uber.org/fx" "sneak.berlin/go/webhooker/internal/config" "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/datadir" "sneak.berlin/go/webhooker/internal/delivery" "sneak.berlin/go/webhooker/internal/globals" "sneak.berlin/go/webhooker/internal/handlers" "sneak.berlin/go/webhooker/internal/healthcheck" "sneak.berlin/go/webhooker/internal/logger" "sneak.berlin/go/webhooker/internal/middleware" + "sneak.berlin/go/webhooker/internal/resetpw" "sneak.berlin/go/webhooker/internal/server" "sneak.berlin/go/webhooker/internal/session" ) +// stopTimeout bounds the whole fx stop sequence, not each hook. +// +// fx defaults to 15s, which is longer than Docker's 10s default +// stop grace: the container would be SIGKILLed before the bound +// could fire, so nothing bounded by it would ever be observed. +// 5s leaves headroom inside that grace for signal delivery and +// process exit; the observed wedge case already exits at ~5.3s, +// so a larger bound would trade a rare skipped database close for +// a more common hard kill. +// +// The server's stop hook must fit inside it with room to spare: a +// hook that used the whole budget would exhaust it at that instant, +// and fx would skip every hook after the server — the delivery +// engine, the healthcheck, the webhook DB manager and the database +// close. That hook is the 3s HTTP drain plus the Sentry flush that +// follows it in the same hook, so the flush is clamped to the stop +// context's remaining time less server.TailHookReserve rather than +// running for its own fixed 2s; the reserve is what the tail hooks +// live on, and they are microsecond-scale in normal operation. +// TestStopTimeout_LeavesHeadroomForTailHooks pins the arithmetic +// across every drain length. +// +// This does not make the database close unconditional: the +// ArchiveSweeper and RetentionReaper hooks run before the server +// and can still consume the whole budget on their own. +const stopTimeout = 5 * time.Second + +// exitUsage is the status for a command line this binary cannot make +// sense of, kept distinct from the 1 a refusal exits with so that a +// caller can tell "called wrong" from "declined". +const exitUsage = 2 + +// helpCommand is the subcommand that prints usage. The flag spellings +// beside it in the switch are aliases; this is the name the usage text +// documents and the one tests invoke. +const helpCommand = "help" + // Build-time variables set via -ldflags. // //nolint:gochecknoglobals // Build-time variables injected by the linker. @@ -27,7 +71,102 @@ func main() { globals.Appname = appname globals.Version = version - fx.New( + os.Exit(dispatch(os.Args[1:], os.Stdin, os.Stdout, os.Stderr)) +} + +// dispatch routes the command line to a subcommand. +// +// No arguments runs the server, which is what the image's CMD and +// every existing deployment invoke; that path is unchanged, including +// where the DATA_DIR lock is taken relative to building the fx graph +// and how fx propagates a non-zero exit itself. +// +// The optional .env file is read here, before any subcommand and so +// before anything reads the environment — config.DataDir, which both +// the DATA_DIR lock and resetpw call outside the fx graph, above all. +// It used to be read from an init() in internal/config, which put it +// earlier still but threw the error away: a single malformed line +// applied none of the file and said nothing about it. A file that is +// not there stays fine, since .env is optional and most deployments +// do not have one. +func dispatch( + args []string, + stdin io.Reader, + stdout, stderr io.Writer, +) int { + err := config.LoadDotEnv() + if err != nil { + _, _ = fmt.Fprintf(stderr, "%s: %v\n", appname, err) + + return 1 + } + + if len(args) == 0 { + return run(stderr) + } + + switch args[0] { + case resetpw.Name: + return resetpw.Run(args[1:], stdin, stdout, stderr) + case helpCommand, "-h", "-help", "--help": + usage(stdout) + + return 0 + default: + _, _ = fmt.Fprintf( + stderr, "%s: unknown subcommand %q\n", appname, args[0], + ) + usage(stderr) + + return exitUsage + } +} + +// usage lists what the binary can be asked to do. +func usage(w io.Writer) { + _, _ = fmt.Fprintf(w, `usage: %s [subcommand] + +With no subcommand, runs the webhooker server. + +Subcommands: + %s [-generate] + Set an existing account's password on a stopped deployment. + Recovers an admin account whose bootstrap password was lost. + help + Print this message. +`, appname, resetpw.Name) +} + +// run takes the exclusive DATA_DIR lock, then runs the application +// under it, and returns the process exit status. +// +// The lock is taken here rather than inside the fx graph because it has +// to be held before anything opens a database, and because a refusal +// has to reach the operator as a plain line on standard error rather +// than as one entry in an fx failure dump. It is released by the defer +// on a clean shutdown, and by the kernel closing the descriptor on any +// other exit — including the one fx performs itself when a start or +// stop hook fails, which skips deferred calls. +func run(stderr io.Writer) int { + lock, err := datadir.Acquire(config.DataDir()) + if err != nil { + _, _ = fmt.Fprintf(stderr, "%s: %v\n", appname, err) + + return 1 + } + + defer func() { _ = lock.Release() }() + + newApp().Run() + + return 0 +} + +// newApp builds the application graph. It is separate from main so +// a test can assert the options it carries. +func newApp() *fx.App { + return fx.New( + fx.StopTimeout(stopTimeout), fx.Provide( globals.New, logger.New, @@ -39,10 +178,20 @@ func main() { session.New, handlers.New, middleware.New, + // The one SSRF guard both target-creation validation + // and the delivery dialer consult, so they cannot + // disagree about a destination. + delivery.NewGuard, delivery.New, + delivery.NewArchiveSweeper, // Wire *delivery.Engine as delivery.Notifier so the // webhook handler can notify the engine of new deliveries. func(e *delivery.Engine) delivery.Notifier { return e }, + // Wire *delivery.Engine as delivery.WebhookEvictor so + // deleting a webhook releases its archive writer. + func(e *delivery.Engine) delivery.WebhookEvictor { + return e + }, server.New, ), fx.Invoke( @@ -50,8 +199,9 @@ func main() { *server.Server, *delivery.Engine, *database.RetentionReaper, + *delivery.ArchiveSweeper, ) { }, ), - ).Run() + ) } diff --git a/cmd/webhooker/main_test.go b/cmd/webhooker/main_test.go new file mode 100644 index 0000000..98ca68e --- /dev/null +++ b/cmd/webhooker/main_test.go @@ -0,0 +1,171 @@ +package main + +import ( + "bytes" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/datadir" + "sneak.berlin/go/webhooker/internal/resetpw" + "sneak.berlin/go/webhooker/internal/server" +) + +// dockerStopGrace is Docker's default `docker stop` grace period. +// The Dockerfile sets no STOPSIGNAL or grace override, so this is +// the deadline the container is actually held to, and the fx stop +// timeout has to fit inside it with room for signal delivery and +// process exit. +const dockerStopGrace = 10 * time.Second + +// TestNewApp_StopTimeout pins the fx stop timeout. Without the +// explicit fx.StopTimeout option the app reads fx's 15s +// DefaultTimeout, which exceeds dockerStopGrace: the container is +// SIGKILLed before the bound fires and every shutdown hook bounded +// by it — including the operator-facing timeout log — becomes +// unreachable in the image this repo produces. +// +// fx.New applies options before it executes invokes, so the timeout +// is set whether or not the graph itself can be constructed here. +func TestNewApp_StopTimeout(t *testing.T) { + t.Setenv("DATA_DIR", t.TempDir()) + + got := newApp().StopTimeout() + + require.Equal(t, stopTimeout, got) + require.Less(t, got, dockerStopGrace) +} + +// TestRunRefusesLockedDataDir pins what an operator's second start +// does. The entry point must refuse before it builds the fx graph — +// nothing may open a database in a DATA_DIR another process holds — +// and must exit non-zero with a message naming the directory rather +// than starting a second delivery engine over the same rows. +// +// flock(2) locks descriptors independently, so holding the lock here +// is the same denial a separate process gets; internal/datadir pins +// that property and covers the real two-process case. +func TestRunRefusesLockedDataDir(t *testing.T) { + dir := t.TempDir() + t.Setenv("DATA_DIR", dir) + + lock, err := datadir.Acquire(dir) + require.NoError(t, err) + + defer func() { _ = lock.Release() }() + + var stderr bytes.Buffer + + code := run(&stderr) + + require.Equal( + t, 1, code, "a second instance must exit non-zero", + ) + assert.Contains( + t, stderr.String(), dir, + "the refusal must name the directory", + ) + assert.Contains(t, stderr.String(), "another instance") +} + +// TestDispatch_NoArgumentsRunsTheServer pins the routing of a bare +// invocation, which is what the image's CMD and every deployment use. +// Adding subcommands must not move the server off the empty argument +// list, and must not move the DATA_DIR lock: this asserts the refusal +// arrives with no fx graph built, exactly as run does on its own. +func TestDispatch_NoArgumentsRunsTheServer(t *testing.T) { + dir := t.TempDir() + t.Setenv("DATA_DIR", dir) + + lock, err := datadir.Acquire(dir) + require.NoError(t, err) + + defer func() { _ = lock.Release() }() + + var stdout, stderr bytes.Buffer + + code := dispatch(nil, strings.NewReader(""), &stdout, &stderr) + + require.Equal(t, 1, code) + assert.Contains(t, stderr.String(), "another instance") +} + +// TestDispatch_UnknownSubcommand keeps a mistyped subcommand from +// starting a server. Anything else would have `webhooker resetpww` +// silently take the DATA_DIR lock and serve. +func TestDispatch_UnknownSubcommand(t *testing.T) { + t.Parallel() + + var stdout, stderr bytes.Buffer + + code := dispatch( + []string{"resetpww", "admin"}, + strings.NewReader(""), &stdout, &stderr, + ) + + require.Equal(t, 2, code) + assert.Contains(t, stderr.String(), "unknown subcommand") + assert.Contains( + t, stderr.String(), resetpw.Name, + "the usage must name the subcommand that does exist", + ) +} + +// TestDispatch_Help answers on standard output with a zero status, so +// `webhooker help` is usable in a pipe. +func TestDispatch_Help(t *testing.T) { + t.Parallel() + + var stdout, stderr bytes.Buffer + + code := dispatch( + []string{helpCommand}, strings.NewReader(""), &stdout, &stderr, + ) + + require.Equal(t, 0, code) + assert.Empty(t, stderr.String()) + assert.Contains(t, stdout.String(), resetpw.Name) +} + +// tailHeadroom is the slack the fx stop budget must keep beyond the +// server stop hook. The hooks that run after the server — the +// delivery engine, the healthcheck, the webhook DB manager and the +// database close — are microsecond-scale in normal operation, so +// this is generous for them. +const tailHeadroom = 2 * time.Second + +// TestStopTimeout_LeavesHeadroomForTailHooks pins the relationship +// between the server's stop hook and the fx stop budget. fx bounds +// the whole stop sequence, and returns without running its +// remaining hooks once the stop context has expired. If the hook +// could use the entire budget, every later hook — the database close +// included — would be skipped in exactly the case where the drain +// mattered. +// +// The hook is not just the HTTP drain: a Sentry flush follows it in +// the same hook, and sentry.Flush honours no context, so both halves +// have to be counted. The sweep walks every drain length the hook +// can produce, since a shorter drain leaves the flush more room and +// the worst case is not necessarily at either extreme. +// +// Shrinking either budget, or unbounding the flush again, must fail +// here rather than silently recreating a hook that swallows the +// whole sequence. +func TestStopTimeout_LeavesHeadroomForTailHooks(t *testing.T) { + t.Parallel() + + require.Less(t, server.ShutdownTimeout, stopTimeout) + + const step = 10 * time.Millisecond + + for drain := time.Duration(0); drain <= server.ShutdownTimeout; drain += step { + hook := drain + server.SentryFlushBudget(stopTimeout-drain) + + require.LessOrEqual( + t, hook+tailHeadroom, stopTimeout, + "a %s drain leaves the tail hooks short", drain, + ) + } +} diff --git a/go.mod b/go.mod index 46c3ad5..3fbd1d2 100644 --- a/go.mod +++ b/go.mod @@ -8,15 +8,18 @@ require ( github.com/go-chi/chi v1.5.5 github.com/go-chi/cors v1.2.1 github.com/go-chi/httprate v0.15.0 + github.com/gofrs/flock v0.13.0 github.com/google/uuid v1.6.0 github.com/gorilla/csrf v1.7.3 github.com/gorilla/sessions v1.4.0 github.com/joho/godotenv v1.5.1 github.com/prometheus/client_golang v1.18.0 + github.com/prometheus/client_model v0.5.0 github.com/slok/go-http-metrics v0.11.0 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.11.1 go.uber.org/fx v1.20.1 golang.org/x/crypto v0.38.0 + gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/sqlite v1.5.4 gorm.io/gorm v1.25.5 modernc.org/sqlite v1.28.0 @@ -37,7 +40,6 @@ require ( github.com/mattn/go-sqlite3 v1.14.17 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect @@ -48,11 +50,10 @@ require ( go.uber.org/zap v1.23.0 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/sync v0.14.0 // indirect - golang.org/x/sys v0.33.0 // indirect + golang.org/x/sys v0.37.0 // indirect golang.org/x/text v0.25.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/protobuf v1.31.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/uint128 v1.2.0 // indirect modernc.org/cc/v3 v3.40.0 // indirect modernc.org/ccgo/v3 v3.16.13 // indirect diff --git a/go.sum b/go.sum index f7e0ba6..d2d615e 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,8 @@ github.com/go-chi/httprate v0.15.0 h1:j54xcWV9KGmPf/X4H32/aTH+wBlrvxL7P+SdnRqxh5 github.com/go-chi/httprate v0.15.0/go.mod h1:rzGHhVrsBn3IMLYDOZQsSU4fJNWcjui4fWKJcCId1R4= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= +github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -81,11 +83,11 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj github.com/slok/go-http-metrics v0.11.0 h1:ABJUpekCZSkQT1wQrFvS4kGbhea/w6ndFJaWJeh3zL0= github.com/slok/go-http-metrics v0.11.0/go.mod h1:ZGKeYG1ET6TEJpQx18BqAJAvxw9jBAZXCHU7bWQqqAc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.5.1 h1:4VhoImhV/Bm0ToFkXFi8hXNXwpDRZ/ynw3amt82mzq0= -github.com/stretchr/objx v0.5.1/go.mod h1:/iHQpkQwBD6DLUmQ4pE+s1TXdob1mORJ4/UFdrifcy0= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= @@ -109,8 +111,8 @@ golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= diff --git a/internal/banner/banner.go b/internal/banner/banner.go new file mode 100644 index 0000000..33dc6af --- /dev/null +++ b/internal/banner/banner.go @@ -0,0 +1,47 @@ +// Package banner renders the operator-facing blocks that carry a +// plaintext credential. +// +// A generated password printed as one more structured log line is lost: +// a boot writes roughly 45 fx PROVIDE/RUN/HOOK lines around it, and +// under `docker run -d` it is one line in a log subject to rotation. A +// credential that is shown exactly once has to be findable by eye when +// an operator scrolls back, so it is written as a ruled block rather +// than as a log record. +// +// It is deliberately not a log line: it goes straight to the writer the +// caller names — standard output for both the first-boot account and +// the `resetpw` subcommand — so it is neither levelled, filtered, nor +// rendered as JSON by whichever handler internal/logger installed. +package banner + +import ( + "fmt" + "io" + "strings" +) + +// ruleWidth is the length of the horizontal rules, chosen to fit an +// 80-column terminal without wrapping. +const ruleWidth = 72 + +// Credentials writes a ruled block naming an account and its plaintext +// password. headline says which event produced it, and note says what +// the operator must do about it; both are written verbatim, so a +// multi-line note must already be wrapped. +func Credentials( + w io.Writer, + headline, username, password, note string, +) error { + rule := strings.Repeat("=", ruleWidth) + + _, err := fmt.Fprintf( + w, + "\n%s\n%s\n\n username: %s\n password: %s\n\n%s\n%s\n\n", + rule, headline, username, password, note, rule, + ) + if err != nil { + return fmt.Errorf("writing credentials banner: %w", err) + } + + return nil +} diff --git a/internal/banner/banner_test.go b/internal/banner/banner_test.go new file mode 100644 index 0000000..f91e7cb --- /dev/null +++ b/internal/banner/banner_test.go @@ -0,0 +1,59 @@ +package banner_test + +import ( + "bytes" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/banner" +) + +// TestCredentials_IsFindableByEye pins the properties that make the +// block worth having: rules above and below it, the two fields on +// their own lines, and blank lines separating it from whatever the +// surrounding log wrote. +func TestCredentials_IsFindableByEye(t *testing.T) { + t.Parallel() + + var out bytes.Buffer + + require.NoError(t, banner.Credentials( + &out, "HEADLINE", "admin", "s3cret", "NOTE", + )) + + got := out.String() + lines := strings.Split(strings.Trim(got, "\n"), "\n") + + require.GreaterOrEqual(t, len(lines), 3) + assert.Equal(t, lines[0], lines[len(lines)-1], "rules must match") + assert.Greater( + t, len(lines[0]), 40, "the rule must be visible at a glance", + ) + assert.Equal(t, strings.Repeat("=", len(lines[0])), lines[0]) + + assert.Contains(t, got, "\n username: admin\n") + assert.Contains(t, got, "\n password: s3cret\n") + assert.Contains(t, got, "HEADLINE") + assert.Contains(t, got, "NOTE") + assert.True(t, strings.HasPrefix(got, "\n")) +} + +// failingWriter reports the write error a banner must not swallow: it +// is the one copy of a password that will never be shown again. +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { + return 0, assert.AnError +} + +func TestCredentials_ReportsAWriteFailure(t *testing.T) { + t.Parallel() + + err := banner.Credentials( + failingWriter{}, "HEADLINE", "admin", "s3cret", "NOTE", + ) + + require.ErrorIs(t, err, assert.AnError) +} diff --git a/internal/ciscript/ci_mark_superseded_test.go b/internal/ciscript/ci_mark_superseded_test.go new file mode 100644 index 0000000..860092f --- /dev/null +++ b/internal/ciscript/ci_mark_superseded_test.go @@ -0,0 +1,387 @@ +package ciscript_test + +import ( + "maps" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +const ( + // supersededDesc is the description script/ci-mark-superseded + // writes, and the one an earlier revision of it wrote alongside a + // `skipped` state. + supersededDesc = "Superseded by a newer commit; never tested" + + // liveContext is the commit-status context Gitea uses for this + // repository's runs, as seen in its API. The script derives it from + // the workflow and job names rather than hardcoding it; the + // derivation is checked against this value below. + liveContext = "check / check (push)" + + scriptPath = "../../script/ci-mark-superseded" + workflow = "../../.gitea/workflows/check.yml" + + // failure is the only state that neither folds into a combined + // `success` (as `skipped` does) nor blocks the commit forever (as + // `pending` does). + failure = "failure" +) + +// repo is a throwaway git history: parent is the commit a run would be +// cancelled on, head the commit that superseded it. +type repo struct { + dir string + head string + parent string +} + +// scriptEnv is the run identity the Gitea runner exports and the script +// builds its context string from. +type scriptEnv struct { + workflow string + job string + event string +} + +func defaultEnv() scriptEnv { + return scriptEnv{workflow: "check", job: "check", event: "push"} +} + +func cancelled() commitStatus { + return commitStatus{ + Context: liveContext, + Status: failure, + Description: "Has been cancelled", + } +} + +func running() commitStatus { + return commitStatus{ + Context: liveContext, + Status: "pending", + Description: "Has started running", + } +} + +func TestMarkSuperseded(t *testing.T) { + t.Parallel() + + cases := map[string]struct { + parent commitStatus + wantMark bool + }{ + "a cancelled run is marked": { + parent: cancelled(), + wantMark: true, + }, + "a laundered skipped status is marked": { + parent: commitStatus{ + Context: liveContext, + Status: "skipped", + Description: supersededDesc, + }, + wantMark: true, + }, + "a genuine failure is left alone": { + parent: commitStatus{ + Context: liveContext, + Status: failure, + Description: "Failing after 3m1s", + }, + wantMark: false, + }, + "a passing run is left alone": { + parent: commitStatus{ + Context: liveContext, + Status: "success", + Description: "Successful in 2m52s", + }, + wantMark: false, + }, + "another context is left alone": { + parent: commitStatus{ + Context: "other / other (push)", + Status: failure, + Description: "Has been cancelled", + }, + wantMark: false, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + requireTools(t) + + history := newRepo(t) + fake, api := newFakeGitea(t) + fake.setStatus(history.head, running()) + fake.setStatus(history.parent, tc.parent) + + out, err := runScript(t, history, api, defaultEnv()) + require.NoError(t, err, out) + + posted := fake.postedFor(history.parent) + if !tc.wantMark { + require.Empty(t, posted) + + return + } + + require.Equal(t, []postedStatus{{ + Context: liveContext, + // Not `skipped`: Gitea's combined status folds + // that into `success`, which is what made a + // never-tested commit read green. + State: failure, + Description: supersededDesc, + }}, posted) + }) + } +} + +// A second run must not rewrite what the first one wrote, or every +// later push would post a duplicate status. +func TestMarkSupersededIsIdempotent(t *testing.T) { + t.Parallel() + requireTools(t) + + history := newRepo(t) + fake, api := newFakeGitea(t) + fake.setStatus(history.head, running()) + fake.setStatus(history.parent, cancelled()) + + for range 2 { + out, err := runScript(t, history, api, defaultEnv()) + require.NoError(t, err, out) + } + + require.Len(t, fake.postedFor(history.parent), 1) +} + +// Renaming the workflow or the job changes the context string Gitea +// uses. The script must say so instead of quietly matching nothing. +func TestMarkSupersededRejectsAnUnknownContext(t *testing.T) { + t.Parallel() + requireTools(t) + + history := newRepo(t) + fake, api := newFakeGitea(t) + fake.setStatus(history.head, running()) + fake.setStatus(history.parent, cancelled()) + + env := defaultEnv() + env.job = "renamed" + + out, err := runScript(t, history, api, env) + require.Error(t, err) + require.Contains(t, out, "renamed") + require.Contains(t, out, liveContext) + require.Empty(t, fake.postedFor(history.parent)) +} + +// ANCESTOR_LIMIT is a documented knob. A value that is set but unusable +// must abort: handing it to git and discarding the exit status left the +// walk empty and the step green, marking nothing. +func TestMarkSupersededRejectsAnUnparseableAncestorLimit(t *testing.T) { + t.Parallel() + requireTools(t) + + history := newRepo(t) + fake, api := newFakeGitea(t) + fake.setStatus(history.head, running()) + fake.setStatus(history.parent, cancelled()) + + out, err := runScript( + t, history, api, defaultEnv(), "ANCESTOR_LIMIT=twenty", + ) + require.Error(t, err) + require.Contains(t, out, "ANCESTOR_LIMIT") + require.Contains(t, out, "twenty") + require.Empty(t, fake.postedFor(history.parent)) +} + +// A status read that fails is not the same as a commit with nothing to +// do. Losing curl's exit status through a pipe made the two identical +// and left a laundered commit laundered with no signal. +func TestMarkSupersededFailsOnAnUnreadableAncestorStatus(t *testing.T) { + t.Parallel() + requireTools(t) + + history := newRepo(t) + fake, api := newFakeGitea(t) + fake.setStatus(history.head, running()) + fake.setStatus(history.parent, cancelled()) + fake.failStatusRead(history.parent) + + out, err := runScript(t, history, api, defaultEnv()) + require.Error(t, err) + require.Contains(t, out, history.parent) + require.Contains(t, out, "cannot read commit statuses") + require.Empty(t, fake.postedFor(history.parent)) +} + +// A shallow clone cannot resolve the parent, so it is indistinguishable +// from a root commit to rev-parse and the walk would exit 0 having +// marked nothing. It must abort instead: dropping `fetch-depth: 0` from +// the checkout step is one edit, and a silent no-op there restores the +// false-green bug this script exists to prevent. +func TestMarkSupersededRejectsAShallowRepository(t *testing.T) { + t.Parallel() + requireTools(t) + + history := shallowClone(t, newRepo(t)) + fake, api := newFakeGitea(t) + fake.setStatus(history.head, running()) + fake.setStatus(history.parent, cancelled()) + + out, err := runScript(t, history, api, defaultEnv()) + require.Error(t, err) + require.Contains(t, out, "shallow repository") + require.Empty(t, fake.postedFor(history.parent)) + require.Empty(t, fake.postedFor(history.head)) +} + +// shallowClone returns the same history as a depth-1 clone. The `file://` +// URL is required: git ignores --depth for a plain local path. +func shallowClone(t *testing.T, history repo) repo { + t.Helper() + + dir := t.TempDir() + + //nolint:gosec // fixed argv, arguments are test-local paths + cmd := exec.CommandContext(t.Context(), "git", "clone", "-q", + "--depth=1", "file://"+history.dir, dir) + + out, err := cmd.CombinedOutput() + require.NoError(t, err, string(out)) + + return repo{dir: dir, head: history.head, parent: history.parent} +} + +// The derived context must equal the one Gitea actually uses, which is +// built from the same workflow and job names. +func TestDerivedContextMatchesGitea(t *testing.T) { + t.Parallel() + requireTools(t) + + name, job := workflowIdentity(t) + + history := newRepo(t) + fake, api := newFakeGitea(t) + fake.setStatus(history.head, running()) + fake.setStatus(history.parent, cancelled()) + + out, err := runScript(t, history, api, scriptEnv{ + workflow: name, + job: job, + event: "push", + }) + require.NoError(t, err, out) + + posted := fake.postedFor(history.parent) + require.Len(t, posted, 1) + require.Equal(t, liveContext, posted[0].Context) +} + +// workflowIdentity reads the workflow name and its single job id out of +// the checked-in workflow file. +func workflowIdentity(t *testing.T) (string, string) { + t.Helper() + + raw, err := os.ReadFile(workflow) + require.NoError(t, err) + + var parsed struct { + Name string `yaml:"name"` + Jobs map[string]any `yaml:"jobs"` + } + + require.NoError(t, yaml.Unmarshal(raw, &parsed)) + + jobs := slices.Collect(maps.Keys(parsed.Jobs)) + require.Len(t, jobs, 1) + + return parsed.Name, jobs[0] +} + +func runScript( + t *testing.T, history repo, api string, env scriptEnv, + extra ...string, +) (string, error) { + t.Helper() + + script, err := filepath.Abs(scriptPath) + require.NoError(t, err) + + //nolint:gosec // fixed argv, repo-local script under test + cmd := exec.CommandContext(t.Context(), "sh", script) + cmd.Dir = history.dir + cmd.Env = append(os.Environ(), + "GITHUB_API_URL="+api, + "GITHUB_REPOSITORY=sneak/webhooker", + "GITHUB_SHA="+history.head, + "GITHUB_WORKFLOW="+env.workflow, + "GITHUB_JOB="+env.job, + "GITHUB_EVENT_NAME="+env.event, + "GITEA_TOKEN=test-token", + ) + cmd.Env = append(cmd.Env, extra...) + + out, err := cmd.CombinedOutput() + + return string(out), err +} + +func newRepo(t *testing.T) repo { + t.Helper() + + dir := t.TempDir() + + git := func(args ...string) string { + //nolint:gosec // fixed argv, arguments are test constants + cmd := exec.CommandContext(t.Context(), "git", args...) + cmd.Dir = dir + + out, err := cmd.CombinedOutput() + require.NoError(t, err, string(out)) + + return strings.TrimSpace(string(out)) + } + + commit := func(message string) string { + git( + "-c", "user.email=ci@example.invalid", + "-c", "user.name=ci", + "-c", "commit.gpgsign=false", + "commit", "-q", "--allow-empty", "-m", message, + ) + + return git("rev-parse", "HEAD") + } + + git("init", "-q", "-b", "main") + + parent := commit("parent") + head := commit("head") + + return repo{dir: dir, head: head, parent: parent} +} + +func requireTools(t *testing.T) { + t.Helper() + + for _, tool := range []string{"sh", "git", "curl", "jq"} { + _, err := exec.LookPath(tool) + if err != nil { + t.Skipf("%s is not installed: %v", tool, err) + } + } +} diff --git a/internal/ciscript/doc.go b/internal/ciscript/doc.go new file mode 100644 index 0000000..3867bfd --- /dev/null +++ b/internal/ciscript/doc.go @@ -0,0 +1,10 @@ +// Package ciscript holds the tests for the repository's CI shell +// scripts in script/. It carries no runtime code: the scripts run on +// the CI runner, not inside the binary, but their behaviour still has +// to be verified by the test suite. +// +// The scripts under test are outside the Go build graph, so `go test`'s +// result cache serves a stale PASS when only a script changed: run the +// container build, or GOFLAGS=-count=1, to trust a result here after +// editing script/. +package ciscript diff --git a/internal/ciscript/fakegitea_test.go b/internal/ciscript/fakegitea_test.go new file mode 100644 index 0000000..a3a38e1 --- /dev/null +++ b/internal/ciscript/fakegitea_test.go @@ -0,0 +1,162 @@ +package ciscript_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" +) + +// commitStatus is the part of an entry in Gitea's combined-status +// response that script/ci-mark-superseded reads. +type commitStatus struct { + Context string `json:"context"` + Status string `json:"status"` + Description string `json:"description"` +} + +// postedStatus is the part of a create-status request body the script +// writes. +type postedStatus struct { + Context string `json:"context"` + State string `json:"state"` + Description string `json:"description"` +} + +// fakeGitea serves the two endpoints the script talks to. Like Gitea, +// the newest status for a context replaces the previous one, so a +// second run of the script sees what the first one wrote. +type fakeGitea struct { + mu sync.Mutex + statuses map[string][]commitStatus + posted map[string][]postedStatus + // failRead is a commit whose combined-status read answers HTTP + // 500, standing in for a status API that is down. + failRead string +} + +// newFakeGitea returns the fake and the base URL to hand the script as +// GITHUB_API_URL. +func newFakeGitea(t *testing.T) (*fakeGitea, string) { + t.Helper() + + fake := &fakeGitea{ + mu: sync.Mutex{}, + statuses: map[string][]commitStatus{}, + posted: map[string][]postedStatus{}, + failRead: "", + } + + srv := httptest.NewServer(fake.routes()) + t.Cleanup(srv.Close) + + return fake, srv.URL +} + +func (f *fakeGitea) routes() http.Handler { + mux := http.NewServeMux() + + mux.HandleFunc( + "GET /repos/{owner}/{repo}/commits/{sha}/status", + f.handleCombined, + ) + mux.HandleFunc( + "POST /repos/{owner}/{repo}/statuses/{sha}", + f.handleCreate, + ) + + return mux +} + +func (f *fakeGitea) handleCombined( + w http.ResponseWriter, r *http.Request, +) { + f.mu.Lock() + defer f.mu.Unlock() + + sha := r.PathValue("sha") + if f.failRead != "" && f.failRead == sha { + http.Error(w, "boom", http.StatusInternalServerError) + + return + } + + body := struct { + Statuses []commitStatus `json:"statuses"` + }{Statuses: f.statuses[sha]} + + payload, err := json.Marshal(body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + + return + } + + w.Header().Set("Content-Type", "application/json") + + _, _ = w.Write(payload) +} + +func (f *fakeGitea) handleCreate(w http.ResponseWriter, r *http.Request) { + var got postedStatus + + err := json.NewDecoder(r.Body).Decode(&got) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + + return + } + + sha := r.PathValue("sha") + + f.mu.Lock() + defer f.mu.Unlock() + + f.posted[sha] = append(f.posted[sha], got) + f.replaceLocked(sha, commitStatus{ + Context: got.Context, + Status: got.State, + Description: got.Description, + }) + + w.WriteHeader(http.StatusCreated) +} + +// failStatusRead makes the combined-status read for one commit answer +// HTTP 500. +func (f *fakeGitea) failStatusRead(sha string) { + f.mu.Lock() + defer f.mu.Unlock() + + f.failRead = sha +} + +// setStatus gives a commit its latest status for a context. +func (f *fakeGitea) setStatus(sha string, status commitStatus) { + f.mu.Lock() + defer f.mu.Unlock() + + f.replaceLocked(sha, status) +} + +// postedFor returns the statuses the script created for a commit. +func (f *fakeGitea) postedFor(sha string) []postedStatus { + f.mu.Lock() + defer f.mu.Unlock() + + return append([]postedStatus(nil), f.posted[sha]...) +} + +// replaceLocked requires f.mu. +func (f *fakeGitea) replaceLocked(sha string, status commitStatus) { + for i, existing := range f.statuses[sha] { + if existing.Context == status.Context { + f.statuses[sha][i] = status + + return + } + } + + f.statuses[sha] = append(f.statuses[sha], status) +} diff --git a/internal/config/config.go b/internal/config/config.go index 414a3bb..6a7a628 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -4,19 +4,19 @@ package config import ( "errors" "fmt" + "io/fs" "log/slog" + "net/netip" "os" "strconv" "strings" "time" + "github.com/getsentry/sentry-go" + "github.com/joho/godotenv" "go.uber.org/fx" "sneak.berlin/go/webhooker/internal/globals" "sneak.berlin/go/webhooker/internal/logger" - - // Populates the environment from a ./.env file automatically for - // development configuration. Kept in one place only (here). - _ "github.com/joho/godotenv/autoload" ) const ( @@ -25,18 +25,112 @@ const ( // EnvironmentProd represents production environment. EnvironmentProd = "prod" + // DefaultDataDir is where all SQLite databases live when DATA_DIR + // is unset. The same default applies in every environment. + DefaultDataDir = "/var/lib/webhooker" + // defaultPort is the default HTTP listen port. defaultPort = 8080 + // defaultBindAddress is the interface the plaintext HTTP + // listener claims when BIND_ADDRESS is unset. + // + // Loopback, because the listener speaks cleartext and serves + // both the admin UI and the unauthenticated receiver: a + // wildcard default publishes them on every interface of every + // host that never configured anything, which is the failure + // this default exists to prevent. Reaching webhooker from off + // the host is then a deliberate act — a reverse proxy in front + // of it, or an explicit BIND_ADDRESS. + // + // This is the binary's default only. The Dockerfile ships + // ENV BIND_ADDRESS=0.0.0.0, so a container deployment needs + // nothing set and is unaffected by this constant. The two + // differ because they answer different questions: a container's + // network namespace is already the boundary this default is + // reaching for, so binding every address inside it exposes + // nothing, and what decides exposure there is the publish flag + // (-p 127.0.0.1:8080:8080). A loopback bind inside a container + // buys no security and makes the process unreachable through + // its own published port. + // + // The split is expressed as two explicit defaults rather than + // container auto-detection, because a heuristic that guesses + // wrong opens the cleartext port exactly where nobody is + // looking. + defaultBindAddress = "127.0.0.1" + // defaultRetentionSweepInterval is how often the retention // reaper deletes events older than each webhook's RetentionDays. defaultRetentionSweepInterval = time.Hour + + // defaultSessionIdleTimeout is how long a session may go without + // authenticated activity before it expires. + defaultSessionIdleTimeout = 24 * time.Hour + + // defaultReceiverRateLimit is the default number of requests + // per minute each client IP may send to a single webhook + // receiver entrypoint. Generous for legitimate webhook + // senders while bounding abuse of the one unauthenticated, + // internet-exposed endpoint. + defaultReceiverRateLimit = 120 + + // maxPort is the highest valid TCP port number. The lower + // bound (at least 1) is enforced by envPositiveInt. + maxPort = 65535 + + // mappedV4Offset is the number of leading bits an IPv4-mapped + // IPv6 prefix spends on the ::ffff:0:0/96 wrapper, so a /104 + // covers the same addresses as an IPv4 /8. + mappedV4Offset = 96 + + // DotEnvPath is the optional file of KEY=value lines read into the + // environment at startup, relative to the process working + // directory. Exported so that documentation and tests name the + // same path the loader opens. + DotEnvPath = ".env" ) // ErrInvalidEnvironment is returned when WEBHOOKER_ENVIRONMENT // contains an unrecognised value. var ErrInvalidEnvironment = errors.New("invalid environment") +// ErrNonPositiveValue is returned when an environment variable that +// requires a positive integer is set to zero or a negative number. +var ErrNonPositiveValue = errors.New("value must be positive") + +// ErrInvalidPort is returned when an environment variable holding a +// TCP port number is set above the valid port range. +var ErrInvalidPort = errors.New("invalid port") + +// ErrInvalidCIDR is returned when an environment variable holding a +// list of CIDR blocks contains an entry that is neither a CIDR block +// nor a bare IP address. +var ErrInvalidCIDR = errors.New("invalid CIDR") + +// ErrInvalidBindAddress is returned when BIND_ADDRESS is set to +// something that is not an IP address literal. +var ErrInvalidBindAddress = errors.New("invalid bind address") + +// ErrInvalidSentryDSN is returned when SENTRY_DSN is set to something +// the Sentry SDK cannot parse as a DSN. +var ErrInvalidSentryDSN = errors.New("invalid Sentry DSN") + +// ErrDotEnvUnreadable is returned when the optional .env file exists +// but cannot be read or parsed. A file that is not there is not an +// error; a file that is there and broken is. +var ErrDotEnvUnreadable = errors.New("unreadable .env file") + +// ErrIncompleteMetricsAuth is returned when exactly one of +// METRICS_USERNAME and METRICS_PASSWORD carries a value. Neither +// fallback is acceptable: serving /metrics on the username alone +// publishes an endpoint whose password is the empty string, and +// silently leaving it unmounted withholds an endpoint the operator +// asked for. Half-set is a configuration error, so startup fails. +var ErrIncompleteMetricsAuth = errors.New( + "incomplete metrics credentials", +) + //nolint:revive // ConfigParams is a standard fx naming convention. type ConfigParams struct { fx.In @@ -57,9 +151,52 @@ type Config struct { Port int SentryDSN string + // BindAddress is the IP address the plaintext HTTP listener + // binds, as an address literal. It defaults to + // defaultBindAddress and is never empty: an empty string would + // mean the wildcard to net.Listen, which is the opposite of the + // default this ships. + BindAddress string + // RetentionSweepInterval is how often the retention reaper runs. + // Always positive: it becomes a time.NewTicker period. RetentionSweepInterval time.Duration + // SessionIdleTimeout is the sliding inactivity window after + // which a session expires. Non-positive disables idle expiry. + SessionIdleTimeout time.Duration + + // ReceiverRateLimit is the number of requests per minute each + // client IP may send to a single webhook receiver entrypoint. + ReceiverRateLimit int + + // TrustedProxies is the set of networks whose members are + // allowed to speak for the client with X-Forwarded-For, the + // only forwarded header read. It is empty unless + // TRUSTED_PROXIES is set, and empty means no peer is + // trusted: forwarded headers are then ignored entirely and + // clients are identified by the connection's own address. + // Members can choose their own rate-limit key, so this must + // name proxy hosts only, never a block that also covers + // clients. + TrustedProxies []netip.Prefix + + // AllowedEgressCIDRs is the set of networks a delivery target + // may reach even though the SSRF guard's default blocklist + // covers them. It is empty unless ALLOWED_EGRESS_CIDRS is set, + // and empty means every private/reserved range stays refused. + // + // This only ever adds destinations to what the guard would + // otherwise refuse. The guard itself is always on: there is no + // setting that disables SSRF protection, and delivery's + // alwaysBlockedNetworks stays blocked no matter what is listed + // here. That set is link-local plus the cloud metadata + // endpoints outside it that disclose credentials or user data + // at a provider-fixed address; it is not exhaustive of every + // cloud's metadata address. See alwaysBlockedNetworks for the + // authoritative list and the criterion it is built from. + AllowedEgressCIDRs []netip.Prefix + params *ConfigParams log *slog.Logger } @@ -74,34 +211,166 @@ func (c *Config) IsProd() bool { return c.Environment == EnvironmentProd } +// MetricsAuthEnabled reports whether /metrics is served behind basic +// auth. It is the only answer to that question in the codebase: the +// route mount, the Prometheus recording middleware and the startup +// log's hasMetricsAuth field all read this one method, so the log +// cannot report auth as off while the route is mounted. +// +// It requires both credentials rather than the username alone. +// loadFromEnv already rejects a half-set pair, but a Config built in +// code bypasses that, and the failure mode this guards is an endpoint +// mounted with a credential map whose only password is the empty +// string. +func (c *Config) MetricsAuthEnabled() bool { + return c.MetricsUsername != "" && c.MetricsPassword != "" +} + +// SentryEnabled reports whether error reporting is shipped to Sentry. +// It is the only answer to that question in the codebase: the SDK +// initialisation, the sentryhttp middleware registration and the +// startup log's sentryEnabled field all read this one method, so the +// log cannot report reporting as on while nothing is sending. +// +// A non-empty DSN is enough because loadFromEnv already parsed it with +// the SDK's own parser and refused to build a Config around one the +// SDK would reject, and because initialising the SDK with a DSN that +// parsed and failed anyway aborts the process rather than leaving this +// true and the client absent. +func (c *Config) SentryEnabled() bool { + return c.SentryDSN != "" +} + // envString returns the value of the named environment variable, // or an empty string if not set. func envString(key string) string { return os.Getenv(key) } -// envBool returns the value of the named environment variable -// parsed as a boolean. Returns defaultValue if not set. -func envBool(key string, defaultValue bool) bool { - if v := os.Getenv(key); v != "" { - return strings.EqualFold(v, "true") || v == "1" - } - - return defaultValue +// LoadDotEnv reads DotEnvPath into the environment when that file is +// present, and reports a file that is present but broken. +// +// It has to run before anything reads the environment, so that every +// reader agrees on what the environment holds — the DATA_DIR lock +// taken before the fx graph exists as much as loadFromEnv itself. A +// variable already set in the real environment wins: godotenv never +// overwrites one. +// +// A missing file is not an error. It is a development convenience and +// most deployments set the environment directly. +// +// Any other failure is. godotenv parses the whole file before setting +// anything, so a single malformed line applies none of it: every +// variable in the file silently reverts to its default, which defeats +// the fail-loud guarantee for all of them at once. +func LoadDotEnv() error { + return loadDotEnvFile(DotEnvPath) } -// envInt returns the value of the named environment variable -// parsed as an integer. Returns defaultValue if not set or -// unparseable. -func envInt(key string, defaultValue int) int { - if v := os.Getenv(key); v != "" { - i, err := strconv.Atoi(v) - if err == nil { - return i - } +// loadDotEnvFile is LoadDotEnv over a named file, so tests can point +// at a temporary one instead of the process working directory. +func loadDotEnvFile(path string) error { + err := godotenv.Load(path) + if err == nil || errors.Is(err, fs.ErrNotExist) { + return nil } - return defaultValue + return fmt.Errorf( + "%w: %s: %w; nothing in it was applied, so fix the file or "+ + "remove it", + ErrDotEnvUnreadable, path, err, + ) +} + +// DataDir resolves DATA_DIR, applying DefaultDataDir when it is unset +// or empty. It is exported so that entry points which must act on the +// data directory before the fx graph exists — taking the exclusive +// directory lock, above all — resolve it exactly as Config does. +func DataDir() string { + dir := envString("DATA_DIR") + if dir == "" { + return DefaultDataDir + } + + return dir +} + +// envBool returns the value of the named environment variable +// parsed as a boolean. Returns defaultValue if not set. If the +// variable is set but cannot be parsed, it returns a wrapped error +// naming the key and the bad value, so startup fails loudly rather +// than silently falling back to the default. +// +// Parsing is strconv.ParseBool, which accepts 1, t, T, TRUE, true, +// True, 0, f, F, FALSE, false and False. Anything else — "yes", +// "on", or a typo like "ture" — is an error rather than a silent +// false. +func envBool(key string, defaultValue bool) (bool, error) { + v := os.Getenv(key) + if v == "" { + return defaultValue, nil + } + + b, err := strconv.ParseBool(v) + if err != nil { + return false, fmt.Errorf( + "invalid boolean for %s: %q: %w", key, v, err, + ) + } + + return b, nil +} + +// envPositiveInt returns the value of the named environment variable +// parsed as a positive integer. Returns defaultValue if not set. If +// the variable is set but cannot be parsed, or parses to less than +// one, it returns a wrapped error naming the key and the bad value, +// so startup fails loudly rather than silently falling back to the +// default. +func envPositiveInt( + key string, + defaultValue int, +) (int, error) { + v := os.Getenv(key) + if v == "" { + return defaultValue, nil + } + + i, err := strconv.Atoi(v) + if err != nil { + return 0, fmt.Errorf( + "invalid integer for %s: %q: %w", key, v, err, + ) + } + + if i < 1 { + return 0, fmt.Errorf( + "%w: %s must be at least 1, got %q", + ErrNonPositiveValue, key, v, + ) + } + + return i, nil +} + +// envPort returns the value of the named environment variable parsed +// as a TCP port number. Returns defaultValue if not set. A set value +// that is unparseable, below 1, or above maxPort is a hard error +// naming the key and the bad value. +func envPort(key string, defaultValue int) (int, error) { + port, err := envPositiveInt(key, defaultValue) + if err != nil { + return 0, err + } + + if port > maxPort { + return 0, fmt.Errorf( + "%w: %s must be at most %d, got %d", + ErrInvalidPort, key, maxPort, port, + ) + } + + return port, nil } // envDuration returns the value of the named environment variable @@ -128,33 +397,262 @@ func envDuration( return d, nil } -// New creates a Config by reading environment variables. +// envPositiveDuration returns the value of the named environment +// variable parsed as a Go duration that must be greater than zero. +// Returns defaultValue if not set. A set value that is unparseable or +// non-positive is a hard error naming the key and the bad value. // -//nolint:revive // lc parameter is required by fx even if unused. -func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) { - log := params.Logger.Get() +// This is for durations that reach time.NewTicker, which panics on a +// non-positive period, in a goroutine started after startup has +// already reported success. It is deliberately not used for durations +// where non-positive means "disabled" (SESSION_IDLE_TIMEOUT). +func envPositiveDuration( + key string, + defaultValue time.Duration, +) (time.Duration, error) { + d, err := envDuration(key, defaultValue) + if err != nil { + return 0, err + } - // Determine environment from WEBHOOKER_ENVIRONMENT env var, - // default to dev + if d <= 0 { + return 0, fmt.Errorf( + "%w: %s must be greater than zero, got %s", + ErrNonPositiveValue, key, d, + ) + } + + return d, nil +} + +// parseCIDR parses one trusted-proxy list entry, which may be a +// CIDR block ("10.0.0.0/8") or a bare address ("10.0.0.1", treated +// as a single-host block). +// +// Both forms are unmapped, because peer addresses are unmapped +// before they are matched against the list: an IPv4-mapped prefix +// left in that form would silently never match. +func parseCIDR(entry string) (netip.Prefix, error) { + if strings.Contains(entry, "/") { + prefix, err := netip.ParsePrefix(entry) + if err != nil { + return netip.Prefix{}, err //nolint:wrapcheck // wrapped by caller + } + + if addr := prefix.Addr(); addr.Is4In6() && + prefix.Bits() >= mappedV4Offset { + prefix = netip.PrefixFrom( + addr.Unmap(), prefix.Bits()-mappedV4Offset, + ) + } + + return prefix.Masked(), nil + } + + addr, err := netip.ParseAddr(entry) + if err != nil { + return netip.Prefix{}, err //nolint:wrapcheck // wrapped by caller + } + + return netip.PrefixFrom(addr.Unmap(), addr.Unmap().BitLen()), nil +} + +// envPrefixList returns the value of the named environment variable +// parsed as a comma-separated list of CIDR blocks (bare addresses +// allowed). An unset, empty, or blank value yields an empty list. A +// set value containing an unparseable entry is a hard error naming +// the key and the bad entry, so startup fails loudly rather than +// silently running with a list the operator did not intend. +func envPrefixList(key string) ([]netip.Prefix, error) { + v := strings.TrimSpace(os.Getenv(key)) + if v == "" { + return nil, nil + } + + var prefixes []netip.Prefix + + for entry := range strings.SplitSeq(v, ",") { + entry = strings.TrimSpace(entry) + if entry == "" { + continue + } + + prefix, err := parseCIDR(entry) + if err != nil { + return nil, fmt.Errorf( + "%w: %s: %q: %w", ErrInvalidCIDR, key, entry, err, + ) + } + + prefixes = append(prefixes, prefix) + } + + return prefixes, nil +} + +// envBindAddress returns the value of the named environment variable +// parsed as an IP address literal. An unset (or empty, or +// whitespace-only) value yields defaultValue. +// +// Only literals are accepted: no hostname is resolved, so `localhost` +// is an error rather than a DNS lookup at startup whose answer could +// be either loopback family, could change under the process, and +// could return several addresses of which only one would be bound. A +// value with a port in it (`127.0.0.1:8080`) is likewise an error — +// the port is PORT's business, and silently accepting it would bind +// something other than what was asked for. +// +// A set value that is not a literal is a hard error naming the key +// and the bad value, so startup fails loudly rather than falling back +// to a default the operator plainly did not want. A literal that is +// not an address of this host parses here and fails at listen time +// instead, which ends the process non-zero. +func envBindAddress(key, defaultValue string) (string, error) { + v := strings.TrimSpace(os.Getenv(key)) + if v == "" { + return defaultValue, nil + } + + addr, err := netip.ParseAddr(v) + if err != nil { + return "", fmt.Errorf( + "%w: %s: %q must be an IP address literal such as "+ + "127.0.0.1, 0.0.0.0 or ::, not a hostname and not "+ + "host:port: %w", + ErrInvalidBindAddress, key, v, err, + ) + } + + return addr.String(), nil +} + +// envSentryDSN returns the value of the named environment variable +// checked as a Sentry DSN. An unset (or empty, or whitespace-only) +// value yields "", which means error reporting stays off — the common +// case, and a normal start. +// +// A set value is parsed with sentry.NewDsn, which is the call +// sentry.Init makes on the DSN it is handed, so what passes here is +// exactly what the SDK will accept later and the two cannot disagree. +// Reproducing the check by hand instead would cost this package its +// dependency on the SDK — already a module dependency, already linked +// into the binary — in exchange for a second definition of "valid DSN" +// free to drift from the one that decides. +// +// A set value that does not parse is a hard error naming the key, so +// startup fails loudly. Losing error reporting is the failure this +// variable exists to prevent, and a typo in a DSN is silent forever: +// nothing later in the process can notice that reports are going +// nowhere. The bad value is quoted because it is a URL to a public +// endpoint carrying a public key, not a secret. +func envSentryDSN(key string) (string, error) { + v := strings.TrimSpace(os.Getenv(key)) + if v == "" { + return "", nil + } + + _, err := sentry.NewDsn(v) + if err != nil { + return "", fmt.Errorf( + "%w: %s: %q: %w", ErrInvalidSentryDSN, key, v, err, + ) + } + + return v, nil +} + +// resolveMetricsAuth reads the /metrics basic-auth credentials and +// rejects a half-set pair, naming both variables either way. The +// error carries neither value: the password is a secret. +func resolveMetricsAuth() (string, string, error) { + username := envString("METRICS_USERNAME") + password := envString("METRICS_PASSWORD") + + if (username == "") == (password == "") { + return username, password, nil + } + + set, empty := "METRICS_USERNAME", "METRICS_PASSWORD" + if username == "" { + set, empty = empty, set + } + + return "", "", fmt.Errorf( + "%w: %s is set but %s is empty; METRICS_USERNAME and "+ + "METRICS_PASSWORD must both be set to serve /metrics, "+ + "or both be empty to leave it unmounted", + ErrIncompleteMetricsAuth, set, empty, + ) +} + +// resolveEnvironment reads WEBHOOKER_ENVIRONMENT, defaulting to +// dev, and rejects unrecognised values. +func resolveEnvironment() (string, error) { environment := os.Getenv("WEBHOOKER_ENVIRONMENT") if environment == "" { environment = EnvironmentDev } - // Validate environment if environment != EnvironmentDev && environment != EnvironmentProd { - return nil, fmt.Errorf( + return "", fmt.Errorf( "%w: WEBHOOKER_ENVIRONMENT must be '%s' or '%s', got '%s'", ErrInvalidEnvironment, EnvironmentDev, EnvironmentProd, environment, ) } - // Parse the retention sweep interval; a set-but-unparseable value - // is a hard error so fx aborts startup rather than silently using - // the default. - retentionSweepInterval, err := envDuration( + return environment, nil +} + +// resolveListener reads the two variables that describe the HTTP +// listener: which port it claims and which address it claims it on. +// They are read together because neither is meaningful alone, and +// because a validation failure in either has to abort startup before +// anything binds. +func resolveListener() (int, string, error) { + port, err := envPort("PORT", defaultPort) + if err != nil { + return 0, "", err + } + + bindAddress, err := envBindAddress( + "BIND_ADDRESS", defaultBindAddress, + ) + if err != nil { + return 0, "", err + } + + return port, bindAddress, nil +} + +// loadFromEnv builds a Config from the environment. Every value that +// needs parsing fails loudly when it is set but unparseable: the +// documented defaults apply only to variables that are unset (or +// empty), never as a substitute for a value the operator actually +// provided. +func loadFromEnv() (*Config, error) { + environment, err := resolveEnvironment() + if err != nil { + return nil, err + } + + port, bindAddress, err := resolveListener() + if err != nil { + return nil, err + } + + debug, err := envBool("DEBUG", false) + if err != nil { + return nil, err + } + + maintenanceMode, err := envBool("MAINTENANCE_MODE", false) + if err != nil { + return nil, err + } + + retentionSweepInterval, err := envPositiveDuration( "RETENTION_SWEEP_INTERVAL", defaultRetentionSweepInterval, ) @@ -162,29 +660,164 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) { return nil, err } - // Load configuration values from environment variables - s := &Config{ - DataDir: envString("DATA_DIR"), - Debug: envBool("DEBUG", false), - MaintenanceMode: envBool("MAINTENANCE_MODE", false), - Environment: environment, - MetricsUsername: envString("METRICS_USERNAME"), - MetricsPassword: envString("METRICS_PASSWORD"), - Port: envInt("PORT", defaultPort), - SentryDSN: envString("SENTRY_DSN"), - RetentionSweepInterval: retentionSweepInterval, - log: log, - params: ¶ms, + // Non-positive is "disabled" here, not invalid, so this stays on + // envDuration. + sessionIdleTimeout, err := envDuration( + "SESSION_IDLE_TIMEOUT", + defaultSessionIdleTimeout, + ) + if err != nil { + return nil, err } - // Set default DataDir. All SQLite databases (main application - // DB and per-webhook event DBs) live here. The same default is - // used regardless of environment; override with DATA_DIR if - // needed. - if s.DataDir == "" { - s.DataDir = "/var/lib/webhooker" + receiverRateLimit, err := envPositiveInt( + "RECEIVER_RATE_LIMIT", + defaultReceiverRateLimit, + ) + if err != nil { + return nil, err } + trustedProxies, err := envPrefixList("TRUSTED_PROXIES") + if err != nil { + return nil, err + } + + allowedEgressCIDRs, err := envPrefixList("ALLOWED_EGRESS_CIDRS") + if err != nil { + return nil, err + } + + metricsUsername, metricsPassword, err := resolveMetricsAuth() + if err != nil { + return nil, err + } + + sentryDSN, err := envSentryDSN("SENTRY_DSN") + if err != nil { + return nil, err + } + + return &Config{ + DataDir: DataDir(), + Debug: debug, + MaintenanceMode: maintenanceMode, + Environment: environment, + MetricsUsername: metricsUsername, + MetricsPassword: metricsPassword, + Port: port, + BindAddress: bindAddress, + SentryDSN: sentryDSN, + RetentionSweepInterval: retentionSweepInterval, + SessionIdleTimeout: sessionIdleTimeout, + ReceiverRateLimit: receiverRateLimit, + TrustedProxies: trustedProxies, + AllowedEgressCIDRs: allowedEgressCIDRs, + }, nil +} + +// PrefixStrings renders a prefix list as its CIDR strings, for +// logging a list an operator has to be able to read back. +func PrefixStrings(prefixes []netip.Prefix) []string { + out := make([]string, 0, len(prefixes)) + + for _, prefix := range prefixes { + out = append(out, prefix.String()) + } + + return out +} + +// warnEgressAllowlist logs the effective ALLOWED_EGRESS_CIDRS +// whenever it is non-empty. +// +// It prints the blocks themselves rather than a count, because +// this is the one setting that lets a delivery target reach the +// host's own network: an operator reading the startup log has to +// be able to see exactly which hole is open. Silence means the +// list is empty and the SSRF guard is refusing every +// private/reserved range, which is the default. +func (c *Config) warnEgressAllowlist(log *slog.Logger) { + if len(c.AllowedEgressCIDRs) == 0 { + return + } + + log.Warn( + "ALLOWED_EGRESS_CIDRS lets delivery targets reach these "+ + "otherwise-blocked private/reserved networks. Anyone "+ + "who can create a delivery target can now make this "+ + "process issue requests into them, and read back the "+ + "response. Link-local and the known cloud instance "+ + "metadata endpoints outside it stay blocked "+ + "regardless of what is listed here.", + "allowedEgressCIDRs", + strings.Join(PrefixStrings(c.AllowedEgressCIDRs), ","), + ) +} + +// warnSharedRateLimitBucket logs a startup warning whenever +// TRUSTED_PROXIES is empty, in any environment. +// +// With no trusted proxies every rate limiter keys on the connecting +// peer's address. Whether that is harmless or dangerous depends on +// what is in front of the process, which this code cannot observe: +// with nothing in front, the peer is the client and the limits are +// per-client as intended; behind a reverse proxy the peer is the proxy +// for every request, so all clients share one bucket per limiter. +// +// The login endpoint no longer spends budget on arrival — it verifies +// credentials first and charges only failures — so a shared bucket +// cannot deny the operator a correct password. What it does collapse +// is the failure counting: one client's wrong passwords throttle +// everyone else's wrong passwords, and the receiver's limits become +// service-wide ceilings. +// +// The warning is deliberately not gated on WEBHOOKER_ENVIRONMENT. That +// variable defaults to dev, so gating on it would silence the warning +// for exactly the operator who forgot to configure the deployment — +// the case it exists to catch. +// +// The default of trusting nobody is deliberate — trusting forwarded +// headers from arbitrary peers lets any client choose its own bucket — +// so this warns rather than failing startup or changing the key. +func (c *Config) warnSharedRateLimitBucket(log *slog.Logger) { + if len(c.TrustedProxies) > 0 { + return + } + + log.Warn( + "TRUSTED_PROXIES is empty: every rate limit keys on the "+ + "connecting peer's address. With nothing proxying to "+ + "this process that is the client itself and the limits "+ + "are per-client as intended. Behind a reverse proxy the "+ + "peer is the proxy on every request, so all clients "+ + "share one bucket per limit: the receiver limits become "+ + "service-wide ceilings, and one client's failed logins "+ + "throttle every other client's failed logins — a "+ + "correct password still gets in. If anything proxies to "+ + "this process, set TRUSTED_PROXIES to its address.", + "environment", c.Environment, + "trustedProxies", len(c.TrustedProxies), + ) +} + +// New creates a Config by reading environment variables. +// +//nolint:revive // lc parameter is required by fx even if unused. +func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) { + log := params.Logger.Get() + + // A set-but-unparseable value anywhere in the environment is a + // hard error, so fx aborts startup rather than running with a + // silently substituted default. + s, err := loadFromEnv() + if err != nil { + return nil, err + } + + s.log = log + s.params = ¶ms + if s.Debug { params.Logger.EnableDebugLogging() } @@ -193,14 +826,28 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) { log.Info("Configuration loaded", "environment", s.Environment, "port", s.Port, + // Logged because which interfaces the cleartext listener + // answers on is not otherwise observable from inside a + // container, and it decides whether anything but the local + // host can reach the admin UI. + "bindAddress", s.BindAddress, "debug", s.Debug, "maintenanceMode", s.MaintenanceMode, "dataDir", s.DataDir, "retentionSweepInterval", s.RetentionSweepInterval.String(), - "hasSentryDSN", s.SentryDSN != "", - "hasMetricsAuth", - s.MetricsUsername != "" && s.MetricsPassword != "", + // Logged because a perfectly valid non-positive value here + // disables idle expiry entirely, and that is worth showing + // back to the operator. + "sessionIdleTimeout", s.SessionIdleTimeout.String(), + "receiverRateLimit", s.ReceiverRateLimit, + "trustedProxies", len(s.TrustedProxies), + "allowedEgressCIDRs", len(s.AllowedEgressCIDRs), + "sentryEnabled", s.SentryEnabled(), + "hasMetricsAuth", s.MetricsAuthEnabled(), ) + s.warnSharedRateLimitBucket(log) + s.warnEgressAllowlist(log) + return s, nil } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 7e3c2c8..f38f7fd 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -1,6 +1,8 @@ package config_test import ( + "bytes" + "log/slog" "os" "testing" "time" @@ -14,6 +16,24 @@ import ( "sneak.berlin/go/webhooker/internal/logger" ) +// Shared subtest names for the env-parsing tables below, which all +// exercise the same three cases against different variables. +const ( + caseUnsetUsesDefault = "unset uses default" + caseValidValueParsed = "valid value is parsed" + caseUnparseableFails = "unparseable value fails startup" + + // cidrPrivateV4 is the sample trusted-proxy block the + // TRUSTED_PROXIES cases are built from. + cidrPrivateV4 = "10.0.0.0/8" + + // metricsAuthValue is the sample METRICS_PASSWORD the metrics + // credential cases are built from. It is asserted absent from + // the startup error, so it must not be a substring of either + // variable name that error prints. + metricsAuthValue = "s3cret" +) + func TestEnvironmentConfig(t *testing.T) { tests := []struct { name string @@ -127,25 +147,47 @@ func TestRetentionSweepInterval(t *testing.T) { set bool value string expectError bool - expected time.Duration + // sentinel, when set, must be wrapped by the startup + // error; every error case must additionally name the + // variable in its message. + sentinel error + expected time.Duration }{ { - name: "unset uses default", + name: caseUnsetUsesDefault, set: false, expected: time.Hour, }, { - name: "valid value is parsed", + name: caseValidValueParsed, set: true, value: "15m", expected: 15 * time.Minute, }, { - name: "unparseable value fails startup", + name: caseUnparseableFails, set: true, value: "not-a-duration", expectError: true, }, + { + // A non-positive period panics the ticker in the + // reaper and archive-sweeper goroutines, long after + // startup has reported success, so it has to fail + // here instead. + name: "zero fails startup", + set: true, + value: "0s", + expectError: true, + sentinel: config.ErrNonPositiveValue, + }, + { + name: "negative fails startup", + set: true, + value: "-1h", + expectError: true, + sentinel: config.ErrNonPositiveValue, + }, } for _, tt := range tests { @@ -163,7 +205,9 @@ func TestRetentionSweepInterval(t *testing.T) { } if tt.expectError { - testRetentionSweepIntervalError(t) + expectStartupErrorFor( + t, "RETENTION_SWEEP_INTERVAL", tt.sentinel, + ) } else { testRetentionSweepIntervalSuccess(t, tt.expected) } @@ -171,7 +215,10 @@ func TestRetentionSweepInterval(t *testing.T) { } } -func testRetentionSweepIntervalError(t *testing.T) { +// startupError builds the app config.New belongs to and returns +// the error fx reports, which is non-nil whenever an environment +// value is set but invalid. +func startupError(t *testing.T) error { t.Helper() var cfg *config.Config @@ -186,7 +233,33 @@ func testRetentionSweepIntervalError(t *testing.T) { fx.Populate(&cfg), ) - assert.Error(t, app.Err()) + return app.Err() +} + +// expectStartupError asserts that fx refuses to build the app, +// which is what a set-but-invalid environment value must cause. +func expectStartupError(t *testing.T) { + t.Helper() + + assert.Error(t, startupError(t)) +} + +// expectStartupErrorFor asserts that startup fails, that the error +// names the offending variable so an operator can find it, and, +// when sentinel is non-nil, that it wraps that sentinel. +func expectStartupErrorFor( + t *testing.T, + key string, + sentinel error, +) { + t.Helper() + + err := startupError(t) + require.ErrorContains(t, err, key) + + if sentinel != nil { + require.ErrorIs(t, err, sentinel) + } } func testRetentionSweepIntervalSuccess( @@ -215,6 +288,98 @@ func testRetentionSweepIntervalSuccess( assert.Equal(t, expected, cfg.RetentionSweepInterval) } +func TestSessionIdleTimeout(t *testing.T) { + tests := []struct { + name string + set bool + value string + expectError bool + expected time.Duration + }{ + { + name: caseUnsetUsesDefault, + set: false, + expected: 24 * time.Hour, + }, + { + name: caseValidValueParsed, + set: true, + value: "30m", + expected: 30 * time.Minute, + }, + { + name: caseUnparseableFails, + set: true, + value: "not-a-duration", + expectError: true, + }, + { + // Non-positive is "idle expiry disabled" for this + // variable, not a configuration error: unlike + // RETENTION_SWEEP_INTERVAL it never becomes a ticker + // period. + name: "zero disables idle expiry", + set: true, + value: "0s", + expected: 0, + }, + { + name: "negative disables idle expiry", + set: true, + value: "-1h", + expected: -time.Hour, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Cannot use t.Parallel() here because t.Setenv + // is incompatible with parallel subtests. + t.Setenv("WEBHOOKER_ENVIRONMENT", "dev") + + if tt.set { + t.Setenv("SESSION_IDLE_TIMEOUT", tt.value) + } else { + require.NoError(t, os.Unsetenv( + "SESSION_IDLE_TIMEOUT", + )) + } + + if tt.expectError { + expectStartupError(t) + } else { + testSessionIdleTimeoutSuccess(t, tt.expected) + } + }) + } +} + +func testSessionIdleTimeoutSuccess( + t *testing.T, + expected time.Duration, +) { + t.Helper() + + var cfg *config.Config + + app := fxtest.New( + t, + fx.Provide( + globals.New, + logger.New, + config.New, + ), + fx.Populate(&cfg), + ) + require.NoError(t, app.Err()) + + app.RequireStart() + + defer app.RequireStop() + + assert.Equal(t, expected, cfg.SessionIdleTimeout) +} + func TestDefaultDataDir(t *testing.T) { for _, env := range []string{"", "dev", "prod"} { name := env @@ -258,3 +423,688 @@ func TestDefaultDataDir(t *testing.T) { }) } } + +// TestDataDirHelper pins the exported resolver against the value +// Config carries. The entry point takes the DATA_DIR lock through the +// helper before the fx graph exists, so the two disagreeing would mean +// locking one directory and writing to another. +func TestDataDirHelper(t *testing.T) { + for _, set := range []string{"", "/tmp/webhooker-datadir-helper"} { + name := "set" + if set == "" { + name = "unset" + } + + t.Run(name, func(t *testing.T) { + // Cannot use t.Parallel() here because t.Setenv + // is incompatible with parallel subtests. + if set == "" { + require.NoError(t, os.Unsetenv("DATA_DIR")) + } else { + t.Setenv("DATA_DIR", set) + } + + expected := set + if expected == "" { + expected = config.DefaultDataDir + } + + assert.Equal(t, expected, config.DataDir()) + }) + } +} + +func TestReceiverRateLimit(t *testing.T) { + tests := []struct { + name string + set bool + value string + expectError bool + // sentinel, when set, must be wrapped by the startup + // error; every error case must additionally name the + // variable in its message. + sentinel error + expected int + }{ + { + name: caseUnsetUsesDefault, + set: false, + expected: 120, + }, + { + name: caseValidValueParsed, + set: true, + value: "30", + expected: 30, + }, + { + name: caseUnparseableFails, + set: true, + value: "not-a-number", + expectError: true, + }, + { + name: "zero fails startup", + set: true, + value: "0", + expectError: true, + sentinel: config.ErrNonPositiveValue, + }, + { + name: "negative fails startup", + set: true, + value: "-5", + expectError: true, + sentinel: config.ErrNonPositiveValue, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Cannot use t.Parallel() here because t.Setenv + // is incompatible with parallel subtests. + t.Setenv("WEBHOOKER_ENVIRONMENT", "dev") + + if tt.set { + t.Setenv("RECEIVER_RATE_LIMIT", tt.value) + } else { + require.NoError(t, os.Unsetenv( + "RECEIVER_RATE_LIMIT", + )) + } + + if tt.expectError { + expectStartupErrorFor( + t, "RECEIVER_RATE_LIMIT", tt.sentinel, + ) + } else { + testReceiverRateLimitSuccess(t, tt.expected) + } + }) + } +} + +func testReceiverRateLimitSuccess( + t *testing.T, + expected int, +) { + t.Helper() + + var cfg *config.Config + + app := fxtest.New( + t, + fx.Provide( + globals.New, + logger.New, + config.New, + ), + fx.Populate(&cfg), + ) + require.NoError(t, app.Err()) + + app.RequireStart() + + defer app.RequireStop() + + assert.Equal(t, expected, cfg.ReceiverRateLimit) +} + +func TestTrustedProxies(t *testing.T) { + tests := []struct { + name string + set bool + value string + expectError bool + expected []string + }{ + { + // The default must be "trust nobody": an empty list + // means forwarded headers are ignored, never that + // every peer may speak for the client. + name: caseUnsetUsesDefault, + set: false, + expected: []string{}, + }, + { + name: "blank value trusts nothing", + set: true, + value: " ", + expected: []string{}, + }, + { + name: caseValidValueParsed, + set: true, + value: cidrPrivateV4 + ", 192.168.1.7 ,2001:db8::/32", + expected: []string{ + cidrPrivateV4, "192.168.1.7/32", "2001:db8::/32", + }, + }, + { + name: "host bits are masked off", + set: true, + value: "10.1.2.3/8", + expected: []string{cidrPrivateV4}, + }, + { + // Peer addresses are unmapped before they are + // matched, so an IPv4-mapped prefix kept in that + // form could never match anything. + name: "IPv4-mapped prefix is unmapped", + set: true, + value: "::ffff:10.0.0.0/104", + expected: []string{cidrPrivateV4}, + }, + { + name: caseUnparseableFails, + set: true, + value: cidrPrivateV4 + ",not-an-address", + expectError: true, + }, + { + name: "out-of-range prefix length fails startup", + set: true, + value: "10.0.0.0/33", + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Cannot use t.Parallel() here because t.Setenv + // is incompatible with parallel subtests. + t.Setenv("WEBHOOKER_ENVIRONMENT", "dev") + + if tt.set { + t.Setenv("TRUSTED_PROXIES", tt.value) + } else { + require.NoError(t, os.Unsetenv("TRUSTED_PROXIES")) + } + + if tt.expectError { + expectStartupErrorFor( + t, "TRUSTED_PROXIES", config.ErrInvalidCIDR, + ) + } else { + testTrustedProxiesSuccess(t, tt.expected) + } + }) + } +} + +func testTrustedProxiesSuccess( + t *testing.T, + expected []string, +) { + t.Helper() + + var cfg *config.Config + + app := fxtest.New( + t, + fx.Provide( + globals.New, + logger.New, + config.New, + ), + fx.Populate(&cfg), + ) + require.NoError(t, app.Err()) + + app.RequireStart() + + defer app.RequireStop() + + got := make([]string, 0, len(cfg.TrustedProxies)) + for _, prefix := range cfg.TrustedProxies { + got = append(got, prefix.String()) + } + + assert.Equal(t, expected, got) +} + +// TestAllowedEgressCIDRs covers ALLOWED_EGRESS_CIDRS, the escape +// hatch that lets a self-hosted deployment forward to its own +// network. Unset it must stay empty, so the SSRF guard keeps +// refusing every private/reserved range; a set-but-unparseable +// value must abort startup naming the variable rather than +// silently running with a list the operator did not write. +func TestAllowedEgressCIDRs(t *testing.T) { + tests := []struct { + name string + set bool + value string + expected []string + expectError bool + }{ + { + name: caseUnsetUsesDefault, + set: false, + expected: []string{}, + }, + { + name: "empty value yields empty list", + set: true, + value: "", + expected: []string{}, + }, + { + name: caseValidValueParsed, + set: true, + value: cidrPrivateV4, + expected: []string{cidrPrivateV4}, + }, + { + name: "multiple blocks with whitespace", + set: true, + value: " 10.0.0.0/8 , 127.0.0.0/8 ", + expected: []string{cidrPrivateV4, "127.0.0.0/8"}, + }, + { + name: "bare address becomes a single host", + set: true, + value: "172.17.0.5", + expected: []string{"172.17.0.5/32"}, + }, + { + name: caseUnparseableFails, + set: true, + value: cidrPrivateV4 + ",not-an-address", + expectError: true, + }, + { + name: "out-of-range prefix length fails startup", + set: true, + value: "10.0.0.0/33", + expectError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Cannot use t.Parallel() here because t.Setenv + // is incompatible with parallel subtests. + t.Setenv("WEBHOOKER_ENVIRONMENT", "dev") + + if tt.set { + t.Setenv("ALLOWED_EGRESS_CIDRS", tt.value) + } else { + require.NoError( + t, os.Unsetenv("ALLOWED_EGRESS_CIDRS"), + ) + } + + if tt.expectError { + expectStartupErrorFor( + t, "ALLOWED_EGRESS_CIDRS", config.ErrInvalidCIDR, + ) + } else { + testAllowedEgressCIDRsSuccess(t, tt.expected) + } + }) + } +} + +func testAllowedEgressCIDRsSuccess( + t *testing.T, + expected []string, +) { + t.Helper() + + var cfg *config.Config + + app := fxtest.New( + t, + fx.Provide( + globals.New, + logger.New, + config.New, + ), + fx.Populate(&cfg), + ) + require.NoError(t, app.Err()) + + app.RequireStart() + + defer app.RequireStop() + + assert.Equal( + t, expected, config.PrefixStrings(cfg.AllowedEgressCIDRs), + ) +} + +// TestEgressAllowlistWarning covers the startup log that shows an +// operator the hole ALLOWED_EGRESS_CIDRS opened. It must stay +// silent on the default (empty) list and, when set, print the +// blocks themselves rather than a count. +func TestEgressAllowlistWarning(t *testing.T) { + tests := []struct { + name string + allowed string + expectWarning bool + }{ + { + name: "empty allowlist is quiet", + expectWarning: false, + }, + { + name: "non-empty allowlist warns", + allowed: "10.0.0.0/8,127.0.0.0/8", + expectWarning: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Cannot use t.Parallel() here because t.Setenv + // is incompatible with parallel subtests. + t.Setenv("WEBHOOKER_ENVIRONMENT", config.EnvironmentDev) + + if tt.allowed == "" { + require.NoError( + t, os.Unsetenv("ALLOWED_EGRESS_CIDRS"), + ) + } else { + t.Setenv("ALLOWED_EGRESS_CIDRS", tt.allowed) + } + + var buf bytes.Buffer + + log := slog.New(slog.NewJSONHandler( + &buf, &slog.HandlerOptions{ + Level: slog.LevelDebug, + }, + )) + + require.NoError( + t, config.WarnEgressAllowlistForTest(log), + ) + + if !tt.expectWarning { + assert.Empty(t, buf.String()) + + return + } + + logged := buf.String() + + assert.Contains(t, logged, `"level":"WARN"`) + assert.Contains(t, logged, "ALLOWED_EGRESS_CIDRS") + // The blocks themselves, not a count: the operator has + // to be able to read back which networks are open. + assert.Contains(t, logged, "10.0.0.0/8") + assert.Contains(t, logged, "127.0.0.0/8") + // What stays shut. Asserted on the clause naming the + // wider set rather than on "Link-local" alone, so the + // string cannot narrow back to link-local only while + // the always-blocked set covers ULA, CGNAT and two + // public metadata addresses as well. + assert.Contains(t, logged, "metadata endpoints outside it") + }) + } +} + +// TestSharedRateLimitBucketWarning covers the startup warning that +// tells an operator a deployment behind a reverse proxy shares one +// rate-limit bucket between every client, which turns the receiver +// limits into service-wide ceilings and collapses login failure +// counting. It must fire whenever TRUSTED_PROXIES is empty, +// in any environment: WEBHOOKER_ENVIRONMENT defaults to dev, so gating +// on it would silence the warning for exactly the operator who never +// configured the deployment. It stays quiet once proxies are named. +func TestSharedRateLimitBucketWarning(t *testing.T) { + tests := []struct { + name string + environment string + trustedProxies string + expectWarning bool + }{ + { + name: "prod without trusted proxies warns", + environment: config.EnvironmentProd, + expectWarning: true, + }, + { + name: "prod with trusted proxies is quiet", + environment: config.EnvironmentProd, + trustedProxies: cidrPrivateV4, + expectWarning: false, + }, + { + // The default environment. An internet-exposed + // deployment whose operator never set + // WEBHOOKER_ENVIRONMENT lands here and has exactly + // the exposure the warning announces. + name: "dev without trusted proxies warns", + environment: config.EnvironmentDev, + expectWarning: true, + }, + { + name: "dev with trusted proxies is quiet", + environment: config.EnvironmentDev, + trustedProxies: cidrPrivateV4, + expectWarning: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Cannot use t.Parallel() here because t.Setenv + // is incompatible with parallel subtests. + t.Setenv("WEBHOOKER_ENVIRONMENT", tt.environment) + + if tt.trustedProxies == "" { + require.NoError( + t, os.Unsetenv("TRUSTED_PROXIES"), + ) + } else { + t.Setenv("TRUSTED_PROXIES", tt.trustedProxies) + } + + var buf bytes.Buffer + + log := slog.New(slog.NewJSONHandler( + &buf, &slog.HandlerOptions{ + Level: slog.LevelDebug, + }, + )) + + require.NoError( + t, + config.WarnSharedRateLimitBucketForTest(log), + ) + + if !tt.expectWarning { + assert.Empty(t, buf.String()) + + return + } + + logged := buf.String() + + assert.Contains(t, logged, `"level":"WARN"`) + assert.Contains(t, logged, "TRUSTED_PROXIES") + assert.Contains(t, logged, "share one bucket") + assert.Contains( + t, logged, "throttle every other client's failed logins", + ) + // The warning must not claim a lockout the login + // endpoint no longer permits: credentials are verified + // before any budget is spent. + assert.Contains( + t, logged, "a correct password still gets in", + ) + // The text must stay accurate for a developer with + // nothing in front of the process, where an empty + // list costs nothing. + assert.Contains( + t, logged, "nothing proxying to this process", + ) + }) + } +} + +// metricsEnv describes what one subtest below puts in the +// environment for a single METRICS_ variable. A variable that is +// set to the empty string and one that is not set at all are +// distinct inputs here, because the reported bug arrived through +// the first of them. +type metricsEnv struct { + set bool + value string +} + +// unset leaves the variable out of the environment entirely. +func unset() metricsEnv { + return metricsEnv{set: false, value: ""} +} + +// setTo sets the variable, including to the empty string. +func setTo(value string) metricsEnv { + return metricsEnv{set: true, value: value} +} + +// metricsAuthCase is one row of the table in TestMetricsAuthConfig, +// named so the table can live in its own function and keep the test +// itself short. +type metricsAuthCase struct { + name string + username metricsEnv + password metricsEnv + expectError bool + expectAuth bool +} + +// metricsAuthCases enumerates every combination of the two +// credentials, counting "set to the empty string" and "not set at +// all" as separate inputs on each side. +func metricsAuthCases() []metricsAuthCase { + return []metricsAuthCase{ + { + name: "both unset leaves metrics unmounted", + username: unset(), + password: unset(), + }, + { + name: "both empty leaves metrics unmounted", + username: setTo(""), + password: setTo(""), + }, + { + name: "both set enables metrics auth", + username: setTo("metrics"), + password: setTo(metricsAuthValue), + expectAuth: true, + }, + { + name: "username with unset password fails", + username: setTo("metrics"), + password: unset(), + expectError: true, + }, + { + name: "username with empty password fails", + username: setTo("metrics"), + password: setTo(""), + expectError: true, + }, + { + name: "password with unset username fails", + username: unset(), + password: setTo(metricsAuthValue), + expectError: true, + }, + { + name: "password with empty username fails", + username: setTo(""), + password: setTo(metricsAuthValue), + expectError: true, + }, + } +} + +// TestMetricsAuthConfig covers every combination of METRICS_USERNAME +// and METRICS_PASSWORD. Either both carry a value, in which case +// /metrics is served behind basic auth, or neither does, in which +// case the route is never mounted. One without the other is a +// startup error rather than a fallback: mounting on the username +// alone published /metrics behind a credential map that accepted an +// empty password, which is the defect this test exists to pin. See +// https://git.eeqj.de/sneak/webhooker/issues/205. +func TestMetricsAuthConfig(t *testing.T) { + for _, tt := range metricsAuthCases() { + t.Run(tt.name, func(t *testing.T) { + // Cannot use t.Parallel() here because t.Setenv + // is incompatible with parallel subtests. + if tt.username.set { + t.Setenv("METRICS_USERNAME", tt.username.value) + } else { + require.NoError( + t, os.Unsetenv("METRICS_USERNAME"), + ) + } + + if tt.password.set { + t.Setenv("METRICS_PASSWORD", tt.password.value) + } else { + require.NoError( + t, os.Unsetenv("METRICS_PASSWORD"), + ) + } + + if tt.expectError { + assertMetricsAuthRejected(t) + + return + } + + assertMetricsAuthAccepted(t, tt.expectAuth) + }) + } +} + +// assertMetricsAuthRejected requires that fx refused to build the +// graph, that the failure is ErrIncompleteMetricsAuth, and that the +// operator is told both variable names — the point of failing here +// rather than degrading is that the message says what to fix. +func assertMetricsAuthRejected(t *testing.T) { + t.Helper() + + var cfg *config.Config + + app := fx.New( + fx.NopLogger, + fx.Provide(globals.New, logger.New, config.New), + fx.Populate(&cfg), + ) + + err := app.Err() + require.Error(t, err) + require.ErrorIs(t, err, config.ErrIncompleteMetricsAuth) + assert.Contains(t, err.Error(), "METRICS_USERNAME") + assert.Contains(t, err.Error(), "METRICS_PASSWORD") + // The password is a secret and must not reach a startup error. + assert.NotContains(t, err.Error(), metricsAuthValue) +} + +// assertMetricsAuthAccepted requires that startup succeeded and that +// MetricsAuthEnabled — the single value the /metrics mount and the +// startup log both read — reports what the environment asked for. +func assertMetricsAuthAccepted(t *testing.T, expectAuth bool) { + t.Helper() + + var cfg *config.Config + + app := fxtest.New( + t, + fx.Provide(globals.New, logger.New, config.New), + fx.Populate(&cfg), + ) + require.NoError(t, app.Err()) + + app.RequireStart() + + defer app.RequireStop() + + assert.Equal(t, expectAuth, cfg.MetricsAuthEnabled()) +} diff --git a/internal/config/dotenv_test.go b/internal/config/dotenv_test.go new file mode 100644 index 0000000..952bb32 --- /dev/null +++ b/internal/config/dotenv_test.go @@ -0,0 +1,158 @@ +package config_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/config" +) + +// dotEnvKey is a throwaway variable name the .env tests write and +// read, so they cannot disturb real configuration. +const dotEnvKey = "WEBHOOKER_TEST_DOTENV_VALUE" + +// malformedDotEnv is a file godotenv cannot parse. The first line is +// the realistic typo — a space where the `=` belongs — and the rest +// make sure nothing downstream treats the file as salvageable line by +// line. +const malformedDotEnv = "PORT 19615\n" + + "this is not = valid ! syntax\n" + + "\"unclosed\n" + +// unsetDotEnvKey makes dotEnvKey genuinely absent for the duration of +// the test and restores it afterwards. t.Setenv registers the restore; +// the Unsetenv that follows is what the test actually needs, because a +// variable set to the empty string is still present in os.Environ and +// godotenv would refuse to overwrite it. +func unsetDotEnvKey(t *testing.T) { + t.Helper() + t.Setenv(dotEnvKey, "placeholder") + require.NoError(t, os.Unsetenv(dotEnvKey)) +} + +// writeDotEnv writes contents to a .env file in a fresh temporary +// directory and returns its path. +func writeDotEnv(t *testing.T, contents string) string { + t.Helper() + + path := filepath.Join(t.TempDir(), config.DotEnvPath) + require.NoError(t, os.WriteFile(path, []byte(contents), 0o600)) + + return path +} + +// TestLoadDotEnv_MissingFileIsFine pins the case most deployments are +// in. The file is optional: it is a development convenience, and a +// deployment that configures the environment directly must start +// normally rather than be refused for a file it was never meant to +// have. +// +//nolint:paralleltest // unsetDotEnvKey uses t.Setenv. +func TestLoadDotEnv_MissingFileIsFine(t *testing.T) { + unsetDotEnvKey(t) + + absent := filepath.Join(t.TempDir(), config.DotEnvPath) + require.NoError(t, config.LoadDotEnvFileForTest(absent)) + + _, present := os.LookupEnv(dotEnvKey) + assert.False(t, present, "nothing may be set from an absent file") +} + +// TestLoadDotEnv_AppliesValues pins that a well-formed file still +// reaches the environment, which is the whole reason the file is read +// at all. +// +//nolint:paralleltest // unsetDotEnvKey uses t.Setenv. +func TestLoadDotEnv_AppliesValues(t *testing.T) { + unsetDotEnvKey(t) + + path := writeDotEnv(t, "# a comment\n"+dotEnvKey+"=from-dot-env\n") + + require.NoError(t, config.LoadDotEnvFileForTest(path)) + assert.Equal(t, "from-dot-env", os.Getenv(dotEnvKey)) +} + +// TestLoadDotEnv_RealEnvironmentWins pins that the file cannot +// override a variable the process was actually started with. A +// deployment that sets DATA_DIR in its unit file must not have it +// silently replaced by a stale .env left in the working directory. +func TestLoadDotEnv_RealEnvironmentWins(t *testing.T) { + t.Setenv(dotEnvKey, "from-environment") + + path := writeDotEnv(t, dotEnvKey+"=from-dot-env\n") + + require.NoError(t, config.LoadDotEnvFileForTest(path)) + assert.Equal(t, "from-environment", os.Getenv(dotEnvKey)) +} + +// TestLoadDotEnv_MalformedFileAborts is the defect this fixes. One bad +// line makes godotenv apply none of the file, so every variable in it +// reverts to its default; the process used to start that way with no +// log line naming the file at all. +// +//nolint:paralleltest // unsetDotEnvKey uses t.Setenv. +func TestLoadDotEnv_MalformedFileAborts(t *testing.T) { + unsetDotEnvKey(t) + + path := writeDotEnv( + t, malformedDotEnv+dotEnvKey+"=from-dot-env\n", + ) + + err := config.LoadDotEnvFileForTest(path) + + require.Error(t, err) + require.ErrorIs(t, err, config.ErrDotEnvUnreadable) + assert.Contains( + t, err.Error(), config.DotEnvPath, + "the failure must name the file it could not read", + ) + + _, present := os.LookupEnv(dotEnvKey) + assert.False( + t, present, + "a rejected file must apply nothing, not part of itself", + ) +} + +// TestLoadDotEnv_UnreadableFileAborts pins that only absence is +// tolerated. A .env that exists but cannot be read is a file the +// operator meant to be applied, so it fails like a malformed one +// rather than being treated as though it were not there. +func TestLoadDotEnv_UnreadableFileAborts(t *testing.T) { + t.Parallel() + + // A directory in the file's place: open succeeds and the read + // fails, which no umask or root-ness can turn back into success + // the way a chmod could. + path := filepath.Join(t.TempDir(), config.DotEnvPath) + require.NoError(t, os.Mkdir(path, 0o750)) + + err := config.LoadDotEnvFileForTest(path) + + require.Error(t, err) + require.ErrorIs(t, err, config.ErrDotEnvUnreadable) +} + +// TestLoadDotEnv_ReadsTheWorkingDirectory pins the path LoadDotEnv +// itself opens, which the tests above bypass. It is relative to the +// process working directory, as it was under godotenv/autoload and as +// the README documents. +// +//nolint:paralleltest // t.Chdir moves the whole process. +func TestLoadDotEnv_ReadsTheWorkingDirectory(t *testing.T) { + unsetDotEnvKey(t) + + dir := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(dir, config.DotEnvPath), + []byte(dotEnvKey+"=from-working-directory\n"), + 0o600, + )) + t.Chdir(dir) + + require.NoError(t, config.LoadDotEnv()) + assert.Equal(t, "from-working-directory", os.Getenv(dotEnvKey)) +} diff --git a/internal/config/env_test.go b/internal/config/env_test.go new file mode 100644 index 0000000..29985c6 --- /dev/null +++ b/internal/config/env_test.go @@ -0,0 +1,688 @@ +package config_test + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/fx" + "sneak.berlin/go/webhooker/internal/config" + "sneak.berlin/go/webhooker/internal/globals" + "sneak.berlin/go/webhooker/internal/logger" +) + +// testEnvKey is a throwaway variable name used only by the helper +// tables below, so they cannot disturb real configuration. +const testEnvKey = "WEBHOOKER_TEST_VALUE" + +// Real configuration variables exercised by the config.New tests. +const ( + envKeyPort = "PORT" + envKeyDebug = "DEBUG" + envKeyMaintenanceMode = "MAINTENANCE_MODE" + envKeyBindAddress = "BIND_ADDRESS" +) + +// Sample BIND_ADDRESS values used by the tables below. +const ( + // bindAddressDefault is the shipped default. It is asserted + // against the package's own constant in + // TestNewUsesDefaultsWhenUnset, so the two cannot drift. + bindAddressDefault = "127.0.0.1" + + // bindAddressWildcard is the value a container deployment sets. + bindAddressWildcard = "0.0.0.0" + + // bindAddressSample is an arbitrary specific address, standing + // for "one interface of several". + bindAddressSample = "10.1.2.3" +) + +// envBoolCase is one row of the envBool table. +type envBoolCase struct { + name string + set bool + value string + defaultValue bool + expectError bool + expected bool +} + +// envBoolCases is the envBool table, kept out of the test body so +// the test itself stays readable. +func envBoolCases() []envBoolCase { + return []envBoolCase{ + { + name: "unset uses default false", + defaultValue: false, + expected: false, + }, + { + name: "unset uses default true", + defaultValue: true, + expected: true, + }, + { + name: "empty uses default true", + set: true, + value: "", + defaultValue: true, + expected: true, + }, + { + name: "true is parsed", + set: true, + value: "true", + expected: true, + }, + { + name: "one is parsed", + set: true, + value: "1", + expected: true, + }, + { + name: "False is parsed", + set: true, + value: "False", + defaultValue: true, + expected: false, + }, + { + name: "zero is parsed", + set: true, + value: "0", + defaultValue: true, + expected: false, + }, + { + name: "yes is rejected", + set: true, + value: "yes", + expectError: true, + }, + { + name: "on is rejected", + set: true, + value: "on", + expectError: true, + }, + { + name: "typo is rejected", + set: true, + value: "ture", + expectError: true, + }, + } +} + +func TestEnvBool(t *testing.T) { + for _, tt := range envBoolCases() { + t.Run(tt.name, func(t *testing.T) { + // Cannot use t.Parallel() here because t.Setenv + // is incompatible with parallel subtests. + if tt.set { + t.Setenv(testEnvKey, tt.value) + } else { + require.NoError(t, os.Unsetenv(testEnvKey)) + } + + got, err := config.EnvBoolForTest( + testEnvKey, tt.defaultValue, + ) + + if tt.expectError { + require.Error(t, err) + assert.Contains(t, err.Error(), testEnvKey) + assert.Contains(t, err.Error(), tt.value) + + return + } + + require.NoError(t, err) + assert.Equal(t, tt.expected, got) + }) + } +} + +func TestEnvPositiveInt(t *testing.T) { + const defaultValue = 7 + + tests := []struct { + name string + set bool + value string + expectError bool + errIs error + expected int + }{ + { + name: "unset returns the default integer", + expected: defaultValue, + }, + { + name: "empty returns the default integer", + set: true, + value: "", + expected: defaultValue, + }, + { + name: "positive value is parsed", + set: true, + value: "42", + expected: 42, + }, + { + name: "unparseable value is rejected", + set: true, + value: "not-a-number", + expectError: true, + }, + { + name: "zero is rejected", + set: true, + value: "0", + expectError: true, + errIs: config.ErrNonPositiveValue, + }, + { + name: "negative is rejected", + set: true, + value: "-5", + expectError: true, + errIs: config.ErrNonPositiveValue, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Cannot use t.Parallel() here because t.Setenv + // is incompatible with parallel subtests. + if tt.set { + t.Setenv(testEnvKey, tt.value) + } else { + require.NoError(t, os.Unsetenv(testEnvKey)) + } + + got, err := config.EnvPositiveIntForTest( + testEnvKey, defaultValue, + ) + + if tt.expectError { + require.Error(t, err) + assert.Contains(t, err.Error(), testEnvKey) + assert.Contains(t, err.Error(), tt.value) + + if tt.errIs != nil { + require.ErrorIs(t, err, tt.errIs) + } + + return + } + + require.NoError(t, err) + assert.Equal(t, tt.expected, got) + }) + } +} + +func TestEnvPort(t *testing.T) { + const defaultValue = 8080 + + tests := []struct { + name string + set bool + value string + expectError bool + errIs error + expected int + }{ + { + name: "unset returns the default port", + expected: defaultValue, + }, + { + name: "valid port is parsed", + set: true, + value: "9000", + expected: 9000, + }, + { + name: "highest port is accepted", + set: true, + value: "65535", + expected: 65535, + }, + { + name: "unparseable value is rejected", + set: true, + value: "not-a-port", + expectError: true, + }, + { + name: "zero is rejected", + set: true, + value: "0", + expectError: true, + errIs: config.ErrNonPositiveValue, + }, + { + name: "above the port range is rejected", + set: true, + value: "65536", + expectError: true, + errIs: config.ErrInvalidPort, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Cannot use t.Parallel() here because t.Setenv + // is incompatible with parallel subtests. + if tt.set { + t.Setenv(testEnvKey, tt.value) + } else { + require.NoError(t, os.Unsetenv(testEnvKey)) + } + + got, err := config.EnvPortForTest( + testEnvKey, defaultValue, + ) + + if tt.expectError { + require.Error(t, err) + assert.Contains(t, err.Error(), testEnvKey) + + if tt.errIs != nil { + require.ErrorIs(t, err, tt.errIs) + } + + return + } + + require.NoError(t, err) + assert.Equal(t, tt.expected, got) + }) + } +} + +// TestEnvBindAddress covers BIND_ADDRESS parsing. +// +// Only IP address literals are accepted. Every rejection below is a +// value an operator plausibly writes — a hostname, a host:port, a +// CIDR block — and each has to abort startup rather than fall back to +// the default, because falling back would bind an address other than +// the one asked for and, in the wildcard-default case this setting +// exists to end, publish cleartext on every interface. +func TestEnvBindAddress(t *testing.T) { + for _, tt := range envBindAddressCases() { + t.Run(tt.name, func(t *testing.T) { + // Cannot use t.Parallel() here because t.Setenv + // is incompatible with parallel subtests. + if tt.set { + t.Setenv(testEnvKey, tt.value) + } else { + require.NoError(t, os.Unsetenv(testEnvKey)) + } + + got, err := config.EnvBindAddressForTest( + testEnvKey, bindAddressDefault, + ) + + if tt.expectError { + require.Error(t, err) + require.ErrorIs(t, err, config.ErrInvalidBindAddress) + assert.Contains(t, err.Error(), testEnvKey) + assert.Contains(t, err.Error(), tt.value) + + return + } + + require.NoError(t, err) + assert.Equal(t, tt.expected, got) + }) + } +} + +// envBindAddressCase is one row of the envBindAddress table. +type envBindAddressCase struct { + name string + set bool + value string + expectError bool + expected string +} + +// envBindAddressCases is the envBindAddress table, kept out of the +// test body so the test itself stays readable. +func envBindAddressCases() []envBindAddressCase { + return append( + envBindAddressAcceptedCases(), + envBindAddressRejectedCases()..., + ) +} + +// envBindAddressAcceptedCases are the values that parse: the three +// spellings of "unset" that take the default, and the literals. +func envBindAddressAcceptedCases() []envBindAddressCase { + return []envBindAddressCase{ + { + name: "unset returns the default", + expected: bindAddressDefault, + }, + { + name: "empty returns the default", + set: true, + value: "", + expected: bindAddressDefault, + }, + { + name: "whitespace returns the default", + set: true, + value: " ", + expected: bindAddressDefault, + }, + { + name: "ipv4 wildcard is parsed", + set: true, + value: bindAddressWildcard, + expected: bindAddressWildcard, + }, + { + name: "ipv4 literal is parsed", + set: true, + value: bindAddressSample, + expected: bindAddressSample, + }, + { + name: "surrounding whitespace is trimmed", + set: true, + value: " " + bindAddressSample + " ", + expected: bindAddressSample, + }, + { + name: "ipv6 wildcard is parsed", + set: true, + value: "::", + expected: "::", + }, + { + name: "ipv6 literal is parsed", + set: true, + value: "2001:db8::5", + expected: "2001:db8::5", + }, + } +} + +// envBindAddressRejectedCases are the values that abort startup. +// Each is something an operator plausibly writes, and none may fall +// back to the default: the default is loopback, so a silent fallback +// would bind somewhere other than what was asked for. +func envBindAddressRejectedCases() []envBindAddressCase { + return []envBindAddressCase{ + { + name: "garbage is rejected", + set: true, + value: "not-an-address", + expectError: true, + }, + { + name: "hostname is rejected", + set: true, + value: "localhost", + expectError: true, + }, + { + name: "unresolvable hostname is rejected", + set: true, + value: "no-such-host.invalid", + expectError: true, + }, + { + name: "host and port is rejected", + set: true, + value: bindAddressDefault + ":8080", + expectError: true, + }, + { + name: "bracketed ipv6 is rejected", + set: true, + value: "[::1]", + expectError: true, + }, + { + name: "CIDR block is rejected", + set: true, + value: "10.0.0.0/8", + expectError: true, + }, + } +} + +// buildConfig constructs a Config through fx exactly as the +// application does, returning the config and any construction error. +func buildConfig(t *testing.T) (*config.Config, error) { + t.Helper() + + var cfg *config.Config + + app := fx.New( + fx.NopLogger, + fx.Provide( + globals.New, + logger.New, + config.New, + ), + fx.Populate(&cfg), + ) + + return cfg, app.Err() +} + +func TestNewRejectsBadEnvValues(t *testing.T) { + for _, tt := range badEnvValueCases() { + t.Run(tt.name, func(t *testing.T) { + // Cannot use t.Parallel() here because t.Setenv + // is incompatible with parallel subtests. + t.Setenv("WEBHOOKER_ENVIRONMENT", "dev") + t.Setenv(tt.key, tt.value) + + cfg, err := buildConfig(t) + + if tt.expectError { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.key) + assert.Contains(t, err.Error(), tt.value) + + return + } + + require.NoError(t, err) + require.NotNil(t, cfg) + tt.check(t, cfg) + }) + } +} + +// badEnvValueCase is one row of the config.New table: a variable, the +// value it is set to, and either the assertion that startup fails +// naming both, or a check on the Config that resulted. +type badEnvValueCase struct { + name string + key string + value string + expectError bool + check func(t *testing.T, cfg *config.Config) +} + +// badEnvValueCases is the config.New table, kept out of the test body +// so the test itself stays readable. It is assembled from per-variable +// groups because one literal covering every variable outgrew the +// function-length budget. +func badEnvValueCases() []badEnvValueCase { + cases := listenerEnvValueCases() + cases = append(cases, flagEnvValueCases()...) + cases = append(cases, sentryEnvValueCases()...) + + return cases +} + +// listenerEnvValueCases covers the two variables that describe the +// HTTP listener. +func listenerEnvValueCases() []badEnvValueCase { + return []badEnvValueCase{ + { + name: "valid PORT is used", + key: envKeyPort, + value: "9001", + check: func(t *testing.T, cfg *config.Config) { + t.Helper() + assert.Equal(t, 9001, cfg.Port) + }, + }, + { + name: "unparseable PORT aborts startup", + key: envKeyPort, + value: "eighty-eighty", + expectError: true, + }, + { + name: "out-of-range PORT aborts startup", + key: envKeyPort, + value: "70000", + expectError: true, + }, + { + name: "valid BIND_ADDRESS is used", + key: envKeyBindAddress, + value: bindAddressWildcard, + check: func(t *testing.T, cfg *config.Config) { + t.Helper() + assert.Equal( + t, bindAddressWildcard, cfg.BindAddress, + ) + }, + }, + { + name: "unparseable BIND_ADDRESS aborts startup", + key: envKeyBindAddress, + value: "not-an-address", + expectError: true, + }, + { + name: "hostname BIND_ADDRESS aborts startup", + key: envKeyBindAddress, + value: "localhost", + expectError: true, + }, + { + name: "BIND_ADDRESS with a port aborts startup", + key: envKeyBindAddress, + value: bindAddressDefault + ":8080", + expectError: true, + }, + } +} + +// flagEnvValueCases covers the boolean variables. +func flagEnvValueCases() []badEnvValueCase { + return []badEnvValueCase{ + { + name: "valid DEBUG is used", + key: envKeyDebug, + value: "true", + check: func(t *testing.T, cfg *config.Config) { + t.Helper() + assert.True(t, cfg.Debug) + }, + }, + { + name: "unparseable DEBUG aborts startup", + key: envKeyDebug, + value: "ture", + expectError: true, + }, + { + name: "unparseable MAINTENANCE_MODE aborts startup", + key: envKeyMaintenanceMode, + value: "sometimes", + expectError: true, + }, + } +} + +// sentryEnvValueCases covers SENTRY_DSN. The three rejected values are +// the ones measured on the defect: each initialised the SDK with an +// error and left the process serving with error reporting off. +func sentryEnvValueCases() []badEnvValueCase { + return []badEnvValueCase{ + { + name: "valid SENTRY_DSN is used", + key: envKeySentryDSN, + value: validSentryDSN, + check: func(t *testing.T, cfg *config.Config) { + t.Helper() + assert.Equal(t, validSentryDSN, cfg.SentryDSN) + assert.True(t, cfg.SentryEnabled()) + }, + }, + { + name: "unparseable SENTRY_DSN aborts startup", + key: envKeySentryDSN, + value: "not-a-dsn", + expectError: true, + }, + { + name: "SENTRY_DSN that is not a URL aborts startup", + key: envKeySentryDSN, + value: "%%%", + expectError: true, + }, + { + name: "keyless SENTRY_DSN aborts startup", + key: envKeySentryDSN, + value: "https://example.invalid/1", + expectError: true, + }, + } +} + +// TestNewUsesDefaultsWhenUnset proves the fail-loud behaviour did not +// break the legitimate unset case: absent variables still get their +// documented defaults. +func TestNewUsesDefaultsWhenUnset(t *testing.T) { + t.Setenv("WEBHOOKER_ENVIRONMENT", "dev") + + for _, key := range []string{ + envKeyPort, envKeyDebug, envKeyMaintenanceMode, + envKeyBindAddress, envKeySentryDSN, + } { + require.NoError(t, os.Unsetenv(key)) + } + + cfg, err := buildConfig(t) + require.NoError(t, err) + require.NotNil(t, cfg) + + assert.Equal(t, 8080, cfg.Port) + assert.False(t, cfg.Debug) + assert.False(t, cfg.MaintenanceMode) + + // Loopback, not the wildcard: the default must not publish the + // cleartext admin UI and the unauthenticated receiver on every + // interface of a host that configured nothing. The value is read + // from the package rather than repeated, so the README's + // documented default and the compiled-in one are pinned to the + // same constant. + assert.Equal( + t, config.DefaultBindAddressForTest, cfg.BindAddress, + ) + assert.Equal(t, bindAddressDefault, cfg.BindAddress) + + // An absent SENTRY_DSN is the common case and must stay a normal + // start with error reporting off, not a refusal. + assert.Empty(t, cfg.SentryDSN) + assert.False(t, cfg.SentryEnabled()) +} diff --git a/internal/config/export_test.go b/internal/config/export_test.go new file mode 100644 index 0000000..d4d3be8 --- /dev/null +++ b/internal/config/export_test.go @@ -0,0 +1,74 @@ +package config + +import "log/slog" + +// This file exposes the unexported environment parsing helpers to +// the external config_test package so each helper can be covered by +// its own table-driven test without weakening the package API. + +// WarnSharedRateLimitBucketForTest loads a Config from the current +// environment and emits its startup warnings to log. The real logger +// writes to stdout, so this lets the warning's firing condition be +// asserted against a handler the test controls. +func WarnSharedRateLimitBucketForTest(log *slog.Logger) error { + c, err := loadFromEnv() + if err != nil { + return err + } + + c.warnSharedRateLimitBucket(log) + + return nil +} + +// WarnEgressAllowlistForTest loads a Config from the current +// environment and emits its egress-allowlist startup warning to +// log, so a test can assert both that the warning fires only when +// the list is non-empty and that it names the blocks it opened. +func WarnEgressAllowlistForTest(log *slog.Logger) error { + c, err := loadFromEnv() + if err != nil { + return err + } + + c.warnEgressAllowlist(log) + + return nil +} + +// EnvBoolForTest exposes envBool. +func EnvBoolForTest(key string, defaultValue bool) (bool, error) { + return envBool(key, defaultValue) +} + +// EnvPositiveIntForTest exposes envPositiveInt. +func EnvPositiveIntForTest(key string, defaultValue int) (int, error) { + return envPositiveInt(key, defaultValue) +} + +// EnvPortForTest exposes envPort. +func EnvPortForTest(key string, defaultValue int) (int, error) { + return envPort(key, defaultValue) +} + +// EnvSentryDSNForTest exposes envSentryDSN. +func EnvSentryDSNForTest(key string) (string, error) { + return envSentryDSN(key) +} + +// LoadDotEnvFileForTest exposes the loader LoadDotEnv runs, over a +// caller-named file rather than the process working directory, so +// each .env state can be covered without moving the test process. +func LoadDotEnvFileForTest(path string) error { + return loadDotEnvFile(path) +} + +// EnvBindAddressForTest exposes envBindAddress. +func EnvBindAddressForTest(key, defaultValue string) (string, error) { + return envBindAddress(key, defaultValue) +} + +// DefaultBindAddressForTest exposes the compiled-in BIND_ADDRESS +// default, so a test pins the documented value rather than repeating +// a literal that could drift from it. +const DefaultBindAddressForTest = defaultBindAddress diff --git a/internal/config/sentry_test.go b/internal/config/sentry_test.go new file mode 100644 index 0000000..0b475bf --- /dev/null +++ b/internal/config/sentry_test.go @@ -0,0 +1,141 @@ +package config_test + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/config" +) + +// envKeySentryDSN is the variable envSentryDSN reads in production. +const envKeySentryDSN = "SENTRY_DSN" + +// validSentryDSN is a syntactically complete DSN. The host is under +// .invalid (RFC 2606), so nothing a test builds around it can reach a +// real Sentry installation. +const validSentryDSN = "https://abc123@sentry.invalid/42" + +// envSentryDSNCase is one row of the envSentryDSN table. +type envSentryDSNCase struct { + name string + set bool + value string + expectError bool + expected string +} + +// envSentryDSNCases is the envSentryDSN table. The three invalid +// values are the ones measured on the defect: each initialised the SDK +// with an error and left the process serving with reporting off. +func envSentryDSNCases() []envSentryDSNCase { + return []envSentryDSNCase{ + { + name: "unset means reporting off", + expected: "", + }, + { + name: "empty means reporting off", + set: true, + value: "", + expected: "", + }, + { + name: "whitespace means reporting off", + set: true, + value: " ", + expected: "", + }, + { + name: "a valid DSN is kept", + set: true, + value: validSentryDSN, + expected: validSentryDSN, + }, + { + name: "surrounding whitespace is trimmed", + set: true, + value: " " + validSentryDSN + "\t", + expected: validSentryDSN, + }, + { + name: "a value that is not a URL is rejected", + set: true, + value: "not-a-dsn", + expectError: true, + }, + { + name: "an unparseable URL is rejected", + set: true, + value: "%%%", + expectError: true, + }, + { + name: "a DSN without a public key is rejected", + set: true, + value: "https://example.invalid/1", + expectError: true, + }, + { + name: "a DSN without a project id is rejected", + set: true, + value: "https://abc123@sentry.invalid/", + expectError: true, + }, + { + name: "a non-HTTP scheme is rejected", + set: true, + value: "ftp://abc123@sentry.invalid/42", + expectError: true, + }, + } +} + +// TestEnvSentryDSN covers the helper directly. What it pins beyond the +// value is the failure shape: a set-but-unparseable DSN names the +// variable and the value, exactly as the other fail-loud helpers do, +// so an operator reads the fix off the message. +func TestEnvSentryDSN(t *testing.T) { + for _, tt := range envSentryDSNCases() { + t.Run(tt.name, func(t *testing.T) { + // Cannot use t.Parallel() here because t.Setenv + // is incompatible with parallel subtests. + if tt.set { + t.Setenv(envKeySentryDSN, tt.value) + } else { + require.NoError(t, os.Unsetenv(envKeySentryDSN)) + } + + got, err := config.EnvSentryDSNForTest(envKeySentryDSN) + + if tt.expectError { + require.Error(t, err) + require.ErrorIs(t, err, config.ErrInvalidSentryDSN) + assert.Contains(t, err.Error(), envKeySentryDSN) + assert.Contains(t, err.Error(), tt.value) + assert.Empty(t, got) + + return + } + + require.NoError(t, err) + assert.Equal(t, tt.expected, got) + }) + } +} + +// TestSentryEnabled_TracksTheDSN pins that the one method answering +// "is anything being reported" agrees with the DSN in every state. The +// startup log, the SDK initialisation and the sentryhttp middleware +// all read it, so a log field cannot report reporting as on while +// nothing is sending. +func TestSentryEnabled_TracksTheDSN(t *testing.T) { + t.Parallel() + + assert.False(t, (&config.Config{}).SentryEnabled()) + assert.True( + t, + (&config.Config{SentryDSN: validSentryDSN}).SentryEnabled(), + ) +} diff --git a/internal/database/bootstrap_banner_test.go b/internal/database/bootstrap_banner_test.go new file mode 100644 index 0000000..f56b467 --- /dev/null +++ b/internal/database/bootstrap_banner_test.go @@ -0,0 +1,85 @@ +package database_test + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/database" +) + +// passwordField is the banner line carrying the plaintext. +const passwordField = "password: " + +// bannerPassword returns the password the banner printed. +func bannerPassword(t *testing.T, out string) string { + t.Helper() + + for line := range strings.SplitSeq(out, "\n") { + _, value, found := strings.Cut(line, passwordField) + if found { + return strings.TrimSpace(value) + } + } + + t.Fatalf("no %q line in the banner:\n%s", passwordField, out) + + return "" +} + +// TestFirstBoot_PrintsTheAdminPasswordAsABanner is the bootstrap half +// of https://git.eeqj.de/sneak/webhooker/issues/208. +// +// The password is shown exactly once, and it used to be shown as one +// slog record among the roughly 45 fx PROVIDE/RUN/HOOK lines a boot +// writes — which is how deployments lost it and, with no reset path, +// locked themselves out. It must be emitted as a block an operator can +// find by eye, it must carry the plaintext that actually opens the +// account, and it must name the command that recovers it. +func TestFirstBoot_PrintsTheAdminPasswordAsABanner(t *testing.T) { + t.Parallel() + + db, lc := setupTestDB(t) + + var out bytes.Buffer + + db.ExportSetBannerOut(&out) + + ctx := context.Background() + require.NoError(t, lc.Start(ctx)) + + defer func() { require.NoError(t, lc.Stop(ctx)) }() + + printed := out.String() + + require.Contains( + t, printed, strings.Repeat("=", 20), + "the banner must be ruled off, not read as one more log line", + ) + require.Contains(t, printed, "username: admin") + assert.Contains( + t, printed, "resetpw", + "the banner must name the command that recovers the account", + ) + + password := bannerPassword(t, printed) + require.NotEmpty(t, password) + + // The printed plaintext must be the one that opens the account: + // a banner showing a different string would be worse than none. + var user database.User + + require.NoError( + t, + db.DB().Where("username = ?", "admin").First(&user).Error, + ) + + ok, err := database.VerifyPassword(password, user.Password) + require.NoError(t, err) + assert.True( + t, ok, "the printed password must open the seeded account", + ) +} diff --git a/internal/database/database.go b/internal/database/database.go index bbee40c..f2880e8 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -4,10 +4,10 @@ package database import ( "context" "crypto/rand" - "database/sql" "encoding/base64" "errors" "fmt" + "io" "log/slog" "os" "path/filepath" @@ -15,8 +15,9 @@ import ( "go.uber.org/fx" "gorm.io/driver/sqlite" "gorm.io/gorm" - _ "modernc.org/sqlite" // Pure Go SQLite driver + "sneak.berlin/go/webhooker/internal/banner" "sneak.berlin/go/webhooker/internal/config" + "sneak.berlin/go/webhooker/internal/gormlog" "sneak.berlin/go/webhooker/internal/logger" ) @@ -26,6 +27,20 @@ const ( sessionKeyLen = 32 ) +// MainDBFileName is the main application database inside DATA_DIR. It +// is exported so that an entry point acting on a data directory +// outside the fx graph can test for a deployment's existence without +// spelling the name a second time. +const MainDBFileName = "webhooker.db" + +// BootstrapPasswordNote is what the first-boot banner tells the +// operator to do about the password it just printed. It names the +// recovery command, because the moment that line scrolls away is +// exactly when the operator needs to know one exists. +const BootstrapPasswordNote = "Save this password now: it is shown " + + "only here, and only once.\nIf it is lost, run `webhooker " + + "resetpw admin` on a stopped deployment." + //nolint:revive // DatabaseParams is a standard fx naming convention. type DatabaseParams struct { fx.In @@ -39,6 +54,39 @@ type Database struct { db *gorm.DB log *slog.Logger params *DatabaseParams + + // bannerOut receives the first-boot credentials banner. Nil means + // os.Stdout, resolved at write time rather than at construction so + // that a caller which redirects the variable still captures it. + bannerOut io.Writer +} + +// Open connects to the main database in dataDir and migrates it, +// without the fx lifecycle and without seeding an admin account. +// +// It is for entry points that act on an existing deployment's data +// directory from outside the server graph — `webhooker resetpw`. Such a +// caller must already hold the DATA_DIR lock (see internal/datadir), +// and must Close the result. +// +// It does not create the admin account: seeding belongs to a server +// start, and a maintenance command that silently invented an account +// would answer "no such user" by creating one. +func Open(dataDir string, log *slog.Logger) (*Database, error) { + d := &Database{log: log} + + err := d.connectTo(dataDir) + if err != nil { + return nil, err + } + + return d, nil +} + +// Close closes the underlying connection. It is the exported form of +// the fx stop hook, for callers that built the Database with Open. +func (d *Database) Close() error { + return d.close() } // New creates a Database that connects on fx start and disconnects on stop. @@ -121,10 +169,22 @@ func (d *Database) GetOrCreateSessionKey() (string, error) { return encoded, nil } +// connect opens the configured data directory and, this being a +// server start, seeds the admin account when the deployment has none. func (d *Database) connect() error { - // Ensure the data directory exists before opening the database. - dataDir := d.params.Config.DataDir + err := d.connectTo(d.params.Config.DataDir) + if err != nil { + return err + } + return d.ensureAdminUser() +} + +// connectTo opens and migrates the main database in dataDir. It seeds +// nothing: whether an empty deployment gets an admin account is the +// caller's decision. +func (d *Database) connectTo(dataDir string) error { + // Ensure the data directory exists before opening the database. err := os.MkdirAll(dataDir, dataDirPerm) if err != nil { return fmt.Errorf( @@ -135,14 +195,12 @@ func (d *Database) connect() error { } // Construct the main application database path inside DATA_DIR. - dbPath := filepath.Join(dataDir, "webhooker.db") - dbURL := fmt.Sprintf( - "file:%s?cache=shared&mode=rwc", - dbPath, - ) + dbPath := filepath.Join(dataDir, MainDBFileName) - // Open the database with the pure Go SQLite driver - sqlDB, err := sql.Open("sqlite", dbURL) + // Opened through OpenSQLite so this handle carries the same WAL + // journaling, busy timeout, immediate-transaction locking, and pool + // bounds as every other database file. See sqlite_open.go. + sqlDB, err := OpenSQLite(dbPath, SQLiteModeCreate) if err != nil { d.log.Error( "failed to open database", @@ -155,7 +213,10 @@ func (d *Database) connect() error { // Then use it with GORM db, err := gorm.Open(sqlite.Dialector{ Conn: sqlDB, - }, &gorm.Config{}) + }, &gorm.Config{ + // Never leave this at GORM's default. See internal/gormlog. + Logger: gormlog.New(d.log), + }) if err != nil { d.log.Error( "failed to connect to database", @@ -186,10 +247,16 @@ func (d *Database) migrate() error { d.log.Info("database migrations completed") + return nil +} + +// ensureAdminUser creates the bootstrap admin account when the +// deployment has no users at all. +func (d *Database) ensureAdminUser() error { // Check if admin user exists var userCount int64 - err = d.db.Model(&User{}).Count(&userCount).Error + err := d.db.Model(&User{}).Count(&userCount).Error if err != nil { d.log.Error( "failed to count users", @@ -249,16 +316,46 @@ func (d *Database) createAdminUser() error { return err } - d.log.Info("admin user created", - "username", "admin", - "password", password, - "message", - "SAVE THIS PASSWORD - it will not be shown again!", + // The plaintext leaves this process here and nowhere else. It is + // deliberately not a log field: as one INFO record among the fx + // graph's own output it read as one more startup line, which is + // how deployments lost it. See internal/banner. + err = banner.Credentials( + d.banner(), + "WEBHOOKER FIRST BOOT: an admin account has been created.", + adminUser.Username, + password, + BootstrapPasswordNote, ) + if err != nil { + // Fail the start. The account is already committed, so the + // next boot seeds nothing and prints nothing: continuing here + // would hand the operator a running service whose only + // password was never shown. `webhooker resetpw` recovers it. + d.log.Error( + "failed to print the admin credentials banner", + "error", err, + ) + + return err + } + + d.log.Info("admin user created", "username", adminUser.Username) return nil } +// banner returns where the credentials banner is written. os.Stdout is +// resolved here rather than stored, so that a test which redirects the +// variable captures the banner. +func (d *Database) banner() io.Writer { + if d.bannerOut != nil { + return d.bannerOut + } + + return os.Stdout +} + func (d *Database) close() error { if d.db != nil { sqlDB, err := d.db.DB() diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 22f7312..3b6a939 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -18,6 +18,11 @@ const ( testVersion = "test" // testContentType is the event content type used in tests. testContentType = "application/json" + // testWebhookName is the Webhook.Name used in tests. + testWebhookName = "test-webhook" + // testForeverLabel is Webhook.RetentionLabel for a retain-forever + // webhook. + testForeverLabel = "forever" ) func setupTestDB( diff --git a/internal/database/event_db_isolation.go b/internal/database/event_db_isolation.go new file mode 100644 index 0000000..bd1abc7 --- /dev/null +++ b/internal/database/event_db_isolation.go @@ -0,0 +1,159 @@ +package database + +import ( + "fmt" + "log/slog" + + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +// omitAssociationsCallback is the name the association guard is +// registered under on a per-webhook database's create and update +// callback chains. +const omitAssociationsCallback = "webhooker:omit_associations" + +// omitAssociations makes every create and update issued against a +// per-webhook database skip GORM's automatic association save. +// +// A per-webhook database holds the event tier only, but Delivery +// declares belongs-to Event and Target and the delivery engine fills +// both in memory before writing. Without this guard GORM upserts +// those parent rows here on the delivery and retry write paths, +// copying targets.config, which holds destination URLs and bearer +// credentials, into the file most likely to be backed up or handed +// to someone else. Registering the guard on the connection covers +// every write path, including writes inside a transaction and write +// paths added later. Every event-tier row this file holds is written +// explicitly, so nothing depends on the automatic save. +func omitAssociations(db *gorm.DB) error { + omit := func(tx *gorm.DB) { + tx.Statement.Omits = append( + tx.Statement.Omits, clause.Associations, + ) + } + + err := db.Callback().Create(). + Before("gorm:save_before_associations"). + Register(omitAssociationsCallback, omit) + if err != nil { + return fmt.Errorf( + "registering create association guard: %w", err, + ) + } + + err = db.Callback().Update(). + Before("gorm:save_before_associations"). + Register(omitAssociationsCallback, omit) + if err != nil { + return fmt.Errorf( + "registering update association guard: %w", err, + ) + } + + return nil +} + +// eventDBSweptVersion is the PRAGMA user_version purgeTargetRows +// stamps into a per-webhook database once it has removed any leaked +// target rows *and* the VACUUM that removes their bytes has returned. +// Nothing else in the tree uses user_version, so 0 means "not swept +// by this build". +// +// The stamp, not the DELETE, is what records that a file is done. A +// DELETE commits on its own, so a sweep that is interrupted or whose +// VACUUM fails leaves a file whose rows are gone but whose credential +// bytes are still in the free pages -- indistinguishable, by row +// count, from a file that never leaked. Both leave the stamp unset, +// so the next open sweeps again. +const eventDBSweptVersion = 1 + +// purgeTargetRows deletes target rows that an earlier build's +// association upsert wrote into a per-webhook database, and rewrites +// the file so their bytes are gone with them. AutoMigrate creates a +// targets table in every one of these files because Delivery declares +// a belongs-to Target, but nothing in the event tier may put rows in +// it. The rows it did put there are junk, not history: they carry an +// empty webhook_id, and delivery rows resolve their target against +// the main database, so nothing here refers to them. +// +// The DELETE only unlinks the rows: modernc.org/sqlite leaves +// secure_delete at SQLite's default of off, so the credential bytes +// stay readable in the file's free pages and a backup of a swept file +// would still hand them over. VACUUM rewrites the file without them. +// +// This runs before every migration and is gated on +// eventDBSweptVersion, so a file pays for the rewrite once, on the +// first open that finds it unstamped, and every open after that is a +// PRAGMA read. A file this build created is stamped before its +// targets table exists, so it never vacuums at all. A failure here +// fails the open with the stamp left unset, so the sweep is retried +// rather than skipped -- a webhook whose file cannot be swept stays +// unusable instead of quietly serving from a file that still holds +// recoverable credentials. +func purgeTargetRows( + db *gorm.DB, log *slog.Logger, webhookID string, +) error { + var version int + + // Row().Scan, not (*gorm.DB).Scan: see internal/gormlog. + err := db.Raw("PRAGMA user_version").Row().Scan(&version) + if err != nil { + return fmt.Errorf( + "reading sweep marker of webhook database %s: %w", + webhookID, err, + ) + } + + if version >= eventDBSweptVersion { + return nil + } + + var purged int64 + + if db.Migrator().HasTable("targets") { + res := db.Exec("DELETE FROM targets") + if res.Error != nil { + return fmt.Errorf( + "purging target rows from webhook database %s: %w", + webhookID, res.Error, + ) + } + + purged = res.RowsAffected + + // Unconditional: a zero row count here does not mean there is + // nothing to remove, only that no *live* row is left. See + // eventDBSweptVersion. + err = db.Exec("VACUUM").Error + if err != nil { + return fmt.Errorf( + "purged %d leaked target rows from webhook database "+ + "%s but vacuuming it failed, so the deleted "+ + "target credentials are still recoverable from "+ + "the file; it stays marked unswept and the next "+ + "open retries: %w", + purged, webhookID, err, + ) + } + } + + err = db.Exec(fmt.Sprintf( + "PRAGMA user_version = %d", eventDBSweptVersion, + )).Error + if err != nil { + return fmt.Errorf( + "marking webhook database %s swept: %w", webhookID, err, + ) + } + + if purged > 0 { + log.Warn( + "purged leaked target rows from per-webhook database", + "webhook_id", webhookID, + "rows", purged, + ) + } + + return nil +} diff --git a/internal/database/event_db_isolation_test.go b/internal/database/event_db_isolation_test.go new file mode 100644 index 0000000..66c28e0 --- /dev/null +++ b/internal/database/event_db_isolation_test.go @@ -0,0 +1,438 @@ +package database_test + +import ( + "bytes" + "database/sql" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + _ "modernc.org/sqlite" + "sneak.berlin/go/webhooker/internal/database" +) + +// testDataDirPerm is the mode the test data directory is created +// with. +const testDataDirPerm = 0o750 + +// eventDBDataDir returns a data directory that a WebhookDBManager +// can be pointed at. +func eventDBDataDir(t *testing.T) string { + t.Helper() + + dir := filepath.Join(t.TempDir(), "events") + require.NoError(t, os.MkdirAll(dir, testDataDirPerm)) + + return dir +} + +// openRawEventDB opens the per-webhook database file directly, +// without the manager, so a test can put a file on disk in a state +// the manager has to cope with, or inspect one afterwards. +func openRawEventDB( + t *testing.T, dataDir, webhookID string, +) *sql.DB { + t.Helper() + + path := filepath.Join( + dataDir, fmt.Sprintf("events-%s.db", webhookID), + ) + + sqlDB, err := sql.Open( + "sqlite", + fmt.Sprintf("file:%s?mode=rwc", path), + ) + require.NoError(t, err) + + t.Cleanup(func() { _ = sqlDB.Close() }) + + return sqlDB +} + +// eventDBFileBytes reads a per-webhook database file off disk, so a +// test can assert on what the file itself still holds rather than on +// what a query returns. +func eventDBFileBytes(t *testing.T, dataDir, webhookID string) []byte { + t.Helper() + + //nolint:gosec // reads a file the test just created under t.TempDir() + raw, err := os.ReadFile(filepath.Join( + dataDir, fmt.Sprintf("events-%s.db", webhookID), + )) + require.NoError(t, err) + + return raw +} + +// eventDBUserVersion returns the PRAGMA user_version of a per-webhook +// database file, which is the marker purgeTargetRows stamps once it +// has swept and vacuumed. +func eventDBUserVersion(t *testing.T, sqlDB *sql.DB) int { + t.Helper() + + var version int + + require.NoError(t, sqlDB.QueryRowContext( + t.Context(), "PRAGMA user_version", + ).Scan(&version)) + + return version +} + +// clearEventDBSweptMarker resets the sweep marker to 0, which is what +// a file written by a build without the sweep looks like. Tests that +// seed a leaked row have to create the file through the manager to +// get the real targets table shape, and that stamps it. +func clearEventDBSweptMarker(t *testing.T, sqlDB *sql.DB) { + t.Helper() + + _, err := sqlDB.ExecContext(t.Context(), "PRAGMA user_version = 0") + require.NoError(t, err) +} + +// countTargetRows returns the number of rows in the targets table of +// a per-webhook database file, or -1 if the table does not exist. +func countTargetRows(t *testing.T, sqlDB *sql.DB) int { + t.Helper() + + var tables int + + require.NoError(t, sqlDB.QueryRowContext( + t.Context(), + "SELECT count(*) FROM sqlite_master "+ + "WHERE type = 'table' AND name = 'targets'", + ).Scan(&tables)) + + if tables == 0 { + return -1 + } + + var rows int + + require.NoError(t, sqlDB.QueryRowContext( + t.Context(), "SELECT count(*) FROM targets", + ).Scan(&rows)) + + return rows +} + +// TestOpenPurgesLeakedTargetRows covers the sweep for event +// databases written by a build that let GORM upsert target rows +// into them: opening the database clears them, and opening it again +// is a no-op. +func TestOpenPurgesLeakedTargetRows(t *testing.T) { + t.Parallel() + + dataDir := eventDBDataDir(t) + webhookID := uuid.New().String() + + // Create the file the way the application does, so the targets + // table has exactly the shape AutoMigrate gives it, then write + // a leaked row into it the way the association upsert did. + initial := database.NewTestWebhookDBManager(dataDir) + + _, err := initial.GetDB(webhookID) + require.NoError(t, err) + require.NoError(t, initial.CloseAll()) + + seed := openRawEventDB(t, dataDir, webhookID) + + _, err = seed.ExecContext( + t.Context(), + "INSERT INTO targets "+ + "(id, webhook_id, name, type, config) "+ + "VALUES (?, '', ?, ?, ?)", + uuid.New().String(), + "leaked-target", + "slack", + `{"webhookUrl":"https://hooks.example/T000/B000/secret"}`, + ) + require.NoError(t, err) + require.Equal(t, 1, countTargetRows(t, seed)) + clearEventDBSweptMarker(t, seed) + require.NoError(t, seed.Close()) + + mgr := database.NewTestWebhookDBManager(dataDir) + + _, err = mgr.GetDB(webhookID) + require.NoError(t, err) + require.NoError(t, mgr.CloseAll()) + + check := openRawEventDB(t, dataDir, webhookID) + assert.Zero(t, countTargetRows(t, check)) + assert.Equal( + t, 1, eventDBUserVersion(t, check), + "a completed sweep must mark the file so later opens skip it", + ) + require.NoError(t, check.Close()) + + // Idempotent: a second open leaves it at zero and does not + // error. + again := database.NewTestWebhookDBManager(dataDir) + + _, err = again.GetDB(webhookID) + require.NoError(t, err) + require.NoError(t, again.CloseAll()) + + recheck := openRawEventDB(t, dataDir, webhookID) + assert.Zero(t, countTargetRows(t, recheck)) +} + +// TestOpenPurgeRemovesCredentialBytes covers the sweep at the level +// that matters for a backup handed to someone else: the leaked +// credential must be gone from the raw bytes of the file, not merely +// unreachable by query. A bare DELETE unlinks the row and leaves the +// bytes readable in the free pages, so this fails without the VACUUM +// in purgeTargetRows. +func TestOpenPurgeRemovesCredentialBytes(t *testing.T) { + t.Parallel() + + dataDir := eventDBDataDir(t) + webhookID := uuid.New().String() + credential := "T00000000/B00000000/" + uuid.New().String() + + initial := database.NewTestWebhookDBManager(dataDir) + + _, err := initial.GetDB(webhookID) + require.NoError(t, err) + require.NoError(t, initial.CloseAll()) + + seed := openRawEventDB(t, dataDir, webhookID) + + _, err = seed.ExecContext( + t.Context(), + "INSERT INTO targets "+ + "(id, webhook_id, name, type, config) "+ + "VALUES (?, '', ?, ?, ?)", + uuid.New().String(), + "leaked-target", + "slack", + fmt.Sprintf( + `{"webhookUrl":"https://hooks.example/%s"}`, credential, + ), + ) + require.NoError(t, err) + clearEventDBSweptMarker(t, seed) + require.NoError(t, seed.Close()) + + // The seed has to be in the file for its absence later to mean + // anything. + require.True( + t, + bytes.Contains( + eventDBFileBytes(t, dataDir, webhookID), + []byte(credential), + ), + "seeded credential is not in the file, so this test proves nothing", + ) + + mgr := database.NewTestWebhookDBManager(dataDir) + + _, err = mgr.GetDB(webhookID) + require.NoError(t, err) + require.NoError(t, mgr.CloseAll()) + + assert.NotContains( + t, + string(eventDBFileBytes(t, dataDir, webhookID)), + credential, + "leaked credential is still recoverable from the raw file", + ) +} + +// TestOpenRevacuumsAfterIncompleteSweep covers the case a row count +// cannot see: the rows are already deleted but the file was never +// vacuumed, because an earlier sweep died between the two or its +// VACUUM failed. The credential bytes are still recoverable, and the +// unset marker is the only thing that says so, so the next open must +// vacuum rather than conclude from the empty table that there is +// nothing to do. +func TestOpenRevacuumsAfterIncompleteSweep(t *testing.T) { + t.Parallel() + + dataDir := eventDBDataDir(t) + webhookID := uuid.New().String() + credential := "T00000000/B00000000/" + uuid.New().String() + + initial := database.NewTestWebhookDBManager(dataDir) + + _, err := initial.GetDB(webhookID) + require.NoError(t, err) + require.NoError(t, initial.CloseAll()) + + seed := openRawEventDB(t, dataDir, webhookID) + + _, err = seed.ExecContext( + t.Context(), + "INSERT INTO targets "+ + "(id, webhook_id, name, type, config) "+ + "VALUES (?, '', ?, ?, ?)", + uuid.New().String(), + "leaked-target", + "slack", + fmt.Sprintf( + `{"webhookUrl":"https://hooks.example/%s"}`, credential, + ), + ) + require.NoError(t, err) + + // Exactly the state an interrupted sweep leaves: rows gone, + // marker unset, bytes still in the free pages. + _, err = seed.ExecContext(t.Context(), "DELETE FROM targets") + require.NoError(t, err) + require.Zero(t, countTargetRows(t, seed)) + clearEventDBSweptMarker(t, seed) + require.NoError(t, seed.Close()) + + require.True( + t, + bytes.Contains( + eventDBFileBytes(t, dataDir, webhookID), + []byte(credential), + ), + "the deleted row's bytes must still be in the file, or this "+ + "test proves nothing", + ) + + mgr := database.NewTestWebhookDBManager(dataDir) + + _, err = mgr.GetDB(webhookID) + require.NoError(t, err) + require.NoError(t, mgr.CloseAll()) + + assert.NotContains( + t, + string(eventDBFileBytes(t, dataDir, webhookID)), + credential, + "an interrupted sweep was not retried, so the credential is "+ + "still recoverable from the raw file", + ) + + check := openRawEventDB(t, dataDir, webhookID) + assert.Equal(t, 1, eventDBUserVersion(t, check)) +} + +// TestOpenSkipsSweptDatabase covers the other half of the marker: a +// file this build created is marked without ever being vacuumed, and +// a marked file is not swept again. +func TestOpenSkipsSweptDatabase(t *testing.T) { + t.Parallel() + + dataDir := eventDBDataDir(t) + webhookID := uuid.New().String() + + mgr := database.NewTestWebhookDBManager(dataDir) + + _, err := mgr.GetDB(webhookID) + require.NoError(t, err) + require.NoError(t, mgr.CloseAll()) + + marked := openRawEventDB(t, dataDir, webhookID) + assert.Equal(t, 1, eventDBUserVersion(t, marked)) + + // A marked file is left alone, so a row written into it survives + // a reopen. Nothing writes target rows any more; this stands in + // for the sweep having run. + _, err = marked.ExecContext( + t.Context(), + "INSERT INTO targets "+ + "(id, webhook_id, name, type, config) "+ + "VALUES (?, '', ?, ?, ?)", + uuid.New().String(), "sentinel", "slack", `{}`, + ) + require.NoError(t, err) + require.NoError(t, marked.Close()) + + again := database.NewTestWebhookDBManager(dataDir) + + _, err = again.GetDB(webhookID) + require.NoError(t, err) + require.NoError(t, again.CloseAll()) + + check := openRawEventDB(t, dataDir, webhookID) + assert.Equal( + t, 1, countTargetRows(t, check), + "a marked file must not be swept again", + ) +} + +// TestOpenSucceedsWithoutTargetsTable covers an existing event +// database that never grew a targets table. The sweep must not fail +// startup on it. +func TestOpenSucceedsWithoutTargetsTable(t *testing.T) { + t.Parallel() + + dataDir := eventDBDataDir(t) + webhookID := uuid.New().String() + + seed := openRawEventDB(t, dataDir, webhookID) + + _, err := seed.ExecContext( + t.Context(), + "CREATE TABLE events (id text PRIMARY KEY)", + ) + require.NoError(t, err) + require.NoError(t, seed.Close()) + + mgr := database.NewTestWebhookDBManager(dataDir) + + db, err := mgr.GetDB(webhookID) + require.NoError(t, err) + assert.NotNil(t, db) + require.NoError(t, mgr.CloseAll()) +} + +// TestEventDBCreateOmitsAssociations covers the connection-level +// guard directly: a Delivery carrying its Event and Target in +// memory, written through the manager's handle, must store only the +// delivery row. +func TestEventDBCreateOmitsAssociations(t *testing.T) { + t.Parallel() + + dataDir := eventDBDataDir(t) + webhookID := uuid.New().String() + + mgr := database.NewTestWebhookDBManager(dataDir) + + db, err := mgr.GetDB(webhookID) + require.NoError(t, err) + + target := database.Target{ + WebhookID: webhookID, + Name: "leaky-target", + Type: database.TargetTypeSlack, + Config: `{"webhookUrl":"https://hooks.example/secret"}`, + } + target.ID = uuid.New().String() + + event := database.Event{ + WebhookID: webhookID, + EntrypointID: uuid.New().String(), + Method: "POST", + Headers: `{}`, + Body: `{}`, + } + event.ID = uuid.New().String() + + d := &database.Delivery{ + EventID: event.ID, + TargetID: target.ID, + Status: database.DeliveryStatusPending, + Event: event, + Target: target, + } + d.ID = uuid.New().String() + + require.NoError(t, db.Create(d).Error) + require.NoError(t, db.Model(d). + Update("status", database.DeliveryStatusDelivered). + Error) + require.NoError(t, mgr.CloseAll()) + + check := openRawEventDB(t, dataDir, webhookID) + assert.Zero(t, countTargetRows(t, check)) +} diff --git a/internal/database/export_test.go b/internal/database/export_test.go index 29321fe..5c10271 100644 --- a/internal/database/export_test.go +++ b/internal/database/export_test.go @@ -2,9 +2,12 @@ package database import ( "context" + "io" "log/slog" "os" "time" + + "go.uber.org/fx" ) // NewTestRetentionReaper builds a RetentionReaper backed by the given @@ -29,3 +32,50 @@ func NewTestRetentionReaper( func (r *RetentionReaper) ExportSweep(ctx context.Context) { r.sweep(ctx) } + +// ExportRegisterHooks registers the reaper's real fx lifecycle hooks +// on a lifecycle supplied by a test, so a test can drive the exact +// OnStart/OnStop functions the application runs and hand OnStart the +// kind of context fx actually supplies. +func (r *RetentionReaper) ExportRegisterHooks(lc fx.Lifecycle) { + r.registerHooks(lc) +} + +// ExportStart starts the reaper's background loop for tests. +func (r *RetentionReaper) ExportStart() { + r.start() +} + +// ExportStop stops the reaper's background loop for tests. +func (r *RetentionReaper) ExportStop(ctx context.Context) error { + return r.stop(ctx) +} + +// ExportWedgeLoop adds a goroutine to the reaper's WaitGroup that +// never observes cancellation and returns only when release is +// closed. It stands in for a sweep stuck on a locked database. +func (r *RetentionReaper) ExportWedgeLoop( + release <-chan struct{}, +) { + r.wg.Go(func() { + <-release + }) +} + +// ExportSetInterval overrides the sweep interval for tests. +func (r *RetentionReaper) ExportSetInterval(d time.Duration) { + r.interval = d +} + +// ExportSetBannerOut redirects the first-boot credentials banner, so a +// test can read what the operator would have seen. It must be called +// before the fx start hook runs, which is where the account is seeded. +func (d *Database) ExportSetBannerOut(w io.Writer) { + d.bannerOut = w +} + +// DummyPasswordHashForTest exposes the encoded hash that unknown +// usernames are verified against. +func DummyPasswordHashForTest() string { + return dummyPasswordHash() +} diff --git a/internal/database/model_apikey.go b/internal/database/model_apikey.go index a1bd96b..5e8cee7 100644 --- a/internal/database/model_apikey.go +++ b/internal/database/model_apikey.go @@ -2,12 +2,16 @@ package database import "time" -// APIKey represents an API key for a user +// APIKey represents an API key for a user. +// +// Key is a bearer credential, so it is never marshalled with the +// model. A creation handler that has to show it once returns it in its +// own response type. type APIKey struct { BaseModel UserID string `gorm:"type:uuid;not null" json:"userId"` - Key string `gorm:"uniqueIndex;not null" json:"key"` + Key string `gorm:"uniqueIndex;not null" json:"-"` Description string `json:"description"` LastUsedAt *time.Time `json:"lastUsedAt,omitempty"` diff --git a/internal/database/model_delivery.go b/internal/database/model_delivery.go index a1fdbe9..71f6b6d 100644 --- a/internal/database/model_delivery.go +++ b/internal/database/model_delivery.go @@ -11,6 +11,23 @@ const ( DeliveryStatusRetrying DeliveryStatus = "retrying" ) +// Terminal reports whether a delivery in this status has finished, so +// the delivery engine will make no further attempt of its own. +// +// It is what decides which deliveries the event log offers to replay: +// a pending or retrying delivery is still the engine's, and replaying +// one would race it. +func (s DeliveryStatus) Terminal() bool { + switch s { + case DeliveryStatusDelivered, DeliveryStatusFailed: + return true + case DeliveryStatusPending, DeliveryStatusRetrying: + return false + default: + return false + } +} + // Delivery represents a delivery attempt for an event to a target type Delivery struct { BaseModel diff --git a/internal/database/model_entrypoint.go b/internal/database/model_entrypoint.go index 607f836..3021f48 100644 --- a/internal/database/model_entrypoint.go +++ b/internal/database/model_entrypoint.go @@ -6,7 +6,10 @@ type Entrypoint struct { WebhookID string `gorm:"type:uuid;not null" json:"webhookId"` - // Path is the URL path for this entrypoint. + // Path is the URL path for this entrypoint. It is the + // entrypoint's only credential: possession of the UUID + // authorises submission, so the receiver checks nothing else + // about the sender. Path string `gorm:"uniqueIndex;not null" json:"path"` Description string `json:"description"` diff --git a/internal/database/model_event.go b/internal/database/model_event.go index bd332d6..dafc235 100644 --- a/internal/database/model_event.go +++ b/internal/database/model_event.go @@ -13,6 +13,14 @@ type Event struct { Body string `gorm:"type:text" json:"body"` ContentType string `json:"contentType"` + // ResubmittedFromID names the event this one was copied from by + // an operator resubmit. It is nil for an event that arrived on + // the receiver, which is every event created before the column + // existed. It is not a foreign key: the source event can be + // reaped by retention while its copies remain, and the id is + // kept as the record of where the copy came from either way. + ResubmittedFromID *string `gorm:"type:uuid;index" json:"resubmittedFromId,omitempty"` + // Relations Webhook Webhook `json:"webhook,omitzero"` Entrypoint Entrypoint `json:"entrypoint,omitzero"` diff --git a/internal/database/model_secrets_test.go b/internal/database/model_secrets_test.go new file mode 100644 index 0000000..84bd98f --- /dev/null +++ b/internal/database/model_secrets_test.go @@ -0,0 +1,107 @@ +package database_test + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/database" +) + +// keptField is a non-secret value planted alongside each secret, so +// the assertions below cannot pass by the model marshalling to nothing. +const keptField = "keepme" + +// marshalModel encodes a model the way a future JSON handler would. +func marshalModel(t *testing.T, v any) string { + t.Helper() + + encoded, err := json.Marshal(v) + require.NoError(t, err) + + return string(encoded) +} + +// TestModelsDoNotMarshalTheirSecrets pins the barrier for the JSON +// path. The /api/v1 route group exists and is empty; delivery's +// TargetView masks the credential for the HTML path only, so without +// these tags the first handler that marshals a model serialises the +// secret with it. Each field below is a live credential: +// +// - Target.Config holds an incoming-webhook URL whose path segments +// are the bearer token. +// - APIKey.Key is a bearer token outright. +// - Setting.Value holds the session encryption key. +// - User.Password holds the Argon2 hash, and was already tagged. +func TestModelsDoNotMarshalTheirSecrets(t *testing.T) { + t.Parallel() + + const marker = "QQMODELMARKERQQ" + + cases := []struct { + name string + model any + }{ + { + name: "target config", + model: database.Target{ + Name: keptField, + Type: database.TargetTypeSlack, + Config: `{"webhookUrl":"https://h/s/` + marker + `"}`, + }, + }, + { + name: "api key", + model: database.APIKey{ + Description: keptField, + Key: marker, + }, + }, + { + name: "setting value", + model: database.Setting{ + Key: keptField, + Value: marker, + }, + }, + { + name: "user password hash", + model: database.User{ + Username: keptField, + Password: marker, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + encoded := marshalModel(t, tc.model) + + assert.NotContains(t, encoded, marker) + assert.Contains(t, encoded, keptField) + }) + } +} + +// TestWebhookMarshalsNoTargetConfig covers the nested case: a webhook +// marshalled with its targets preloaded must not carry the credential +// through the association either. +func TestWebhookMarshalsNoTargetConfig(t *testing.T) { + t.Parallel() + + const marker = "QQNESTEDMARKERQQ" + + encoded := marshalModel(t, database.Webhook{ + Name: keptField, + Targets: []database.Target{{ + Name: "slack", + Config: `{"webhookUrl":"https://h/s/` + marker + `"}`, + }}, + }) + + assert.NotContains(t, encoded, marker) + assert.Contains(t, encoded, keptField) +} diff --git a/internal/database/model_setting.go b/internal/database/model_setting.go index f120fec..cdd068b 100644 --- a/internal/database/model_setting.go +++ b/internal/database/model_setting.go @@ -3,6 +3,9 @@ package database // Setting stores application-level key-value configuration. // Used for auto-generated values like the session encryption key. type Setting struct { - Key string `gorm:"primaryKey" json:"key"` - Value string `gorm:"type:text;not null" json:"value"` + Key string `gorm:"primaryKey" json:"key"` + + // Value holds the session encryption key, so it is never + // marshalled with the model. + Value string `gorm:"type:text;not null" json:"-"` } diff --git a/internal/database/model_target.go b/internal/database/model_target.go index 9c8aa83..9c5f95f 100644 --- a/internal/database/model_target.go +++ b/internal/database/model_target.go @@ -20,8 +20,14 @@ type Target struct { Type TargetType `gorm:"not null" json:"type"` Active bool `gorm:"default:true" json:"active"` - // Configuration fields (JSON stored based on type) - Config string `gorm:"type:text" json:"config"` // JSON configuration + // Configuration fields (JSON stored based on type). + // + // json:"-" because the blob holds the target's credential — a + // Slack incoming-webhook URL, or an http destination whose path + // segments are the secret. delivery.TargetView is the masking + // barrier for the HTML path; this tag is the barrier for any + // handler that marshals the model itself. + Config string `gorm:"type:text" json:"-"` // JSON configuration // For HTTP targets (max_retries=0 means fire-and-forget, // >0 enables retries with backoff) diff --git a/internal/database/model_webhook.go b/internal/database/model_webhook.go index 9b47516..eedb192 100644 --- a/internal/database/model_webhook.go +++ b/internal/database/model_webhook.go @@ -1,6 +1,59 @@ package database +import ( + "math" + "strconv" + "time" + + "gorm.io/gorm" +) + +const ( + // DefaultRetentionDays is the event retention period applied to a + // webhook created without an explicit retention value. It is the + // single source of truth for that policy and must stay in sync + // with the `gorm:"default:30"` column default on + // Webhook.RetentionDays below; a struct tag cannot reference a + // constant, so a test asserts the two agree. + DefaultRetentionDays = 30 + + // RetentionForeverDays is the sentinel RetentionDays value meaning + // "retain events forever". Users express that intent as 0, which + // Webhook.BeforeSave rewrites to this value: the column default + // substitutes DefaultRetentionDays for a zero value at insert + // time, so a zero can never survive a round trip to the database. + // Nothing outside this file may hardcode the number. + RetentionForeverDays = 365 * 1000 + + // MaxFiniteRetentionDays is the largest finite retention period the + // reaper's cutoff arithmetic can represent, and therefore the + // largest one a caller may request. It is derived from that + // arithmetic rather than picked: retentionCutoff computes + // retentionDays * hoursPerDay * time.Hour, and a time.Duration is + // an int64 nanosecond count, so math.MaxInt64 nanoseconds divided + // by an hour and then by a day is the exact ceiling — 106751 days, + // a little over 292 years. + // + // One day more overflows int64, wraps the product negative, and + // turns the cutoff into a timestamp in the far future that matches + // every row in the webhook's database. That is why this bound is + // enforced on input and why retentionCutoff saturates underneath + // it. Note that RetentionForeverDays deliberately sits above this + // ceiling: such webhooks are skipped before any cutoff is + // computed, and never reach the arithmetic at all. + MaxFiniteRetentionDays = int( + math.MaxInt64 / int64(time.Hour) / hoursPerDay, + ) +) + // Webhook represents a webhook processing unit that groups entrypoints and targets +// +// Every method below takes a pointer receiver. BeforeSave has to, +// because it mutates the record and GORM only invokes hooks declared +// that way; the display helpers follow suit so the receiver kinds do +// not mix. Handlers therefore put a *Webhook into template data: +// html/template cannot call a pointer method on a value held in a map, +// because a map element is not addressable. type Webhook struct { BaseModel @@ -8,7 +61,9 @@ type Webhook struct { Name string `gorm:"not null" json:"name"` Description string `json:"description"` - // RetentionDays is the number of days to retain events. + // RetentionDays is the number of days to retain events. A value of + // RetentionForeverDays means retain forever. The column default + // must equal DefaultRetentionDays. RetentionDays int `gorm:"default:30" json:"retentionDays"` // Relations @@ -16,3 +71,55 @@ type Webhook struct { Entrypoints []Entrypoint `json:"entrypoints,omitempty"` Targets []Target `json:"targets,omitempty"` } + +// BeforeSave normalises RetentionDays on every insert and update. A +// non-positive value is the user's way of asking for "retain forever", +// which is stored as the RetentionForeverDays sentinel. +// +// This has to happen in a hook rather than at the call sites. GORM +// substitutes the column default (DefaultRetentionDays) for a zero +// value while building the insert statement, which runs after +// BeforeSave; rewriting any later than this loses that race and the +// row lands at 30 days. Living on the model also means a future call +// site — a REST API, a fixture, a migration — cannot bypass it. +func (w *Webhook) BeforeSave(_ *gorm.DB) error { + if w.RetentionDays <= 0 { + w.RetentionDays = RetentionForeverDays + } + + return nil +} + +// retainsForever reports whether a stored RetentionDays value means +// "keep events indefinitely". It is the single definition of that +// question, shared by Webhook.RetainsForever and by the reaper's +// cutoff computation so the two cannot disagree about which webhooks +// are exempt from reaping. +// +// It accepts the RetentionForeverDays sentinel written by BeforeSave +// and, defensively, the non-positive values that rows written before +// the sentinel existed may still carry. +func retainsForever(retentionDays int) bool { + return retentionDays <= 0 || + retentionDays >= RetentionForeverDays +} + +// RetainsForever reports whether this webhook's events are kept +// indefinitely. +func (w *Webhook) RetainsForever() bool { + return retainsForever(w.RetentionDays) +} + +// RetentionLabel returns the webhook's retention policy as display +// text, so that no template has to know about the sentinel value. +func (w *Webhook) RetentionLabel() string { + if w.RetainsForever() { + return "forever" + } + + if w.RetentionDays == 1 { + return "1 day" + } + + return strconv.Itoa(w.RetentionDays) + " days" +} diff --git a/internal/database/model_webhook_test.go b/internal/database/model_webhook_test.go new file mode 100644 index 0000000..d8be7fa --- /dev/null +++ b/internal/database/model_webhook_test.go @@ -0,0 +1,222 @@ +package database_test + +import ( + "context" + "reflect" + "strconv" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "sneak.berlin/go/webhooker/internal/database" +) + +// startedTestDB returns a started main database for model-level tests. +func startedTestDB(t *testing.T) *gorm.DB { + t.Helper() + + db, lc := setupTestDB(t) + + ctx := context.Background() + require.NoError(t, lc.Start(ctx)) + t.Cleanup(func() { require.NoError(t, lc.Stop(ctx)) }) + + return db.DB() +} + +// storedRetention reads the retention_days column straight out of the +// row, so the assertion is about what was persisted rather than about +// whatever the in-memory struct happens to hold. +func storedRetention(t *testing.T, db *gorm.DB, id string) int { + t.Helper() + + var got int + + require.NoError( + t, + db.Model(&database.Webhook{}). + Where("id = ?", id). + Pluck("retention_days", &got).Error, + ) + + return got +} + +// newWebhookWithRetention creates a webhook through the ordinary Create +// path, so the BeforeSave hook and the GORM column default both apply +// exactly as they do in production. +func newWebhookWithRetention( + t *testing.T, + db *gorm.DB, + wh *database.Webhook, +) string { + t.Helper() + + wh.UserID = uuid.New().String() + wh.Name = testWebhookName + + require.NoError( + t, + db.Omit(clause.Associations).Create(wh).Error, + ) + + return wh.ID +} + +func TestWebhookBeforeSave_ZeroBecomesForeverSentinel(t *testing.T) { + t.Parallel() + + db := startedTestDB(t) + + wh := &database.Webhook{RetentionDays: 0} + id := newWebhookWithRetention(t, db, wh) + + assert.Equal( + t, + database.RetentionForeverDays, + storedRetention(t, db, id), + "a zero retention must be stored as the sentinel, "+ + "not replaced by the column default", + ) +} + +func TestWebhookBeforeSave_NegativeBecomesForeverSentinel(t *testing.T) { + t.Parallel() + + db := startedTestDB(t) + + wh := &database.Webhook{RetentionDays: -5} + id := newWebhookWithRetention(t, db, wh) + + assert.Equal( + t, + database.RetentionForeverDays, + storedRetention(t, db, id), + ) +} + +func TestWebhookBeforeSave_PositiveIsPreserved(t *testing.T) { + t.Parallel() + + db := startedTestDB(t) + + wh := &database.Webhook{RetentionDays: 7} + id := newWebhookWithRetention(t, db, wh) + + assert.Equal(t, 7, storedRetention(t, db, id)) +} + +// TestWebhookBeforeSave_UpdateToZeroBecomesSentinel proves the hook +// fires on update as well as insert, via the same Save call the edit +// handler makes. +func TestWebhookBeforeSave_UpdateToZeroBecomesSentinel(t *testing.T) { + t.Parallel() + + db := startedTestDB(t) + + wh := &database.Webhook{RetentionDays: 30} + id := newWebhookWithRetention(t, db, wh) + require.Equal(t, 30, storedRetention(t, db, id)) + + wh.RetentionDays = 0 + require.NoError(t, db.Omit(clause.Associations).Save(wh).Error) + + assert.Equal( + t, + database.RetentionForeverDays, + storedRetention(t, db, id), + ) +} + +// TestWebhookRetentionColumnDefaultMatchesConstant guards the one place +// the default lives twice: a struct tag cannot reference a constant, so +// this asserts the tag and DefaultRetentionDays agree. +func TestWebhookRetentionColumnDefaultMatchesConstant(t *testing.T) { + t.Parallel() + + field, ok := reflect.TypeFor[database.Webhook](). + FieldByName("RetentionDays") + require.True(t, ok, "Webhook.RetentionDays must exist") + + assert.Equal( + t, + "default:"+strconv.Itoa(database.DefaultRetentionDays), + field.Tag.Get("gorm"), + ) +} + +// TestMaxFiniteRetentionDaysIsTheOverflowCeiling asserts that the +// constant is exactly where the cutoff arithmetic stops working, which +// is what makes it a derived bound rather than a round number someone +// liked. One day more wraps the int64 nanosecond count negative, and a +// negative span is precisely what turned a cutoff into a future +// timestamp that matched — and deleted — every row. +// +// The multiplications are done through variables on purpose: as +// constant expressions the overflowing one would not compile. +func TestMaxFiniteRetentionDaysIsTheOverflowCeiling(t *testing.T) { + t.Parallel() + + const hoursPerDay = 24 + + atCeiling := database.MaxFiniteRetentionDays + overCeiling := database.MaxFiniteRetentionDays + 1 + + assert.Positive( + t, + time.Duration(atCeiling*hoursPerDay)*time.Hour, + "the ceiling itself must still be representable", + ) + assert.Negative( + t, + time.Duration(overCeiling*hoursPerDay)*time.Hour, + "one day past the ceiling must overflow", + ) + + assert.Less( + t, + database.MaxFiniteRetentionDays, + database.RetentionForeverDays, + "the sentinel sits above the ceiling and is only safe "+ + "because retain-forever webhooks skip the arithmetic", + ) +} + +func TestWebhookRetainsForeverAndLabel(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + days int + forever bool + label string + }{ + { + "sentinel", + database.RetentionForeverDays, true, testForeverLabel, + }, + { + "above sentinel", + database.RetentionForeverDays + 1, true, testForeverLabel, + }, + {"legacy zero", 0, true, testForeverLabel}, + {"legacy negative", -1, true, testForeverLabel}, + {"default", database.DefaultRetentionDays, false, "30 days"}, + {"one day", 1, false, "1 day"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + wh := database.Webhook{RetentionDays: tc.days} + + assert.Equal(t, tc.forever, wh.RetainsForever()) + assert.Equal(t, tc.label, wh.RetentionLabel()) + }) + } +} diff --git a/internal/database/password.go b/internal/database/password.go index 24fe5f6..92ce50a 100644 --- a/internal/database/password.go +++ b/internal/database/password.go @@ -8,6 +8,7 @@ import ( "fmt" "math/big" "strings" + "sync" "golang.org/x/crypto/argon2" ) @@ -29,6 +30,10 @@ const hashParts = 6 // triggers per-character-class complexity enforcement. const minPasswordComplexityLen = 4 +// dummyPasswordLen is the length of the throwaway password behind +// dummyPasswordHash. +const dummyPasswordLen = 32 + // Sentinel errors returned by decodeHash. var ( errInvalidHashFormat = errors.New("invalid hash format") @@ -122,6 +127,38 @@ func VerifyPassword( return subtle.ConstantTimeCompare(hash, otherHash) == 1, nil } +// dummyPasswordHash is an encoded Argon2id hash of a random +// password, computed once on first use. Nothing can match it: the +// password it encodes is discarded as soon as it is hashed. It is +// process-wide because building it per request would add a second +// 64 MB Argon2id pass to every login for an unknown username. +// +//nolint:gochecknoglobals // computed once, see above +var dummyPasswordHash = sync.OnceValue(func() string { + password, err := GenerateRandomPassword(dummyPasswordLen) + if err != nil { + panic(fmt.Sprintf("generating the dummy password: %v", err)) + } + + hash, err := HashPassword(password) + if err != nil { + panic(fmt.Sprintf("hashing the dummy password: %v", err)) + } + + return hash +}) + +// VerifyDummyPassword performs a credential verification that cannot +// succeed, at the same cost as a real one. +// +// Login must charge an unknown username the same work as a known +// one. Returning early for an account that does not exist answers in +// microseconds where a real account takes tens of milliseconds, which +// is a username oracle any client can read off the response time. +func VerifyDummyPassword(password string) { + _, _ = VerifyPassword(password, dummyPasswordHash()) +} + // decodeHash extracts parameters, salt, and hash from an // encoded hash string. func decodeHash( diff --git a/internal/database/password_test.go b/internal/database/password_test.go index ae3e2e4..e3f4726 100644 --- a/internal/database/password_test.go +++ b/internal/database/password_test.go @@ -191,3 +191,41 @@ func TestHashPasswordUniqueness(t *testing.T) { ) } } + +// TestVerifyDummyPassword_DoesRealWork covers the anti-enumeration +// path. Login charges an unknown username a verification against a +// dummy hash so that a nonexistent account is not answered in +// microseconds where a real one takes tens of milliseconds. That only +// works if the dummy hash is a real, decodable Argon2id hash: a +// malformed one would make VerifyPassword fail on the decode and +// return before hashing anything. +func TestVerifyDummyPassword_DoesRealWork(t *testing.T) { + t.Parallel() + + // Runs the OnceValue that builds the dummy hash, so a panic in + // it surfaces here rather than on a live login. + database.VerifyDummyPassword("whatever was submitted") + + dummy := database.DummyPasswordHashForTest() + + // A hash the verifier cannot decode would make VerifyPassword + // return on the decode error, before hashing anything — the + // timing oracle this path exists to close. + valid, err := database.VerifyPassword("whatever", dummy) + if err != nil { + t.Fatalf( + "the dummy hash must decode like a real one: %v", err, + ) + } + + if valid { + t.Error("nothing may authenticate against the dummy hash") + } + + if !strings.HasPrefix(dummy, "$argon2id$") { + t.Errorf( + "the dummy hash must use the same algorithm as real "+ + "hashes, got %q", dummy, + ) + } +} diff --git a/internal/database/retention.go b/internal/database/retention.go index 23d516f..13051af 100644 --- a/internal/database/retention.go +++ b/internal/database/retention.go @@ -10,6 +10,7 @@ import ( "go.uber.org/fx" "gorm.io/gorm" "sneak.berlin/go/webhooker/internal/config" + "sneak.berlin/go/webhooker/internal/lifecycle" "sneak.berlin/go/webhooker/internal/logger" ) @@ -56,24 +57,42 @@ func NewRetentionReaper( interval: params.Config.RetentionSweepInterval, } - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - r.start(ctx) - - return nil - }, - OnStop: func(_ context.Context) error { - r.stop() - - return nil - }, - }) + r.registerHooks(lc) return r } -func (r *RetentionReaper) start(ctx context.Context) { - ctx, cancel := context.WithCancel(ctx) +// registerHooks wires the reaper's start and stop into the fx +// lifecycle. The start hook's context is deliberately ignored (see +// start for why the sweep loop must not inherit it); the stop hook's +// context is honoured (see stop). +func (r *RetentionReaper) registerHooks(lc fx.Lifecycle) { + lc.Append(fx.Hook{ + //nolint:contextcheck // Not inheriting the hook context is + // the point: see start. + OnStart: func(_ context.Context) error { + r.start() + + return nil + }, + OnStop: func(ctx context.Context) error { + return r.stop(ctx) + }, + }) +} + +// start launches the background sweep loop. +// +// The loop's context is derived from context.Background(), NOT from +// the fx OnStart hook context. The hook context carries fx's start +// timeout (15s by default) and is cancelled once the start phase +// completes, so a loop derived from it dies 45 minutes before its +// first tick under the default one-hour sweep interval, leaving a +// reaper that never reaps. A long-lived goroutine must outlive the +// startup phase, so its lifetime is bounded by OnStop instead: stop +// cancels this context and waits on the WaitGroup. +func (r *RetentionReaper) start() { + ctx, cancel := context.WithCancel(context.Background()) r.cancel = cancel r.wg.Add(1) @@ -86,15 +105,27 @@ func (r *RetentionReaper) start(ctx context.Context) { ) } -func (r *RetentionReaper) stop() { +// stop cancels the sweep loop's context and waits for it to +// exit, bounded by the stop hook's context: a sweep wedged on a +// locked database must not hang the process past fx's stop +// timeout. +func (r *RetentionReaper) stop(ctx context.Context) error { r.log.Info("retention reaper stopping") if r.cancel != nil { r.cancel() } - r.wg.Wait() + err := lifecycle.WaitForShutdown( + ctx, r.log, "retention reaper", &r.wg, + ) + if err != nil { + return err + } + r.log.Info("retention reaper stopped") + + return nil } func (r *RetentionReaper) run(ctx context.Context) { @@ -114,7 +145,8 @@ func (r *RetentionReaper) run(ctx context.Context) { } // sweep lists every webhook from the main database and reaps expired -// rows from each per-webhook database whose RetentionDays is positive. +// rows from each per-webhook database that has a finite retention +// policy. Webhooks set to retain forever are skipped entirely. func (r *RetentionReaper) sweep(ctx context.Context) { var webhooks []Webhook @@ -139,8 +171,13 @@ func (r *RetentionReaper) sweep(ctx context.Context) { wh := webhooks[i] - // RetentionDays of zero or less means retain forever. - if wh.RetentionDays <= 0 { + // Skip retain-forever webhooks before building any query. + // RetainsForever covers both the RetentionForeverDays + // sentinel and the non-positive values that predate it: the + // sentinel is a positive number, so without this the reaper + // would compute a cutoff a thousand years in the past and + // issue a DELETE matching nothing on every single sweep. + if wh.RetainsForever() { continue } @@ -171,9 +208,10 @@ func (r *RetentionReaper) reapWebhook( return } - cutoff := time.Now().Add( - -time.Duration(retentionDays*hoursPerDay) * time.Hour, - ) + cutoff, ok := retentionCutoff(time.Now(), retentionDays) + if !ok { + return + } deleted, err := reapExpired(db, cutoff) if err != nil { @@ -196,6 +234,37 @@ func (r *RetentionReaper) reapWebhook( } } +// retentionCutoff returns the timestamp before which a webhook's +// events have expired, and whether any cutoff applies at all. It +// reports false for a retain-forever policy, so no DELETE is issued. +// +// The day count is clamped to MaxFiniteRetentionDays first. This is +// defense in depth rather than decoration: a time.Duration is an int64 +// nanosecond count, so an unclamped multiplication overflows above +// that ceiling and wraps the span negative. Subtracting a negative +// span moves the cutoff into the far future, where it matches every +// row in the database: the sweep then deletes every event, delivery, +// and delivery result, including ones created seconds ago. Rejecting +// out-of-range input at the form is the primary guard; saturating here +// means an old row, a migration, or a future call site cannot turn a +// too-large retention into total data loss. +func retentionCutoff( + now time.Time, + retentionDays int, +) (time.Time, bool) { + if retainsForever(retentionDays) { + return time.Time{}, false + } + + if retentionDays > MaxFiniteRetentionDays { + retentionDays = MaxFiniteRetentionDays + } + + return now.Add( + -time.Duration(retentionDays*hoursPerDay) * time.Hour, + ), true +} + // reapExpired hard-deletes, in foreign-key-safe order, the delivery // results, deliveries, and events associated with events older than // cutoff. Deletes are unscoped so rows are physically removed rather diff --git a/internal/database/retention_lifecycle_test.go b/internal/database/retention_lifecycle_test.go new file mode 100644 index 0000000..ed92695 --- /dev/null +++ b/internal/database/retention_lifecycle_test.go @@ -0,0 +1,272 @@ +package database_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/fx" + "gorm.io/gorm" + "sneak.berlin/go/webhooker/internal/database" +) + +const ( + // reaperTestInterval is the sweep interval a lifecycle test + // runs the reaper at, so a loop that survives startup produces + // an observable sweep quickly. + reaperTestInterval = 10 * time.Millisecond + + // reaperStopTimeout bounds how long a lifecycle test waits for + // the reaper's OnStop hook to return before declaring the + // shutdown hung. + reaperStopTimeout = 10 * time.Second + + // reaperTestRetentionDays is the retention policy the lifecycle + // tests give their webhook. + reaperTestRetentionDays = 30 + + // reaperWedgeStopTimeout is the stop timeout the wedged-shutdown + // test hands OnStop, standing in for fx's StopTimeout. The test + // asserts only that the hook returns at all, and allows it + // reaperStopTimeout — forty times this budget — to do so, so no + // assertion races the wall clock. + reaperWedgeStopTimeout = 250 * time.Millisecond +) + +// recordingLifecycle is a minimal fx.Lifecycle that records the +// hooks a component registers, so a test can invoke the real +// OnStart/OnStop functions with a context of its choosing. +type recordingLifecycle struct { + hooks []fx.Hook +} + +func (l *recordingLifecycle) Append(h fx.Hook) { + l.hooks = append(l.hooks, h) +} + +// startReaperViaHook drives the genuine fx hooks the application +// registers for the reaper, handing OnStart a context that is +// already done. It returns the recorded lifecycle so the caller +// can drive OnStop too. +func startReaperViaHook( + t *testing.T, r *database.RetentionReaper, +) *recordingLifecycle { + t.Helper() + + lc := &recordingLifecycle{} + r.ExportRegisterHooks(lc) + require.Len(t, lc.hooks, 1) + + // fx hands OnStart a context carrying the application start + // timeout, and cancels it when the start phase ends. An + // already-cancelled context is that same defect taken to its + // limit, and unlike a plain context.Background() it actually + // distinguishes a correctly rooted loop from a broken one. + hookCtx, cancel := context.WithCancel(context.Background()) + cancel() + + require.NoError(t, lc.hooks[0].OnStart(hookCtx)) + + return lc +} + +// eventGone reports whether an event row has been removed. It +// takes no *testing.T because it is polled from an +// assert.Eventually condition, which runs off the test goroutine +// where testify assertions must not be used. +func eventGone(db *gorm.DB, eventID string) bool { + var n int64 + + err := db.Unscoped().Model(&database.Event{}). + Where("id = ?", eventID).Count(&n).Error + if err != nil { + return false + } + + return n == 0 +} + +// seedExpiredWebhook creates a webhook with a finite retention +// policy plus one long-expired event chain, and returns the +// webhook's database and the chain's event ID. +func seedExpiredWebhook( + t *testing.T, env *retentionTestEnv, +) (*gorm.DB, string) { + t.Helper() + + webhookID := createWebhook( + t, env.mainDB.DB(), reaperTestRetentionDays, + ) + + db, err := env.mgr.GetDB(webhookID) + require.NoError(t, err) + + chain := seedEventChain( + t, db, webhookID, + time.Now().Add(-365*24*time.Hour), + ) + + return db, chain.eventID +} + +// TestRetentionReaper_LoopOutlivesStartHookContext is the +// regression test for a reaper that never reaped. fx calls +// OnStart with a context carrying the application's start timeout +// (15s by default) and cancels it when the start phase ends, so a +// sweep loop rooted in it is dead three quarters of an hour +// before its first tick under the default one-hour interval, and +// per-webhook event databases grow without bound exactly as they +// did before retention existed. +// +// Driving OnStart with an already-cancelled context is that +// defect taken to its limit: a loop that inherits the hook +// context never ticks once, while a correctly rooted loop keeps +// sweeping for as long as the process lives. +func TestRetentionReaper_LoopOutlivesStartHookContext( + t *testing.T, +) { + t.Parallel() + + env := setupRetentionTest(t) + + db, eventID := seedExpiredWebhook(t, env) + + env.reaper.ExportSetInterval(reaperTestInterval) + + lc := startReaperViaHook(t, env.reaper) + t.Cleanup(func() { + _ = lc.hooks[0].OnStop(context.Background()) + }) + + assert.Eventually( + t, + func() bool { return eventGone(db, eventID) }, + 5*time.Second, + reaperTestInterval, + "the sweep loop must keep running after the start "+ + "hook's context is done; it reaped nothing, so it "+ + "inherited the hook context and died", + ) +} + +// TestRetentionReaper_StopHookStopsLoop proves the fix did not +// trade a startup bug for a shutdown hang: now that the sweep +// loop no longer observes the start hook's cancellation, OnStop +// is the only thing that can stop it, and it must both return +// promptly and actually leave the loop stopped. +func TestRetentionReaper_StopHookStopsLoop(t *testing.T) { + t.Parallel() + + env := setupRetentionTest(t) + + db, eventID := seedExpiredWebhook(t, env) + + env.reaper.ExportSetInterval(reaperTestInterval) + + lc := startReaperViaHook(t, env.reaper) + + // Let the loop prove it is running before stopping it, so a + // fast OnStop cannot pass by stopping something already dead. + require.Eventually( + t, + func() bool { return eventGone(db, eventID) }, + 5*time.Second, + reaperTestInterval, + ) + + var stopErr error + + stopped := make(chan struct{}) + + go func() { + defer close(stopped) + + // stop blocks on the loop's WaitGroup, so returning at all + // proves the goroutine observed the cancellation. + stopErr = lc.hooks[0].OnStop(context.Background()) + }() + + select { + case <-stopped: + case <-time.After(reaperStopTimeout): + t.Fatal( + "OnStop did not return: the retention reaper's " + + "WaitGroup is still waiting on a loop that never " + + "observed cancellation", + ) + } + + require.NoError(t, stopErr) + + // With the loop gone, a newly expired chain must survive. + survivor := seedEventChain( + t, db, "stopped-webhook", + time.Now().Add(-365*24*time.Hour), + ) + + time.Sleep(20 * reaperTestInterval) + + assert.False( + t, + eventGone(db, survivor.eventID), + "a stopped reaper must not sweep anything", + ) +} + +// TestRetentionReaper_StopHookHonoursStopTimeout is the +// regression test for a shutdown that could never complete. fx +// hands OnStop a context carrying the application's stop timeout; +// an OnStop that discards it and calls wg.Wait() bare hangs the +// process forever on a sweep blocked on a locked SQLite database +// — precisely when a bounded shutdown matters most. +// +// The wedged goroutine here never observes cancellation, so the +// hook can only return by honouring its context, and it must say +// so rather than reporting a clean stop. +func TestRetentionReaper_StopHookHonoursStopTimeout( + t *testing.T, +) { + t.Parallel() + + env := setupRetentionTest(t) + + env.reaper.ExportSetInterval(reaperTestInterval) + + lc := startReaperViaHook(t, env.reaper) + + release := make(chan struct{}) + + t.Cleanup(func() { close(release) }) + + env.reaper.ExportWedgeLoop(release) + + stopCtx, cancel := context.WithTimeout( + context.Background(), reaperWedgeStopTimeout, + ) + defer cancel() + + var stopErr error + + stopped := make(chan struct{}) + + go func() { + defer close(stopped) + + stopErr = lc.hooks[0].OnStop(stopCtx) + }() + + select { + case <-stopped: + case <-time.After(reaperStopTimeout): + t.Fatal( + "OnStop did not return: it discarded the stop " + + "context and is waiting on a wedged goroutine " + + "that will never observe cancellation", + ) + } + + require.ErrorIs(t, stopErr, context.DeadlineExceeded) + require.ErrorContains(t, stopErr, "retention reaper") +} diff --git a/internal/database/retention_test.go b/internal/database/retention_test.go index 0ff0c90..c2dccef 100644 --- a/internal/database/retention_test.go +++ b/internal/database/retention_test.go @@ -77,7 +77,7 @@ func createWebhook( wh := &database.Webhook{ UserID: uuid.New().String(), - Name: "test-webhook", + Name: testWebhookName, RetentionDays: retentionDays, } require.NoError( @@ -85,10 +85,11 @@ func createWebhook( db.Omit(clause.Associations).Create(wh).Error, ) - // The RetentionDays column carries a GORM default of 30, so a - // zero (or negative) value passed to Create is replaced by that - // default. Force the requested value explicitly so the - // retain-forever (<= 0) path can be exercised. + // Webhook.BeforeSave rewrites a non-positive RetentionDays to the + // retain-forever sentinel, and the column's GORM default would + // otherwise substitute 30. Force the requested value with a + // column-level update so tests can plant legacy rows that predate + // the sentinel and still carry a literal 0 or negative value. require.NoError( t, db.Model(wh). @@ -98,6 +99,30 @@ func createWebhook( return wh.ID } +// createWebhookNormally inserts a webhook through the ordinary Create +// path, with no column-level forcing, so Webhook.BeforeSave applies +// exactly as it does in production. Passing 0 therefore yields a row +// holding the RetentionForeverDays sentinel. +func createWebhookNormally( + t *testing.T, + db *gorm.DB, + retentionDays int, +) string { + t.Helper() + + wh := &database.Webhook{ + UserID: uuid.New().String(), + Name: testWebhookName, + RetentionDays: retentionDays, + } + require.NoError( + t, + db.Omit(clause.Associations).Create(wh).Error, + ) + + return wh.ID +} + // eventChain is the set of row IDs seeded for a single event. type eventChain struct { eventID string @@ -256,12 +281,111 @@ func TestRetentionReaper_ReapsExpiredKeepsRecent(t *testing.T) { assertChainPresent(t, db, recent) } +// TestRetentionReaper_SkipsSentinelReapsFiniteInSameSweep covers the +// end-to-end retain-forever path: a webhook created the normal way with +// a requested retention of 0 lands on the RetentionForeverDays +// sentinel, and the reaper leaves its ancient events alone while still +// reaping a finite-retention webhook in the very same sweep. +func TestRetentionReaper_SkipsSentinelReapsFiniteInSameSweep( + t *testing.T, +) { + t.Parallel() + + env := setupRetentionTest(t) + + foreverID := createWebhookNormally(t, env.mainDB.DB(), 0) + + var stored database.Webhook + + require.NoError( + t, + env.mainDB.DB().Where("id = ?", foreverID). + First(&stored).Error, + ) + require.Equal( + t, + database.RetentionForeverDays, + stored.RetentionDays, + "a requested retention of 0 must persist as the sentinel", + ) + + finiteID := createWebhookNormally(t, env.mainDB.DB(), 30) + + foreverDB, err := env.mgr.GetDB(foreverID) + require.NoError(t, err) + + finiteDB, err := env.mgr.GetDB(finiteID) + require.NoError(t, err) + + ancient := time.Now().Add(-365 * 24 * time.Hour) + kept := seedEventChain(t, foreverDB, foreverID, ancient) + doomed := seedEventChain(t, finiteDB, finiteID, ancient) + + env.reaper.ExportSweep(context.Background()) + + assertChainPresent(t, foreverDB, kept) + assertChainGone(t, finiteDB, doomed) +} + +// TestRetentionReaper_HugeFiniteRetentionRetainsRecentEvents pins the +// overflow that made a large finite retention destroy everything. +// +// The cutoff is a time.Duration, an int64 nanosecond count. A day +// count above MaxFiniteRetentionDays multiplied out unclamped wraps +// negative, so subtracting it moves the cutoff into the far future, +// where "created_at < cutoff" matches every row: an event created a +// moment ago, and its delivery and delivery result, were all deleted +// on the first sweep. 200000 is inside that band and below the +// retain-forever sentinel, so it is treated as a finite policy and +// really does reach the arithmetic. +// +// The row is planted at the column level because such a value can no +// longer be submitted through the form; the point of the test is that +// a row from an older version, or a future call site, still cannot +// trigger the wipe. +func TestRetentionReaper_HugeFiniteRetentionRetainsRecentEvents( + t *testing.T, +) { + t.Parallel() + + env := setupRetentionTest(t) + + const overflowingRetentionDays = 200000 + + require.Greater( + t, + overflowingRetentionDays, + database.MaxFiniteRetentionDays, + "the test value must exceed what the cutoff can represent", + ) + require.Less( + t, + overflowingRetentionDays, + database.RetentionForeverDays, + "the test value must not be rescued by the forever skip", + ) + + webhookID := createWebhook( + t, env.mainDB.DB(), overflowingRetentionDays, + ) + + db, err := env.mgr.GetDB(webhookID) + require.NoError(t, err) + + fresh := seedEventChain(t, db, webhookID, time.Now()) + + env.reaper.ExportSweep(context.Background()) + + assertChainPresent(t, db, fresh) +} + func TestRetentionReaper_RetainsForeverWhenNonPositive(t *testing.T) { t.Parallel() env := setupRetentionTest(t) - // RetentionDays of zero means retain forever. + // A legacy row written before the sentinel existed still carries a + // literal 0; the <= 0 guard must keep honouring it. webhookID := createWebhook(t, env.mainDB.DB(), 0) db, err := env.mgr.GetDB(webhookID) diff --git a/internal/database/sqlite_mode_test.go b/internal/database/sqlite_mode_test.go new file mode 100644 index 0000000..3f75a14 --- /dev/null +++ b/internal/database/sqlite_mode_test.go @@ -0,0 +1,240 @@ +package database_test + +import ( + "context" + "io/fs" + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/fx/fxtest" + "sneak.berlin/go/webhooker/internal/config" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/globals" + "sneak.berlin/go/webhooker/internal/logger" +) + +// ownerOnly is the mode every SQLite file the service owns must have. +// Spelled out rather than referencing database.SQLiteFilePerm so the +// test fails if the constant itself is loosened. +const ownerOnly fs.FileMode = 0o600 + +// requireOwnerOnly asserts that path exists and is readable and +// writable by its owner and by nobody else. +func requireOwnerOnly(t *testing.T, path string) { + t.Helper() + + info, err := os.Stat(path) + require.NoError(t, err, "%s must exist", path) + assert.Equal( + t, + ownerOnly, + info.Mode().Perm(), + "%s holds credentials and must not be readable by "+ + "anyone but its owner", + path, + ) +} + +// requireDatabaseSetOwnerOnly asserts the mode of a database file and +// of both WAL sidecars. The sidecars carry the same rows as the +// database, so tightening only the main file fixes nothing. +func requireDatabaseSetOwnerOnly(t *testing.T, dbPath string) { + t.Helper() + + requireOwnerOnly(t, dbPath) + requireOwnerOnly(t, dbPath+"-wal") + requireOwnerOnly(t, dbPath+"-shm") +} + +// TestMainDatabaseFilesAreOwnerOnly covers the tier the defect was +// reported against: webhooker.db holds targets.config in plaintext — +// bearer tokens, API keys, Slack webhook URLs — and the session +// encryption key. +func TestMainDatabaseFilesAreOwnerOnly(t *testing.T) { + t.Parallel() + + lc := fxtest.NewLifecycle(t) + + l, err := logger.New(lc, logger.LoggerParams{ + Globals: &globals.Globals{ + Appname: testAppname, + Version: testVersion, + }, + }) + require.NoError(t, err) + + // A directory the application creates itself, not one t.TempDir + // made at 0700, so the mode below is the application's. + dataDir := filepath.Join(t.TempDir(), "data") + + db, err := database.New(lc, database.DatabaseParams{ + Config: &config.Config{DataDir: dataDir}, + Logger: l, + }) + require.NoError(t, err) + + ctx := context.Background() + require.NoError(t, lc.Start(ctx)) + + defer func() { require.NoError(t, lc.Stop(ctx)) }() + + // Write through the real model so the WAL is populated and both + // sidecars are on disk while the handle is open. + require.NoError(t, db.DB().Create(&database.Webhook{ + Name: testWebhookName, + }).Error) + + requireDatabaseSetOwnerOnly( + t, filepath.Join(dataDir, database.MainDBFileName), + ) + + // The data directory grants nothing to `other`. Asserted as a + // property rather than as an exact 0750, because MkdirAll applies + // the ambient umask: the exact mode is the developer's umask as + // much as the application's request, and pinning it would make + // `make check` pass or fail on where it is run. The group bits are + // deliberately left unasserted — deployments may rely on them. + info, err := os.Stat(dataDir) + require.NoError(t, err) + assert.Zero( + t, + info.Mode().Perm()&0o007, + "the data directory must not be world-accessible", + ) +} + +// TestPerWebhookEventDatabaseFilesAreOwnerOnly covers the events-*.db +// tier. These carry no credential canaries since +// https://git.eeqj.de/sneak/webhooker/issues/206, but they hold every +// received request body and header. +func TestPerWebhookEventDatabaseFilesAreOwnerOnly(t *testing.T) { + t.Parallel() + + mgr, lc := setupTestWebhookDBManager(t) + ctx := context.Background() + require.NoError(t, lc.Start(ctx)) + + defer func() { require.NoError(t, lc.Stop(ctx)) }() + + webhookID := uuid.New().String() + + db, err := mgr.GetDB(webhookID) + require.NoError(t, err) + + require.NoError(t, db.Create(&database.Event{ + WebhookID: webhookID, + EntrypointID: uuid.New().String(), + Method: http.MethodPost, + Body: "{}", + }).Error) + + requireDatabaseSetOwnerOnly(t, mgr.DBPath(webhookID)) +} + +// TestArchiveDatabaseFilesAreOwnerOnly covers the archive-*.db tier. +// internal/delivery builds that path and opens it through OpenSQLite, +// the same single open path exercised here, so the mode is settled for +// all three tiers in one place. +func TestArchiveDatabaseFilesAreOwnerOnly(t *testing.T) { + t.Parallel() + + ctx := context.Background() + path := filepath.Join( + t.TempDir(), "archive-"+uuid.New().String()+".db", + ) + + sqlDB, err := database.OpenSQLite(path, database.SQLiteModeCreate) + require.NoError(t, err) + + defer func() { require.NoError(t, sqlDB.Close()) }() + + _, err = sqlDB.ExecContext(ctx, "create table t (id integer)") + require.NoError(t, err) + + requireDatabaseSetOwnerOnly(t, path) +} + +// TestOpenSQLiteTightensFilesLeftWorldReadable is the upgrade case: a +// data directory an earlier build left at 0644, including a +// developer's own scratch directory, is fixed when it is opened rather +// than staying exposed until it is recreated. +func TestOpenSQLiteTightensFilesLeftWorldReadable(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + path := filepath.Join(dir, database.MainDBFileName) + + // A database and both sidecars as the pre-fix build left them. + for _, p := range []string{path, path + "-wal", path + "-shm"} { + require.NoError(t, os.WriteFile(p, nil, 0o644)) //nolint:gosec // the mode under test + } + + sqlDB, err := database.OpenSQLite(path, database.SQLiteModeCreate) + require.NoError(t, err) + + require.NoError(t, sqlDB.Close()) + + requireDatabaseSetOwnerOnly(t, path) +} + +// TestOpenSQLiteExistingModeDoesNotCreateTheFile guards the mechanism +// the fix uses: OpenSQLite now creates the database file itself, and +// must not do so for a caller that asked for an existing database. An +// empty file materialized here would turn a missing-database error +// into a silently empty one. +func TestOpenSQLiteExistingModeDoesNotCreateTheFile(t *testing.T) { + t.Parallel() + + ctx := context.Background() + path := filepath.Join(t.TempDir(), "absent.db") + + sqlDB, err := database.OpenSQLite(path, database.SQLiteModeExisting) + if err == nil { + // sql.Open is lazy: force the connection that fails. + require.Error(t, sqlDB.PingContext(ctx)) + require.NoError(t, sqlDB.Close()) + } + + _, statErr := os.Stat(path) + assert.ErrorIs(t, statErr, fs.ErrNotExist) +} + +// TestReopenAfterRestartKeepsFilesOwnerOnly is the restart case: a +// process that closed its files must be able to open them again at +// 0600, including through a gorm handle, and the sidecars must come +// back at 0600 too rather than at SQLite's own default. +func TestReopenAfterRestartKeepsFilesOwnerOnly(t *testing.T) { + t.Parallel() + + ctx := context.Background() + dir := t.TempDir() + path := filepath.Join(dir, database.MainDBFileName) + + first, err := database.OpenSQLite(path, database.SQLiteModeCreate) + require.NoError(t, err) + + _, err = first.ExecContext(ctx, "create table t (id integer)") + require.NoError(t, err) + require.NoError(t, first.Close()) + + second, err := database.OpenSQLite(path, database.SQLiteModeCreate) + require.NoError(t, err) + + defer func() { require.NoError(t, second.Close()) }() + + _, err = second.ExecContext(ctx, "insert into t (id) values (1)") + require.NoError(t, err) + + requireDatabaseSetOwnerOnly(t, path) + + var got int + + require.NoError(t, + second.QueryRowContext(ctx, "select id from t").Scan(&got)) + assert.Equal(t, 1, got) +} diff --git a/internal/database/sqlite_open.go b/internal/database/sqlite_open.go new file mode 100644 index 0000000..9003efe --- /dev/null +++ b/internal/database/sqlite_open.go @@ -0,0 +1,252 @@ +package database + +import ( + "database/sql" + "errors" + "fmt" + "io/fs" + "net/url" + "os" + "time" + + _ "modernc.org/sqlite" // Pure Go SQLite driver +) + +// Every SQLite file this service opens — the main database, the +// per-webhook event databases, and the archive databases — is opened +// through OpenSQLite, so the durability settings below are properties +// of the service rather than of one call site. +// +// modernc.org/sqlite installs no busy handler and issues no pragmas of +// its own: it executes only the pragmas named in explicit `_pragma=` +// DSN parameters, and gorm.io/driver/sqlite adds none when it is +// handed an existing *sql.DB. Every setting therefore has to be +// spelled out here or it is simply not in effect. +// SQLite URI open modes. +const ( + // SQLiteModeCreate creates the database file when it is missing. + SQLiteModeCreate = "rwc" + + // SQLiteModeExisting requires the file to exist already. + SQLiteModeExisting = "rw" +) + +const ( + // SQLiteBusyTimeout is how long SQLite retries a lock conflict + // before returning SQLITE_BUSY. + // + // Under WAL a reader never blocks a writer, so the only conflict + // left is writer against writer: this process's delivery workers + // against each other, or against another process holding the write + // lock. Those clear in milliseconds. Ten seconds is far above that + // and still well inside the receiver's request budget, so an + // inbound webhook waits rather than being rejected with a 500. + SQLiteBusyTimeout = 10 * time.Second + + // sqliteMaxOpenConns bounds the connection pool for one database + // file. + // + // The pool needs a bound at all because database/sql cannot detect + // a connection left mid-transaction: modernc.org/sqlite implements + // neither driver.Validator nor driver.SessionResetter, so a + // connection whose COMMIT failed is returned to the pool with its + // transaction still open and handed out again indefinitely. That is + // what turned four `database is locked` errors into 593 + // `cannot start a transaction within a transaction` in + // https://git.eeqj.de/sneak/webhooker/issues/256. + // + // Four is above the one writer SQLite allows at a time, so reads + // still proceed while a write is in flight, and low enough that + // contention is resolved by the busy handler rather than by piling + // up connections against a lock only one of them can hold. + sqliteMaxOpenConns = 4 + + // sqliteMaxIdleConns keeps the pool warm without holding every + // connection open through an idle period. + sqliteMaxIdleConns = 2 + + // sqliteConnMaxLifetime and sqliteConnMaxIdleTime retire pooled + // connections on a schedule. With _txlock=immediate a failed + // COMMIT should no longer be reachable, but these bound the damage + // if one happens anyway: a poisoned connection is closed and + // replaced within the lifetime instead of wedging the file until + // the process restarts. + sqliteConnMaxLifetime = 5 * time.Minute + sqliteConnMaxIdleTime = time.Minute +) + +// SQLiteFilePerm is the mode every SQLite file this service owns is +// created with and held at: owner read/write, nothing for group or +// other. +// +// These files hold credentials in plaintext. The main database stores +// `targets.config` — bearer tokens, API keys, Slack webhook URLs — and +// the session encryption key. SQLite left to itself creates them 0644 +// (see reserveSQLiteFile), which made the 0750 data directory the only +// barrier; a bind-mounted directory supplied at 0755 removes it and +// every local user on the host can read every stored credential. +// +// This is a file-mode fix and not encryption at rest. An unattended +// process needs a key it can read without a human, so the key lands +// beside the data and an attacker who can read the database can read +// it too. See https://git.eeqj.de/sneak/webhooker/issues/212. +const SQLiteFilePerm fs.FileMode = 0o600 + +// reserveSQLiteFile puts path at SQLiteFilePerm before the driver ever +// touches it, and tightens any sidecar already on disk. +// +// The mode has to be settled here rather than by a chmod after opening, +// because SQLite picks it: robust_open substitutes +// SQLITE_DEFAULT_FILE_PERMISSIONS (0644) whenever it is handed mode 0, +// and findCreateFileMode yields 0 for a main database opened by URI +// with no `modeof` parameter. A chmod afterwards would leave a window +// in which the credentials are on disk world-readable. +// +// Creating the file ourselves also settles the sidecars, which is the +// half that could quietly not work. SQLite does not create those at a +// mode we choose — it derives both from the main database file: +// `-wal` through findCreateFileMode, which stats the path with the +// suffix stripped, and `-shm` in unixOpenSharedMemory from an fstat of +// the already-open database descriptor. A main file at 0600 therefore +// produces sidecars at 0600. A zero-length file is a valid empty +// database, so reserving it changes nothing else. +// +// create says whether the caller is opening in a mode that may create +// the database. When it is false a missing file is left missing, so +// SQLite still reports the absence rather than this function +// materializing an empty database the caller asked not to create. +// +// Chmod of a file that already exists is what tightens a data +// directory an earlier build left at 0644 — including a developer's +// own scratch directory — without any migration machinery. +func reserveSQLiteFile(path string, create bool) error { + if create { + // gosec G304: the path is the database file the caller asked + // to open, and the driver is about to open the same path + // anyway. Creating it here is what fixes its mode. + f, err := os.OpenFile( //nolint:gosec // see above + path, os.O_RDWR|os.O_CREATE, SQLiteFilePerm, + ) + if err != nil { + return fmt.Errorf("creating %s: %w", path, err) + } + + err = f.Close() + if err != nil { + return fmt.Errorf("closing %s: %w", path, err) + } + } + + // O_CREATE leaves an existing file's mode alone, and umask can only + // have narrowed a new one. Chmod settles both cases at exactly + // SQLiteFilePerm. + for _, p := range append( + []string{path}, sqliteSidecarPaths(path)..., + ) { + err := os.Chmod(p, SQLiteFilePerm) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("securing %s: %w", p, err) + } + } + + return nil +} + +// sqliteSidecarPaths returns the files SQLite maintains beside a +// database under WAL. They carry the same rows as the database itself, +// so a fix that tightens only the main file has fixed nothing. +func sqliteSidecarPaths(path string) []string { + return []string{path + "-wal", path + "-shm"} +} + +// SQLiteDSN builds the connection string for one database file. +// +// mode is the SQLite URI open mode: "rwc" to create the file when it +// is missing, "rw" to require that it already exists. +// +// Three settings carry the fix for +// https://git.eeqj.de/sneak/webhooker/issues/256 and none of them is +// optional: +// +// - journal_mode=WAL, so a reader — an operator running +// `sqlite3 .dump` over their own data — takes a snapshot +// instead of blocking every writer behind it. +// +// - busy_timeout, so a writer that does meet a lock waits for it. +// Without one SQLite gives up immediately; nothing above it +// retries. +// +// - _txlock=immediate, so every transaction takes the write lock at +// BEGIN. A deferred transaction acquires it lazily on its first +// write, and that upgrade returns SQLITE_BUSY *without* consulting +// the busy handler, because SQLite cannot block a transaction that +// may already hold a read snapshot. Such a COMMIT then fails while +// the transaction stays open on the connection. A busy timeout +// alone does not prevent this; BEGIN IMMEDIATE does, by putting +// the wait somewhere the handler applies. +// +// Note what is absent: `cache=shared`. Under a shared cache an +// in-process conflict is reported as SQLITE_LOCKED rather than +// SQLITE_BUSY, and the busy handler does not retry SQLITE_LOCKED — so +// leaving it in would have defeated the busy timeout for exactly the +// contention this service generates. Dropping it is part of the fix, +// not housekeeping. +// +// synchronous is deliberately left at SQLite's default of FULL: this +// is a webhook receiver whose one promise is that an event it answered +// 200 for is durable. +// The order of the _pragma parameters is load-bearing. +// modernc.org/sqlite executes them in the order they appear, on every +// new connection, before the connection is handed to the pool. Setting +// journal_mode first means that pragma itself runs with no busy +// handler installed: the pool opens connections lazily, so the moment +// a new one is created is a moment the database is under load, and +// PRAGMA journal_mode takes a lock. It would fail immediately with +// SQLITE_BUSY and fail the query that caused the connection to be +// opened. busy_timeout is therefore set first, so every pragma after +// it — and the whole life of the connection — is covered. +func SQLiteDSN(path, mode string) string { + q := url.Values{} + q.Set("mode", mode) + q.Set("_txlock", "immediate") + q.Add( + "_pragma", + fmt.Sprintf( + "busy_timeout(%d)", + SQLiteBusyTimeout.Milliseconds(), + ), + ) + q.Add("_pragma", "journal_mode(WAL)") + + return "file:" + path + "?" + q.Encode() +} + +// OpenSQLite opens the SQLite file at path with the service's +// durability settings and pool bounds applied. mode is the SQLite URI +// open mode ("rwc" or "rw"). +// +// The file and its WAL sidecars are settled at SQLiteFilePerm before +// the driver sees the path; see reserveSQLiteFile. +// +// The handle is returned rather than a *gorm.DB because the callers +// wrap it in gorm themselves with their own logger. +func OpenSQLite(path, mode string) (*sql.DB, error) { + err := reserveSQLiteFile(path, mode == SQLiteModeCreate) + if err != nil { + return nil, err + } + + sqlDB, err := sql.Open("sqlite", SQLiteDSN(path, mode)) + if err != nil { + return nil, fmt.Errorf( + "opening sqlite database %s: %w", path, err, + ) + } + + sqlDB.SetMaxOpenConns(sqliteMaxOpenConns) + sqlDB.SetMaxIdleConns(sqliteMaxIdleConns) + sqlDB.SetConnMaxLifetime(sqliteConnMaxLifetime) + sqlDB.SetConnMaxIdleTime(sqliteConnMaxIdleTime) + + return sqlDB, nil +} diff --git a/internal/database/sqlite_open_test.go b/internal/database/sqlite_open_test.go new file mode 100644 index 0000000..448507c --- /dev/null +++ b/internal/database/sqlite_open_test.go @@ -0,0 +1,178 @@ +package database_test + +import ( + "context" + "path/filepath" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + "sneak.berlin/go/webhooker/internal/database" +) + +// livePragma reads a pragma off a live handle. Reading the DSN back +// would prove only that the string was built; these tests assert that +// SQLite actually applied it. +func livePragma(t *testing.T, db *gorm.DB, name string) string { + t.Helper() + + var v string + + row := db.Raw("pragma " + name).Row() + require.NoError(t, row.Scan(&v)) + + return v +} + +func TestSQLiteDSNCarriesTheDurabilitySettings(t *testing.T) { + t.Parallel() + + dsn := database.SQLiteDSN( + "/var/lib/webhooker/webhooker.db", + database.SQLiteModeCreate, + ) + + assert.Contains(t, dsn, "journal_mode%28WAL%29") + assert.Contains(t, dsn, "busy_timeout%2810000%29") + assert.Contains(t, dsn, "_txlock=immediate") + assert.Contains(t, dsn, "mode=rwc") + + // busy_timeout must come first. The driver runs these in order on + // every new connection, and PRAGMA journal_mode takes a lock — a + // connection opened while the database is busy would fail on that + // pragma, with no busy handler yet installed to wait it out. + assert.Less( + t, + strings.Index(dsn, "busy_timeout"), + strings.Index(dsn, "journal_mode"), + "busy_timeout must be applied before journal_mode", + ) + + // cache=shared turns an in-process conflict into SQLITE_LOCKED, + // which the busy handler does not retry. It must never come back. + // See https://git.eeqj.de/sneak/webhooker/issues/256. + assert.NotContains(t, strings.ToLower(dsn), "cache=shared") +} + +// TestPerWebhookDBAppliesPragmasOnALiveHandle is the check the issue +// asks for by name: the settings are confirmed by querying the running +// database, not by inspecting the connection string. +func TestPerWebhookDBAppliesPragmasOnALiveHandle(t *testing.T) { + t.Parallel() + + mgr, lc := setupTestWebhookDBManager(t) + ctx := context.Background() + require.NoError(t, lc.Start(ctx)) + + defer func() { require.NoError(t, lc.Stop(ctx)) }() + + webhookID := uuid.New().String() + + db, err := mgr.GetDB(webhookID) + require.NoError(t, err) + + assert.Equal( + t, "wal", + strings.ToLower(livePragma(t, db, "journal_mode")), + ) + assert.Equal( + t, "10000", livePragma(t, db, "busy_timeout"), + ) +} + +func TestMainDBAppliesPragmasOnALiveHandle(t *testing.T) { + t.Parallel() + + ctx := context.Background() + dir := t.TempDir() + + sqlDB, err := database.OpenSQLite( + filepath.Join(dir, database.MainDBFileName), + database.SQLiteModeCreate, + ) + require.NoError(t, err) + + defer func() { require.NoError(t, sqlDB.Close()) }() + + var journal string + + require.NoError(t, sqlDB. + QueryRowContext(ctx, "pragma journal_mode"). + Scan(&journal)) + assert.Equal(t, "wal", strings.ToLower(journal)) + + var busy string + + require.NoError(t, sqlDB. + QueryRowContext(ctx, "pragma busy_timeout"). + Scan(&busy)) + assert.Equal(t, "10000", busy) +} + +// TestConcurrentReaderDoesNotBlockWrites is the unit-scale form of the +// reproduction in +// https://git.eeqj.de/sneak/webhooker/issues/256: an operator's +// long-held read of their own data used to make every concurrent write +// fail. Under WAL the reader takes a snapshot and the writes proceed. +func TestConcurrentReaderDoesNotBlockWrites(t *testing.T) { + t.Parallel() + + mgr, lc := setupTestWebhookDBManager(t) + ctx := context.Background() + require.NoError(t, lc.Start(ctx)) + + defer func() { require.NoError(t, lc.Stop(ctx)) }() + + webhookID := uuid.New().String() + + db, err := mgr.GetDB(webhookID) + require.NoError(t, err) + + // A second handle on the same file, holding a read transaction + // open across every write below — what `sqlite3 .dump` is. + readerSQL, err := database.OpenSQLite( + mgr.DBPath(webhookID), database.SQLiteModeExisting, + ) + require.NoError(t, err) + + defer func() { require.NoError(t, readerSQL.Close()) }() + + readerConn, err := readerSQL.Conn(ctx) + require.NoError(t, err) + + defer func() { require.NoError(t, readerConn.Close()) }() + + _, err = readerConn.ExecContext(ctx, "begin deferred") + require.NoError(t, err) + + _, err = readerConn.ExecContext( + ctx, "select count(*) from events", + ) + require.NoError(t, err) + + for range 25 { + err = db.Transaction(func(tx *gorm.DB) error { + return tx.Create(&database.Event{ + WebhookID: webhookID, + EntrypointID: uuid.New().String(), + Method: "POST", + Body: "{}", + }).Error + }) + require.NoError(t, err) + } + + _, err = readerConn.ExecContext(ctx, "commit") + require.NoError(t, err) + + var count int64 + + require.NoError( + t, + db.Model(&database.Event{}).Count(&count).Error, + ) + assert.Equal(t, int64(25), count) +} diff --git a/internal/database/testing.go b/internal/database/testing.go index 6327eac..72c8816 100644 --- a/internal/database/testing.go +++ b/internal/database/testing.go @@ -24,11 +24,24 @@ func NewTestDatabase(db *gorm.DB) *Database { // NewTestWebhookDBManager creates a WebhookDBManager backed by the given // data directory. Intended for use in tests without the fx lifecycle. func NewTestWebhookDBManager(dataDir string) *WebhookDBManager { - return &WebhookDBManager{ - dataDir: dataDir, - log: slog.New(slog.NewTextHandler( + return NewTestWebhookDBManagerWithLogger( + dataDir, + slog.New(slog.NewTextHandler( os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug}, )), + ) +} + +// NewTestWebhookDBManagerWithLogger is NewTestWebhookDBManager with the +// logger supplied by the caller. The per-webhook databases this manager +// opens hand that logger to gormlog, so a test that needs to see the SQL +// the service emits can capture it. +func NewTestWebhookDBManagerWithLogger( + dataDir string, log *slog.Logger, +) *WebhookDBManager { + return &WebhookDBManager{ + dataDir: dataDir, + log: log, } } diff --git a/internal/database/webhook_db_manager.go b/internal/database/webhook_db_manager.go index a1f694d..81ca427 100644 --- a/internal/database/webhook_db_manager.go +++ b/internal/database/webhook_db_manager.go @@ -2,7 +2,6 @@ package database import ( "context" - "database/sql" "errors" "fmt" "log/slog" @@ -14,6 +13,7 @@ import ( "gorm.io/driver/sqlite" "gorm.io/gorm" "sneak.berlin/go/webhooker/internal/config" + "sneak.berlin/go/webhooker/internal/gormlog" "sneak.berlin/go/webhooker/internal/logger" ) @@ -233,12 +233,11 @@ func (m *WebhookDBManager) openDB( webhookID string, ) (*gorm.DB, error) { path := m.dbPath(webhookID) - dbURL := fmt.Sprintf( - "file:%s?cache=shared&mode=rwc", - path, - ) - sqlDB, err := sql.Open("sqlite", dbURL) + // See sqlite_open.go: WAL, a busy timeout, immediate-transaction + // locking, and a bounded pool, all of which this file needs most — + // it is the one every delivery worker writes to concurrently. + sqlDB, err := OpenSQLite(path, SQLiteModeCreate) if err != nil { return nil, fmt.Errorf( "opening webhook database %s: %w", @@ -248,7 +247,10 @@ func (m *WebhookDBManager) openDB( db, err := gorm.Open(sqlite.Dialector{ Conn: sqlDB, - }, &gorm.Config{}) + }, &gorm.Config{ + // Never leave this at GORM's default. See internal/gormlog. + Logger: gormlog.New(m.log), + }) if err != nil { _ = sqlDB.Close() @@ -258,6 +260,25 @@ func (m *WebhookDBManager) openDB( ) } + // Keep main-database rows out of this file. See + // event_db_isolation.go. + err = omitAssociations(db) + if err != nil { + _ = sqlDB.Close() + + return nil, fmt.Errorf( + "guarding webhook database %s: %w", + webhookID, err, + ) + } + + err = purgeTargetRows(db, m.log, webhookID) + if err != nil { + _ = sqlDB.Close() + + return nil, err + } + // Run migrations for event-tier models only err = db.AutoMigrate( &Event{}, &Delivery{}, &DeliveryResult{}, diff --git a/internal/database/webhook_db_manager_test.go b/internal/database/webhook_db_manager_test.go index 282d890..771f9e4 100644 --- a/internal/database/webhook_db_manager_test.go +++ b/internal/database/webhook_db_manager_test.go @@ -339,7 +339,12 @@ func TestWebhookDBManager_MultipleWebhooks(t *testing.T) { var events []database.Event require.NoError(t, db2.Find(&events).Error) - assert.Len(t, events, 1) + + // require, not assert: this is exactly the regression the test + // guards, so the empty slice is the expected failure, and a + // non-fatal length check would index into it on the next line and + // panic the whole package test binary instead of failing here. + require.Len(t, events, 1) assert.Equal(t, "PUT", events[0].Method) } diff --git a/internal/datadir/lock.go b/internal/datadir/lock.go new file mode 100644 index 0000000..a50929c --- /dev/null +++ b/internal/datadir/lock.go @@ -0,0 +1,126 @@ +// Package datadir guards exclusive access to the directory holding +// every SQLite database webhooker writes. +// +// Two processes sharing a DATA_DIR each open the same per-webhook +// event databases and each run delivery recovery over the same rows, +// so every pending delivery goes out twice. SQLite's own locking does +// not prevent that: both writers are serialised correctly and both +// deliver. The only thing that prevents it is refusing to be the +// second process. +// +// The lock lives here rather than in the server's fx graph so that any +// entry point which touches DATA_DIR — the server, or a CLI +// subcommand that must not operate on a live deployment's data — takes +// it the same way. +package datadir + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/gofrs/flock" +) + +// LockFileName is the advisory lock file created inside DATA_DIR. Its +// contents are never read: the lock is the flock(2) held on the open +// descriptor, not the file's existence, so a leftover file from a +// process that was killed with SIGKILL blocks nothing. +const LockFileName = "webhooker.lock" + +// dirPerm is the mode Acquire creates DATA_DIR with. It matches what +// internal/database uses, since whichever runs first creates it. +const dirPerm = 0o750 + +// ErrLocked reports that another live process holds the data +// directory. Callers that need to know whether a deployment is running +// — rather than merely failing to start — test for this with +// errors.Is. +var ErrLocked = errors.New( + "data directory is already in use by another instance", +) + +// ErrNoDir reports that Acquire was given an empty directory. +var ErrNoDir = errors.New("no data directory given") + +// Lock is a held exclusive advisory lock on a data directory. It is +// valid only while the process that took it lives: the kernel drops it +// when the descriptor closes, whether that is Release, a normal exit, +// or a SIGKILL. +type Lock struct { + dir string + file *flock.Flock +} + +// Acquire takes the exclusive advisory lock on dir, creating dir if it +// does not exist. It never waits: if another process holds the lock it +// returns an error wrapping ErrLocked and naming dir. +// +// The returned Lock must be held for as long as the caller intends to +// use dir. +func Acquire(dir string) (*Lock, error) { + if dir == "" { + return nil, ErrNoDir + } + + err := os.MkdirAll(dir, dirPerm) + if err != nil { + return nil, fmt.Errorf( + "creating data directory %s: %w", dir, err, + ) + } + + path := filepath.Join(dir, LockFileName) + fl := flock.New(path) + + held, err := fl.TryLock() + if err != nil { + return nil, fmt.Errorf( + "locking data directory %s: %w", dir, err, + ) + } + + if !held { + // A no-op on flock v0.13.0, which closes its own descriptor on + // a failed TryLock; kept so no version can leak one. + _ = fl.Close() + + return nil, fmt.Errorf( + "%w: %s (%s). Only one webhooker may use a data "+ + "directory: two both run delivery recovery over the "+ + "same rows and both deliver", + ErrLocked, dir, path, + ) + } + + return &Lock{dir: dir, file: fl}, nil +} + +// Dir returns the locked directory. +func (l *Lock) Dir() string { + return l.dir +} + +// Path returns the lock file backing the lock. +func (l *Lock) Path() string { + return l.file.Path() +} + +// Release drops the lock and closes the descriptor. It is safe to call +// more than once. +// +// The lock file is deliberately left on disk. Unlinking it would let +// the next process create and lock a fresh inode while a third still +// holds the old one, which is the one outcome this package exists to +// prevent. +func (l *Lock) Release() error { + err := l.file.Unlock() + if err != nil { + return fmt.Errorf( + "releasing lock on data directory %s: %w", l.dir, err, + ) + } + + return nil +} diff --git a/internal/datadir/lock_test.go b/internal/datadir/lock_test.go new file mode 100644 index 0000000..13f4216 --- /dev/null +++ b/internal/datadir/lock_test.go @@ -0,0 +1,250 @@ +package datadir_test + +import ( + "bufio" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/datadir" +) + +// holderEnv names the directory the re-executed test binary should +// lock and hold. When it is unset the child test does nothing, so an +// ordinary run is unaffected. +const holderEnv = "WEBHOOKER_DATADIR_LOCK_HOLDER" + +// holderReadyPrefix labels the child's one-line report that it holds +// the lock, so the parent can find it among the testing package's own +// output on the same descriptor. +const holderReadyPrefix = "DATADIR-LOCK-HELD " + +// holderReadyTimeout bounds the wait for the child to take the lock. +// It only has to cover process start on a loaded shared host. +const holderReadyTimeout = 60 * time.Second + +// holderHold is how long the child keeps the lock if nothing kills it. +// A sleep rather than a bare block, so the runtime's deadlock detector +// has a pending timer and the child cannot outlive a killed test run +// by more than this. +const holderHold = 10 * time.Minute + +// TestLockHolder is the child half of the two-process tests below. It +// takes the lock on the directory named by holderEnv, reports the lock +// file on standard output, and then holds it until it is killed. +func TestLockHolder(t *testing.T) { + t.Parallel() + + dir := os.Getenv(holderEnv) + if dir == "" { + return + } + + lock, err := datadir.Acquire(dir) + require.NoError(t, err) + + // Written to the descriptor directly: the parent reads fd 1, not + // the testing package's buffered report. + _, err = fmt.Fprintf( + os.Stdout, "%s%s\n", holderReadyPrefix, lock.Path(), + ) + require.NoError(t, err) + + time.Sleep(holderHold) +} + +// startHolder re-executes this test binary as a separate process that +// takes and holds the lock on dir, and returns once that process +// actually holds it. The child is killed when the test ends. +func startHolder(t *testing.T, dir string) *exec.Cmd { + t.Helper() + + //nolint:gosec // Re-executing this test binary, with a fixed arg. + cmd := exec.CommandContext( + t.Context(), os.Args[0], "-test.run", "^TestLockHolder$", + ) + + cmd.Env = append(os.Environ(), holderEnv+"="+dir) + cmd.Stderr = os.Stderr + + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + require.NoError(t, cmd.Start()) + + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + + ready := make(chan string, 1) + + go func() { + scanner := bufio.NewScanner(stdout) + for scanner.Scan() { + after, found := strings.CutPrefix( + scanner.Text(), holderReadyPrefix, + ) + if found { + ready <- after + + break + } + } + + close(ready) + + // Keep draining so the child never blocks on a full pipe. + _, _ = io.Copy(io.Discard, stdout) + }() + + select { + case path, ok := <-ready: + require.True( + t, ok, "holder exited without taking the lock", + ) + require.Equal(t, filepath.Join(dir, datadir.LockFileName), path) + case <-time.After(holderReadyTimeout): + t.Fatal("timed out waiting for the holder to take the lock") + } + + return cmd +} + +// TestSecondInstanceRefused is the regression test for the duplicate +// delivery this package exists to prevent: a real second process +// pointed at a data directory a live process already holds must be +// refused, with an error that names the directory. +func TestSecondInstanceRefused(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + startHolder(t, dir) + + lock, err := datadir.Acquire(dir) + + require.Error(t, err, "the second instance took the lock too") + require.Nil(t, lock) + require.ErrorIs( + t, err, datadir.ErrLocked, + "the refusal must be distinguishable from any other failure", + ) + assert.Contains( + t, err.Error(), dir, + "the refusal must name the directory it is about", + ) +} + +// TestRestartAfterHardKill is the other half of the regression: a +// process killed with SIGKILL runs no cleanup and leaves its lock file +// behind, and the next start must not be blocked by it. This is what a +// pidfile would get wrong; the kernel drops a flock when the +// descriptor closes, however the process died. +func TestRestartAfterHardKill(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + holder := startHolder(t, dir) + + require.NoError(t, holder.Process.Kill()) + + // Wait for the kill to have actually happened. Re-acquiring while + // the corpse still holds a descriptor would be a race, and would + // make this test pass or fail on scheduling. + _ = holder.Wait() + + require.FileExists( + t, filepath.Join(dir, datadir.LockFileName), + "the stale lock file is what must not block the restart", + ) + + lock, err := datadir.Acquire(dir) + require.NoError( + t, err, "a hard-killed instance must not block the next start", + ) + + require.NoError(t, lock.Release()) +} + +// TestSecondFdInSameProcessRefused pins the flock(2) property the +// tests in cmd/webhooker rely on: descriptors are locked +// independently, so a second acquisition is denied even when it comes +// from the process that already holds the lock. +func TestSecondFdInSameProcessRefused(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + first, err := datadir.Acquire(dir) + require.NoError(t, err) + + defer func() { _ = first.Release() }() + + _, err = datadir.Acquire(dir) + require.ErrorIs(t, err, datadir.ErrLocked) +} + +// TestReleaseAllowsReacquire covers the clean-shutdown path: the lock +// is released on exit, so a restart is not blocked by the previous +// run. +func TestReleaseAllowsReacquire(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + first, err := datadir.Acquire(dir) + require.NoError(t, err) + require.NoError(t, first.Release()) + + second, err := datadir.Acquire(dir) + require.NoError(t, err) + require.NoError(t, second.Release()) +} + +// TestAcquireCreatesDataDir covers a first start against a DATA_DIR +// that does not exist yet, which is the normal case for a fresh +// deployment: the lock is taken before anything else creates it. +func TestAcquireCreatesDataDir(t *testing.T) { + t.Parallel() + + dir := filepath.Join(t.TempDir(), "nested", "data") + + lock, err := datadir.Acquire(dir) + require.NoError(t, err) + + defer func() { _ = lock.Release() }() + + assert.Equal(t, dir, lock.Dir()) + assert.FileExists(t, filepath.Join(dir, datadir.LockFileName)) +} + +// TestAcquireEmptyDir rejects an empty directory rather than locking +// the process's working directory. +func TestAcquireEmptyDir(t *testing.T) { + t.Parallel() + + _, err := datadir.Acquire("") + require.ErrorIs(t, err, datadir.ErrNoDir) +} + +// TestAcquireUnusableDir reports an unusable DATA_DIR clearly, naming +// it, instead of failing later and deeper. +func TestAcquireUnusableDir(t *testing.T) { + t.Parallel() + + file := filepath.Join(t.TempDir(), "not-a-directory") + require.NoError(t, os.WriteFile(file, nil, 0o600)) + + _, err := datadir.Acquire(file) + + require.Error(t, err) + assert.Contains(t, err.Error(), file) +} diff --git a/internal/delivery/archive_sweeper.go b/internal/delivery/archive_sweeper.go new file mode 100644 index 0000000..2cd2934 --- /dev/null +++ b/internal/delivery/archive_sweeper.go @@ -0,0 +1,239 @@ +package delivery + +import ( + "context" + "errors" + "log/slog" + "sync" + "time" + + "go.uber.org/fx" + "sneak.berlin/go/webhooker/internal/config" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/lifecycle" + "sneak.berlin/go/webhooker/internal/logger" +) + +// ArchiveSweeperParams holds the fx dependencies for the +// ArchiveSweeper. +type ArchiveSweeperParams struct { + fx.In + + Config *config.Config + Database *database.Database + Engine *Engine + Logger *logger.Logger +} + +// ArchiveSweeper periodically prunes expired rows from +// per-webhook archive databases whose database target carries a +// positive expiry. +// +// Without it, pruning happens only when an archive is +// (re)opened, and archives are only ever reopened by writes: an +// archive belonging to a webhook that has stopped receiving +// events would keep its expired rows forever. The sweep closes +// that gap without changing anything for archives whose expiry +// is unset or "never". +// +// It reuses Config.RetentionSweepInterval rather than +// introducing a second interval: this is a retention sweep with +// the same semantics as the event retention reaper. +type ArchiveSweeper struct { + db *database.Database + eng *Engine + log *slog.Logger + interval time.Duration + cancel context.CancelFunc + wg sync.WaitGroup +} + +// NewArchiveSweeper creates the archive sweeper and registers +// its fx lifecycle hooks. The background sweep loop starts on +// OnStart and stops cleanly on OnStop via context cancellation. +func NewArchiveSweeper( + lc fx.Lifecycle, + params ArchiveSweeperParams, +) *ArchiveSweeper { + s := &ArchiveSweeper{ + db: params.Database, + eng: params.Engine, + log: params.Logger.Get(), + interval: params.Config.RetentionSweepInterval, + } + + s.registerHooks(lc) + + return s +} + +// registerHooks wires the sweeper's start and stop into the fx +// lifecycle. The start hook's context is deliberately ignored +// (see start for why the background loop must not inherit it); +// the stop hook's context is honoured (see stop). +func (s *ArchiveSweeper) registerHooks(lc fx.Lifecycle) { + lc.Append(fx.Hook{ + //nolint:contextcheck // Not passing the hook context is + // the point: see start. + OnStart: func(_ context.Context) error { + s.start() + + return nil + }, + OnStop: func(ctx context.Context) error { + return s.stop(ctx) + }, + }) +} + +// start launches the background sweep loop. +// +// The loop's context is derived from context.Background(), NOT +// from the fx OnStart hook context. The hook context carries +// fx's start timeout (15s by default), so a loop derived from it +// is cancelled 15 seconds after the application starts — long +// before the first tick under the default one-hour sweep +// interval, leaving a sweeper that never sweeps. A long-lived +// goroutine must outlive the startup phase, so its lifetime is +// bounded by OnStop instead: stop cancels this context and waits +// on the WaitGroup. +func (s *ArchiveSweeper) start() { + ctx, cancel := context.WithCancel(context.Background()) + s.cancel = cancel + + s.wg.Add(1) + + go s.run(ctx) + + s.log.Info( + "archive sweeper started", + "interval", s.interval.String(), + ) +} + +// stop cancels the sweep loop's context and waits for it to +// exit, bounded by the stop hook's context: a prune wedged on a +// locked archive must not hang the process past fx's stop +// timeout. +func (s *ArchiveSweeper) stop(ctx context.Context) error { + s.log.Info("archive sweeper stopping") + + if s.cancel != nil { + s.cancel() + } + + err := lifecycle.WaitForShutdown( + ctx, s.log, "archive sweeper", &s.wg, + ) + if err != nil { + return err + } + + s.log.Info("archive sweeper stopped") + + return nil +} + +func (s *ArchiveSweeper) run(ctx context.Context) { + defer s.wg.Done() + + ticker := time.NewTicker(s.interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.sweep(ctx) + } + } +} + +// sweep prunes every archive whose database target declares a +// positive expiry. Targets belonging to a deleted webhook are +// soft-deleted along with it, so GORM's default scope already +// excludes them. +// +// A failure for one webhook is logged and the sweep continues, +// matching how the write path already treats a prune error as +// non-fatal. +func (s *ArchiveSweeper) sweep(ctx context.Context) { + var targets []database.Target + + err := s.db.DB(). + Model(&database.Target{}). + Where("type = ?", database.TargetTypeDatabase). + Find(&targets).Error + if err != nil { + s.log.Error( + "archive sweep: failed to list database targets", + "error", err, + ) + + return + } + + for i := range targets { + select { + case <-ctx.Done(): + return + default: + } + + s.sweepTarget(&targets[i]) + } +} + +// sweepTarget prunes the archive of a single database target. +// A missing, empty, or "never" expiry parses as a zero duration +// and is skipped entirely, so those archives keep exactly the +// behaviour they had before the sweep existed. +func (s *ArchiveSweeper) sweepTarget(target *database.Target) { + expiry, err := parseArchiveExpiry(target.Config) + if err != nil { + s.log.Error( + "archive sweep: invalid database target config", + "webhook_id", target.WebhookID, + "target_id", target.ID, + "error", err, + ) + + return + } + + if expiry <= 0 { + return + } + + if s.eng == nil || s.eng.dbTarget == nil { + return + } + + err = s.eng.dbTarget.sweepWebhook(target.WebhookID, expiry) + if err == nil { + return + } + + // A writer evicted underneath the sweep means the operator + // deleted the webhook (or its last database target) while the + // sweep was walking the target list. That is an ordinary + // interleaving, not a failure, so it must not produce an + // error line. + if errors.Is(err, errArchiveWriterEvicted) { + s.log.Debug( + "archive sweep: writer evicted mid-sweep", + "webhook_id", target.WebhookID, + "target_id", target.ID, + ) + + return + } + + s.log.Error( + "archive sweep: failed to prune archive", + "webhook_id", target.WebhookID, + "target_id", target.ID, + "error", err, + ) +} diff --git a/internal/delivery/archive_sweeper_test.go b/internal/delivery/archive_sweeper_test.go new file mode 100644 index 0000000..cdfab09 --- /dev/null +++ b/internal/delivery/archive_sweeper_test.go @@ -0,0 +1,947 @@ +package delivery_test + +import ( + "context" + "database/sql" + "fmt" + "net/http" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "gorm.io/gorm/clause" + _ "modernc.org/sqlite" // Pure Go SQLite driver. + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" +) + +const ( + // sweepRowOld and sweepRowNew are the event ids + // seedArchiveRows assigns to the first and second seeded + // rows. + sweepRowOld = "ev-0" + sweepRowNew = "ev-1" + + // sweepConcurrentWrites is how many deliveries the + // concurrent write-plus-sweep test races against the sweep. + sweepConcurrentWrites = 20 +) + +// sweeperEnv bundles the pieces an archive sweep test drives: +// a main configuration database holding webhooks and targets, a +// delivery engine owning the archive writer registry, and the +// data directory the archive files live in. +type sweeperEnv struct { + sweeper *delivery.ArchiveSweeper + eng *delivery.Engine + mainDB *database.Database + dataDir string +} + +func setupSweeperTest(t *testing.T) *sweeperEnv { + t.Helper() + + dataDir := t.TempDir() + log := archiveTestLogger() + + sqlDB, err := sql.Open( + "sqlite", + fmt.Sprintf( + "file:%s?mode=rwc", + filepath.Join(dataDir, "main.db"), + ), + ) + require.NoError(t, err) + + t.Cleanup(func() { _ = sqlDB.Close() }) + + gdb, err := gorm.Open( + sqlite.Dialector{Conn: sqlDB}, &gorm.Config{}, + ) + require.NoError(t, err) + + mainDB := database.NewTestDatabase(gdb) + require.NoError(t, mainDB.Migrate()) + + eng := delivery.NewTestEngineWithDB( + mainDB, + database.NewTestWebhookDBManager(dataDir), + log, + &http.Client{Timeout: 5 * time.Second}, + 1, + ) + + return &sweeperEnv{ + sweeper: delivery.NewTestArchiveSweeper( + mainDB, eng, log, + ), + eng: eng, + mainDB: mainDB, + dataDir: dataDir, + } +} + +// archivePath returns where the engine keeps a webhook's +// archive file. +func (env *sweeperEnv) archivePath(webhookID string) string { + return filepath.Join( + env.dataDir, fmt.Sprintf("archive-%s.db", webhookID), + ) +} + +// seedDatabaseTarget creates a webhook with one database target +// carrying the given target config JSON, and returns the +// webhook id. +func (env *sweeperEnv) seedDatabaseTarget( + t *testing.T, configJSON string, +) string { + t.Helper() + + wh := &database.Webhook{ + UserID: uuid.New().String(), + Name: "sweep-test", + } + require.NoError( + t, + env.mainDB.DB(). + Omit(clause.Associations). + Create(wh).Error, + ) + + tgt := &database.Target{ + WebhookID: wh.ID, + Name: "archive", + Type: database.TargetTypeDatabase, + Active: true, + Config: configJSON, + } + require.NoError( + t, + env.mainDB.DB(). + Omit(clause.Associations). + Create(tgt).Error, + ) + + return wh.ID +} + +// seedArchiveRows creates the archive file for a webhook and +// inserts one row per supplied archived-at timestamp, returning +// the archive path. The handle is closed before returning, so +// the archive is idle exactly as it would be with no traffic. +func (env *sweeperEnv) seedArchiveRows( + t *testing.T, webhookID string, archivedAt ...time.Time, +) string { + t.Helper() + + path := env.archivePath(webhookID) + + sqlDB, err := sql.Open( + "sqlite", fmt.Sprintf("file:%s?mode=rwc", path), + ) + require.NoError(t, err) + + gdb, err := gorm.Open( + sqlite.Dialector{Conn: sqlDB}, &gorm.Config{}, + ) + require.NoError(t, err) + + require.NoError( + t, gdb.AutoMigrate(&delivery.ExportArchivedEvent{}), + ) + + for i, at := range archivedAt { + row := delivery.ExportArchivedEvent{ + EventID: fmt.Sprintf("ev-%d", i), + WebhookID: webhookID, + Method: http.MethodPost, + Body: `{"seeded":true}`, + ArchivedAt: at, + } + require.NoError(t, gdb.Create(&row).Error) + } + + require.NoError(t, sqlDB.Close()) + + return path +} + +// archivedEventIDs returns the event ids currently stored in an +// archive file, read through a separate read-only handle. +func archivedEventIDs( + t *testing.T, path string, +) []string { + t.Helper() + + var rows []delivery.ExportArchivedEvent + + rdb := openArchiveDBForRead(t, path) + require.NoError(t, rdb.Order("event_id").Find(&rows).Error) + + ids := make([]string, 0, len(rows)) + for i := range rows { + ids = append(ids, rows[i].EventID) + } + + return ids +} + +// countArchivedRows counts the rows in an archive file without +// asserting anything, so it is safe to poll from an +// assert.Eventually condition (which runs off the test +// goroutine, where testify assertions must not be used). +func countArchivedRows(path string) (int64, error) { + sqlDB, err := sql.Open( + "sqlite", fmt.Sprintf("file:%s?mode=ro", path), + ) + if err != nil { + return 0, err + } + + defer func() { _ = sqlDB.Close() }() + + gdb, err := gorm.Open( + sqlite.Dialector{Conn: sqlDB}, &gorm.Config{}, + ) + if err != nil { + return 0, err + } + + var count int64 + + err = gdb.Model(&delivery.ExportArchivedEvent{}). + Count(&count).Error + if err != nil { + return 0, err + } + + return count, nil +} + +// TestArchiveSweeper_LoopOutlivesStartHookContext is the +// regression test for a sweeper that never swept. fx calls +// OnStart with a context carrying the application's start +// timeout (15 seconds by default), so a background loop whose +// context is derived from it is cancelled 15 seconds into the +// process — three quarters of an hour before the first tick +// under the default one-hour sweep interval. +// +// The hook context here is already cancelled, which is the same +// defect taken to its limit: a loop that inherits it never runs +// a single tick, while a correctly rooted loop keeps sweeping +// for as long as the process lives. Handing the hook a plain +// context.Background() would assert nothing at all. +func TestArchiveSweeper_LoopOutlivesStartHookContext( + t *testing.T, +) { + t.Parallel() + + env := setupSweeperTest(t) + + webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) + + now := time.Now() + path := env.seedArchiveRows( + t, webhookID, + now.Add(-48*time.Hour), + now.Add(-time.Minute), + ) + + env.sweeper.ExportSetInterval(10 * time.Millisecond) + + // Drive the genuine fx hooks the application registers, + // rather than a test-only entry point. + lc := &recordingLifecycle{} + env.sweeper.ExportRegisterHooks(lc) + require.Len(t, lc.hooks, 1) + + hookCtx, cancel := context.WithCancel(context.Background()) + cancel() + + require.NoError(t, lc.hooks[0].OnStart(hookCtx)) + + t.Cleanup(func() { + _ = lc.hooks[0].OnStop(context.Background()) + }) + + assert.Eventually( + t, + func() bool { + count, err := countArchivedRows(path) + + return err == nil && count == 1 + }, + 5*time.Second, + 10*time.Millisecond, + "the sweep loop must keep running after the start "+ + "hook's context is done; it pruned nothing, so it "+ + "inherited the hook context and died", + ) +} + +// TestArchiveSweep_DoesNotResurrectEvictedWriter covers the +// interleaving where a sweep tick has already listed a webhook's +// target when the webhook is deleted and its writer evicted. The +// sweep must not put a writer back into the registry: nothing +// would ever evict it again, which is precisely the leak this +// change exists to close. +func TestArchiveSweep_DoesNotResurrectEvictedWriter( + t *testing.T, +) { + t.Parallel() + + env := setupSweeperTest(t) + + webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) + env.seedArchiveRows( + t, webhookID, time.Now().Add(-48*time.Hour), + ) + + // Prime the registry the way a delivery would, then evict as + // the deletion path does. The target row is deliberately left + // in place: this is the tick that listed the webhook before + // the deletion committed. + _, err := env.eng.ExportEnsureArchiveWriter(webhookID) + require.NoError(t, err) + + env.eng.EvictWebhook(webhookID) + require.False(t, env.eng.ExportHasArchiveWriter(webhookID)) + + env.sweeper.ExportSweep(context.Background()) + + assert.False( + t, env.eng.ExportHasArchiveWriter(webhookID), + "a sweep must never re-register a writer for a webhook "+ + "whose registry entry has already been released", + ) +} + +// TestArchiveSweep_LeavesNoRegistryEntry states the same +// invariant in its general form: sweeping an archive whose +// webhook has no cached writer must not leave one behind, so the +// registry keeps holding only writers a delivery created and an +// eviction can reach. +func TestArchiveSweep_LeavesNoRegistryEntry(t *testing.T) { + t.Parallel() + + env := setupSweeperTest(t) + + webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) + path := env.seedArchiveRows( + t, webhookID, + time.Now().Add(-48*time.Hour), + time.Now().Add(-time.Minute), + ) + + require.False(t, env.eng.ExportHasArchiveWriter(webhookID)) + + env.sweeper.ExportSweep(context.Background()) + + assert.Equal( + t, []string{sweepRowNew}, archivedEventIDs(t, path), + "the sweep must still prune an idle archive", + ) + assert.False( + t, env.eng.ExportHasArchiveWriter(webhookID), + "the sweep must release the registry entry it created", + ) +} + +// TestArchiveSweep_KeepsWriterAdoptedByDelivery is the other +// half of that invariant: an entry the sweep created but a +// delivery then claimed belongs to the registry and must survive +// the sweep, or the delivery would be left holding a detached +// writer with an open handle that no eviction can reach. +func TestArchiveSweep_KeepsWriterAdoptedByDelivery( + t *testing.T, +) { + t.Parallel() + + env := setupSweeperTest(t) + + webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) + env.seedArchiveRows( + t, webhookID, time.Now().Add(-48*time.Hour), + ) + + webhookDB := testWebhookDB(t) + event := seedEvent(t, webhookDB, `{"n":1}`) + event.WebhookID = webhookID + d := seedDatabaseTargetDelivery( + t, webhookDB, event, `{"expiry":"1h"}`, + ) + + env.sweeper.ExportSweep(context.Background()) + require.False(t, env.eng.ExportHasArchiveWriter(webhookID)) + + env.eng.ExportDeliverDatabase(webhookDB, d) + + assert.True( + t, env.eng.ExportHasArchiveWriter(webhookID), + "a delivery's writer must stay registered", + ) + + env.sweeper.ExportSweep(context.Background()) + + assert.True( + t, env.eng.ExportHasArchiveWriter(webhookID), + "a sweep must not drop a writer a delivery owns", + ) +} + +// TestArchiveSweep_KeepsWriterAdoptedDuringSweep covers the one +// interleaving the sweepOwned flag exists for, which +// TestArchiveSweep_KeepsWriterAdoptedByDelivery cannot reach: a +// delivery adopting the sweep's own entry WHILE that sweep is +// still running. +// +// The registry operations are driven directly, in the order the +// sweep and a concurrent delivery perform them, so the window is +// exercised deterministically rather than hoped for: +// +// 1. the sweep finds no cached writer and registers one of its +// own, marked sweep-owned; +// 2. a delivery arrives, is handed that very writer, clears the +// flag and opens the archive handle; +// 3. the sweep finishes and releases what it created. +// +// Step 3 must leave the entry alone. Dropping it would detach a +// writer that is holding an open archive handle inside its +// debounce window, and no eviction could ever reach it again — +// exactly the process-lifetime handle leak this change exists to +// close. The eviction at the end proves the entry is still +// reachable. +func TestArchiveSweep_KeepsWriterAdoptedDuringSweep( + t *testing.T, +) { + t.Parallel() + + env := setupSweeperTest(t) + + webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) + env.seedArchiveRows( + t, webhookID, time.Now().Add(-48*time.Hour), + ) + + sweepWriter, created, err := env.eng.ExportSweepWriterFor( + webhookID, + ) + require.NoError(t, err) + require.True( + t, created, + "the sweep must have created the registry entry itself", + ) + + // The delivery lands mid-sweep and adopts the entry. + webhookDB := testWebhookDB(t) + event := seedEvent(t, webhookDB, `{"n":1}`) + event.WebhookID = webhookID + d := seedDatabaseTargetDelivery( + t, webhookDB, event, `{"expiry":"1h"}`, + ) + + env.eng.ExportDeliverDatabase(webhookDB, d) + + adopted := env.eng.ExportArchiveWriterFor(webhookID) + require.NotNil(t, adopted) + require.True( + t, sweepWriter.Same(adopted), + "the delivery must have adopted the sweep's writer", + ) + require.True( + t, env.eng.ExportArchiveHandleOpen(webhookID), + "the delivery leaves the archive handle open", + ) + + // The sweep finishes. + env.eng.ExportReleaseSweepWriter(webhookID, sweepWriter) + + require.True( + t, env.eng.ExportHasArchiveWriter(webhookID), + "a writer adopted by a delivery during a sweep must "+ + "stay registered, or its open handle is unreachable", + ) + + env.eng.EvictWebhook(webhookID) + + assert.False( + t, env.eng.ExportHasArchiveWriter(webhookID), + "the adopted writer must still be evictable", + ) + assert.False( + t, sweepWriter.HandleOpen(), + "eviction must have closed the adopted writer's handle", + ) +} + +// TestArchiveSweep_ContinuesAfterPerWebhookFailure proves a +// failure for one webhook does not abort the sweep for the +// others: an unparseable expiry and an unreadable archive both +// have to be logged and stepped over. +func TestArchiveSweep_ContinuesAfterPerWebhookFailure( + t *testing.T, +) { + t.Parallel() + + env := setupSweeperTest(t) + + // Seeded first so the sweep reaches them before the healthy + // webhook: targets come back in insertion order. + badConfigID := env.seedDatabaseTarget(t, `{"expiry":"!!!"}`) + env.seedArchiveRows( + t, badConfigID, time.Now().Add(-48*time.Hour), + ) + + corruptID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) + require.NoError(t, os.WriteFile( + env.archivePath(corruptID), + []byte("this is not a sqlite database"), + 0o600, + )) + + healthyID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) + healthyPath := env.seedArchiveRows( + t, healthyID, + time.Now().Add(-48*time.Hour), + time.Now().Add(-time.Minute), + ) + + env.sweeper.ExportSweep(context.Background()) + + assert.Equal( + t, []string{sweepRowNew}, + archivedEventIDs(t, healthyPath), + "a failure for an earlier webhook must not stop the "+ + "sweep from pruning the ones after it", + ) +} + +// TestArchiveSweep_OpenExistingDoesNotCreateFile pins the second +// of the two no-create guards. The first is the stat in +// sweepWebhook; this one is the SQLite open mode, which is what +// protects the window between that stat and the open. Flipping +// the sweep's mode to create-if-missing makes this fail. +func TestArchiveSweep_OpenExistingDoesNotCreateFile( + t *testing.T, +) { + t.Parallel() + + dir := t.TempDir() + path := filepath.Join(dir, "archive-absent.db") + + w := delivery.NewExportArchiveWriter( + path, archiveTestLogger(), 0, + ) + + err := w.OpenExisting(time.Hour) + + require.Error( + t, err, + "opening a missing archive without create permission "+ + "must fail rather than conjure the file", + ) + + for _, suffix := range archiveFileSuffixes() { + assert.NoFileExists(t, path+suffix) + } +} + +// TestArchiveSweep_PrunesIdleArchive is the core regression +// test for this issue: an archive that receives no further +// writes must still lose its expired rows. Before the sweeper +// existed, pruning only ever ran on a write-triggered reopen, +// so an idle archive kept expired rows forever. +func TestArchiveSweep_PrunesIdleArchive(t *testing.T) { + t.Parallel() + + env := setupSweeperTest(t) + + webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) + + now := time.Now() + path := env.seedArchiveRows( + t, webhookID, + now.Add(-48*time.Hour), + now.Add(-time.Minute), + ) + + require.Equal( + t, []string{sweepRowOld, sweepRowNew}, + archivedEventIDs(t, path), + ) + + env.sweeper.ExportSweep(context.Background()) + + assert.Equal( + t, []string{sweepRowNew}, archivedEventIDs(t, path), + "the sweep should prune rows older than the expiry "+ + "from an idle archive and keep the rest", + ) +} + +// TestArchiveSweep_LeavesArchiveClosed proves the sweep does +// not hold the archive open afterwards, so an operator can +// still move the file away for offline retention. +// +// The assertion is made on a writer the test holds a reference +// to, and the handle is proven OPEN before the sweep runs, so the +// test observes the sweep closing it rather than a writer that +// merely never opened anything. Asking the registry instead would +// be vacuous here: the sweep releases an entry it created, and a +// missing entry reports "not open" whether or not anything was +// closed. +func TestArchiveSweep_LeavesArchiveClosed(t *testing.T) { + t.Parallel() + + env := setupSweeperTest(t) + + webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) + path := env.seedArchiveRows( + t, webhookID, time.Now().Add(-48*time.Hour), + ) + + w := delivery.NewExportArchiveWriter( + path, archiveTestLogger(), 0, + ) + + require.NoError(t, w.OpenExisting(time.Hour)) + require.True( + t, w.HandleOpen(), + "the writer must hold an open handle before the sweep", + ) + + require.NoError(t, w.SweepExpired(time.Hour)) + + assert.False( + t, w.HandleOpen(), + "an idle archive must end the sweep closed", + ) +} + +// TestArchiveSweep_ClosesHandleOfRegisteredWriter states the same +// guarantee end to end, through the real sweeper and a writer the +// registry keeps. +// +// The delivery leaves the archive handle open inside its debounce +// window and makes the entry delivery-owned, so the sweep finds a +// cached writer (created is false, nothing is released) and the +// registry query afterwards is answered by a writer that really +// exists. A handle left open here would be doubly wrong: it also +// blocks the operator's move-the-file-away workflow. +func TestArchiveSweep_ClosesHandleOfRegisteredWriter( + t *testing.T, +) { + t.Parallel() + + env := setupSweeperTest(t) + + webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) + env.seedArchiveRows( + t, webhookID, time.Now().Add(-48*time.Hour), + ) + + webhookDB := testWebhookDB(t) + event := seedEvent(t, webhookDB, `{"n":1}`) + event.WebhookID = webhookID + d := seedDatabaseTargetDelivery( + t, webhookDB, event, `{"expiry":"1h"}`, + ) + + env.eng.ExportDeliverDatabase(webhookDB, d) + + require.True( + t, env.eng.ExportArchiveHandleOpen(webhookID), + "the delivery must leave the archive handle open", + ) + + env.sweeper.ExportSweep(context.Background()) + + require.True( + t, env.eng.ExportHasArchiveWriter(webhookID), + "the delivery's registry entry must survive the sweep", + ) + assert.False( + t, env.eng.ExportArchiveHandleOpen(webhookID), + "the sweep must leave the archive closed", + ) +} + +// TestArchiveSweep_NeverExpiryUntouched proves the sweep is a +// no-op for the default retention policy, so archives with no +// expiry (or the literal "never") behave exactly as before. +func TestArchiveSweep_NeverExpiryUntouched(t *testing.T) { + t.Parallel() + + for _, configJSON := range []string{ + `{"expiry":"never"}`, + `{"expiry":""}`, + "", + } { + env := setupSweeperTest(t) + + webhookID := env.seedDatabaseTarget(t, configJSON) + path := env.seedArchiveRows( + t, webhookID, + time.Now().Add(-10000*time.Hour), + ) + + env.sweeper.ExportSweep(context.Background()) + + assert.Equal( + t, []string{sweepRowOld}, archivedEventIDs(t, path), + "config %q must keep rows forever", configJSON, + ) + assert.False( + t, env.eng.ExportHasArchiveWriter(webhookID), + "config %q must leave no registry entry behind", + configJSON, + ) + } +} + +// TestArchiveSweep_NeverExpirySkipsBeforeOpening pins the +// expiry <= 0 boundary in sweepTarget, which the row assertions +// above cannot reach: pruning is separately gated on a positive +// expiry, so a "never" archive keeps its rows even if the sweep +// does open it. +// +// The spec is stronger than that — a "never" archive is skipped +// before any file is touched — so the archive here exists but has +// never been migrated. Opening it at all would run AutoMigrate +// and create the archive table, which is exactly what must not +// happen. +func TestArchiveSweep_NeverExpirySkipsBeforeOpening( + t *testing.T, +) { + t.Parallel() + + env := setupSweeperTest(t) + + webhookID := env.seedDatabaseTarget(t, `{"expiry":"never"}`) + path := env.archivePath(webhookID) + + seedUnmigratedArchive(t, path) + require.False(t, archiveTableExists(t, path)) + + env.sweeper.ExportSweep(context.Background()) + + assert.False( + t, archiveTableExists(t, path), + "a never-expiry archive must not be opened at all", + ) +} + +// seedUnmigratedArchive creates an archive file that exists but +// carries no archive schema, so any open of it is observable: the +// archive table appears only if something ran AutoMigrate. +func seedUnmigratedArchive(t *testing.T, path string) { + t.Helper() + + sqlDB, err := sql.Open( + "sqlite", fmt.Sprintf("file:%s?mode=rwc", path), + ) + require.NoError(t, err) + + _, err = sqlDB.ExecContext( + t.Context(), "CREATE TABLE placeholder (id INTEGER)", + ) + require.NoError(t, err) + + require.NoError(t, sqlDB.Close()) +} + +// archiveTableExists reports whether an archive file has had the +// archive schema migrated into it. +func archiveTableExists(t *testing.T, path string) bool { + t.Helper() + + return openArchiveDBForRead(t, path). + Migrator(). + HasTable(&delivery.ExportArchivedEvent{}) +} + +// TestArchiveSweep_DoesNotCreateArchiveFile proves the sweep +// never conjures an archive: a webhook with a database target +// that has never received an event must still have no archive +// file (nor SQLite sidecar) after a sweep. +func TestArchiveSweep_DoesNotCreateArchiveFile(t *testing.T) { + t.Parallel() + + env := setupSweeperTest(t) + + webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) + path := env.archivePath(webhookID) + + require.NoFileExists(t, path) + + env.sweeper.ExportSweep(context.Background()) + + for _, suffix := range archiveFileSuffixes() { + assert.NoFileExists( + t, path+suffix, + "the sweep must not create an archive file", + ) + } +} + +// TestArchiveSweep_DoesNotCreateAfterWriterExists covers the +// same guarantee once a writer is cached in the registry but +// the file itself is still absent (for instance because the +// operator moved the archive away). +func TestArchiveSweep_DoesNotCreateAfterWriterExists( + t *testing.T, +) { + t.Parallel() + + env := setupSweeperTest(t) + + webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) + + path, err := env.eng.ExportEnsureArchiveWriter(webhookID) + require.NoError(t, err) + require.NoFileExists(t, path) + + env.sweeper.ExportSweep(context.Background()) + + assert.NoFileExists(t, path) +} + +// TestArchiveSweep_SkipsDeletedWebhookTargets proves that the +// sweep ignores targets soft-deleted along with their webhook, +// so a deleted webhook's archive is never reopened. +func TestArchiveSweep_SkipsDeletedWebhookTargets(t *testing.T) { + t.Parallel() + + env := setupSweeperTest(t) + + webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) + path := env.seedArchiveRows( + t, webhookID, time.Now().Add(-48*time.Hour), + ) + + require.NoError( + t, + env.mainDB.DB(). + Where("webhook_id = ?", webhookID). + Delete(&database.Target{}).Error, + ) + + env.sweeper.ExportSweep(context.Background()) + + assert.Equal( + t, []string{sweepRowOld}, archivedEventIDs(t, path), + "a deleted target's archive must be left alone", + ) +} + +// TestArchiveSweep_ConcurrentWrites proves the sweep serialises +// against writes through the per-webhook writer mutex. Run +// under -race, an unsynchronised sweep would be caught here. +func TestArchiveSweep_ConcurrentWrites(t *testing.T) { + t.Parallel() + + env := setupSweeperTest(t) + + webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) + + webhookDB := testWebhookDB(t) + + // The deliveries are seeded up front, on the test's own + // goroutine: the seed helpers assert, and testify assertions + // must not run off the test goroutine. + deliveries := make( + []*database.Delivery, 0, sweepConcurrentWrites, + ) + + for range sweepConcurrentWrites { + event := seedEvent(t, webhookDB, `{"n":1}`) + event.WebhookID = webhookID + + deliveries = append( + deliveries, + seedDatabaseTargetDelivery( + t, webhookDB, event, `{"expiry":"1h"}`, + ), + ) + } + + var wg sync.WaitGroup + + wg.Add(2) + + go func() { + defer wg.Done() + + for _, d := range deliveries { + env.eng.ExportDeliverDatabase(webhookDB, d) + } + }() + + go func() { + defer wg.Done() + + for range sweepConcurrentWrites { + env.sweeper.ExportSweep(context.Background()) + } + }() + + wg.Wait() + + assert.FileExists(t, env.archivePath(webhookID)) +} + +// TestArchiveSweeper_StopsCleanly proves the background loop +// exits on OnStop rather than leaking a goroutine. +func TestArchiveSweeper_StopsCleanly(t *testing.T) { + t.Parallel() + + env := setupSweeperTest(t) + + webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`) + env.seedArchiveRows( + t, webhookID, time.Now().Add(-48*time.Hour), + ) + + env.sweeper.ExportSetInterval(time.Millisecond) + env.sweeper.ExportStart() + + // stop blocks on the loop's WaitGroup, so returning without + // error proves the loop observed the cancellation and exited + // well inside the stop context. + require.NoError( + t, env.sweeper.ExportStop(context.Background()), + ) +} + +// TestArchiveSweeper_StopHookHonoursStopTimeout is the sweeper's +// half of the same shutdown defect the engine and the retention +// reaper carried: an OnStop that discards its context and waits +// on the WaitGroup bare hangs the process forever on a prune +// wedged inside a locked archive. +func TestArchiveSweeper_StopHookHonoursStopTimeout( + t *testing.T, +) { + t.Parallel() + + env := setupSweeperTest(t) + + lc := &recordingLifecycle{} + env.sweeper.ExportRegisterHooks(lc) + require.Len(t, lc.hooks, 1) + require.NoError(t, lc.hooks[0].OnStart(context.Background())) + + release := make(chan struct{}) + + t.Cleanup(func() { close(release) }) + + env.sweeper.ExportWedgeLoop(release) + + requireStopHookExpires(t, lc.hooks[0], "archive sweeper") +} diff --git a/internal/delivery/client_ssrf_test.go b/internal/delivery/client_ssrf_test.go index d31164a..0bea976 100644 --- a/internal/delivery/client_ssrf_test.go +++ b/internal/delivery/client_ssrf_test.go @@ -18,26 +18,27 @@ func newSSRFTestEngine() *delivery.Engine { log := slog.New(slog.DiscardHandler) client := &http.Client{ - Timeout: 30 * time.Second, - Transport: delivery.NewSSRFSafeTransport(), + Timeout: 30 * time.Second, + Transport: delivery.NewTestGuard(). + NewSSRFSafeTransport(), } return delivery.NewTestEngine(log, client, 1) } -// TestClientForConfig_TimeoutKeepsSSRFGuard asserts that a -// client returned by clientForConfig for a config with a +// TestClientForRequest_TimeoutKeepsSSRFGuard asserts that a +// client returned by clientForRequest for a config with a // per-target timeout still refuses connections to // private/reserved addresses (the timeout must not drop the // SSRF-safe transport). -func TestClientForConfig_TimeoutKeepsSSRFGuard(t *testing.T) { +func TestClientForRequest_TimeoutKeepsSSRFGuard(t *testing.T) { t.Parallel() engine := newSSRFTestEngine() blocked := []string{ - "http://127.0.0.1/hook", - "http://169.254.169.254/latest/meta-data/", + loopbackHookURL, + metadataURL, "http://[fe80::1]/hook", } @@ -50,7 +51,7 @@ func TestClientForConfig_TimeoutKeepsSSRFGuard(t *testing.T) { Timeout: 5, } - client := engine.ExportClientForConfig(cfg) + client := engine.ExportClientForRequest(cfg, nil) require.NotSame(t, engine.ExportClient(), client, "a per-target timeout must yield a "+ @@ -91,10 +92,11 @@ func TestClientForConfig_TimeoutKeepsSSRFGuard(t *testing.T) { } } -// TestClientForConfig_NoTimeoutUnchanged asserts that with -// no per-target timeout the shared SSRF-safe client is -// returned unchanged. -func TestClientForConfig_NoTimeoutUnchanged(t *testing.T) { +// TestClientForRequest_NoTimeoutUnchanged asserts that a +// request with neither a per-target timeout nor an origin-scoped +// header gets the shared SSRF-safe client unchanged: there is then +// nothing for a redirect policy to strip. +func TestClientForRequest_NoTimeoutUnchanged(t *testing.T) { t.Parallel() engine := newSSRFTestEngine() @@ -103,10 +105,46 @@ func TestClientForConfig_NoTimeoutUnchanged(t *testing.T) { URL: "https://example.com/hook", } - client := engine.ExportClientForConfig(cfg) + client := engine.ExportClientForRequest(cfg, nil) assert.Same(t, engine.ExportClient(), client, "without a per-target timeout the shared client "+ "must be returned unchanged", ) } + +// TestClientForRequest_HeadersKeepSSRFGuard asserts that the +// redirect policy an origin-scoped header installs is added to a +// client that still carries the SSRF-safe transport. The guard is +// a dial hook, so keeping it is what makes each redirect hop pass +// the private-IP check too. +func TestClientForRequest_HeadersKeepSSRFGuard(t *testing.T) { + t.Parallel() + + engine := newSSRFTestEngine() + + cfg := &delivery.HTTPTargetConfig{ + URL: "https://example.com/with-headers", + Headers: map[string]string{ + "X-Api-Key": "configured", + }, + } + + client := engine.ExportClientForRequest( + cfg, []string{"X-Api-Key"}, + ) + + require.NotNil(t, client.CheckRedirect, + "an origin-scoped header must install a redirect policy", + ) + + assert.Same(t, + engine.ExportClient().Transport, client.Transport, + "the SSRF-safe transport must be reused, not dropped", + ) + + assert.Equal(t, + engine.ExportClient().Timeout, client.Timeout, + "the shared client's timeout must be inherited", + ) +} diff --git a/internal/delivery/engine.go b/internal/delivery/engine.go index 6011558..e023918 100644 --- a/internal/delivery/engine.go +++ b/internal/delivery/engine.go @@ -4,6 +4,7 @@ package delivery import ( "context" + "errors" "fmt" "log/slog" "net/http" @@ -13,7 +14,9 @@ import ( "go.uber.org/fx" "gorm.io/gorm" "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/lifecycle" "sneak.berlin/go/webhooker/internal/logger" + "sneak.berlin/go/webhooker/internal/metrics" ) const ( @@ -39,6 +42,31 @@ const ( // sweep runs. retrySweepInterval = 60 * time.Second + // pendingSweepMinAge is how long a delivery must have sat + // untouched at pending before the sweep will look at it. + // + // It is not what keeps the sweep off live work — inflightSet is, + // and it is exact. This bound sets the re-dispatch cadence for a + // delivery that really is stranded: without it, a delivery the + // database will not let the engine settle would be re-sent on + // every 60-second tick. + // + // It is nonetheless set clear of the longest legitimate attempt, + // so that the two guards do not both have to be right. That + // length is MaxTargetTimeoutSeconds (300s), the per-target + // timeout the target form accepts — not httpClientTimeout, which + // is merely the default. Fifteen minutes leaves a margin of + // three times the ceiling rather than the zero margin the two + // equal values would have given. + pendingSweepMinAge = 15 * time.Minute + + // pendingSweepBatch bounds how many stranded pending deliveries + // one sweep of one webhook re-dispatches. The sweep runs every + // retrySweepInterval, so a larger backlog drains across + // successive sweeps instead of arriving as one burst against a + // database that was already struggling to accept writes. + pendingSweepBatch = 500 + // MaxInlineBodySize is the maximum event body size that // will be carried inline in a Task through the channel. // Bodies at or above this size are left nil and fetched @@ -94,6 +122,23 @@ type Notifier interface { Notify(tasks []Task) } +// WebhookEvictor releases the delivery engine's per-webhook +// state for a webhook that no longer needs it — currently the +// cached archive writer of the database target, whose open +// file handle would otherwise outlive the webhook. +// +// It is deliberately separate from Notifier and deliberately +// one method wide: archiving lifecycle is not notification, and +// a single-method interface keeps the handlers package free of +// any dependency on the engine's internals while staying +// trivially fakeable in tests. +// +// EvictWebhook never deletes an archive file. It is idempotent +// and is a no-op for a webhook with no engine state. +type WebhookEvictor interface { + EvictWebhook(webhookID string) +} + // EngineParams are the fx dependencies for the delivery // engine. type EngineParams struct { @@ -102,6 +147,7 @@ type EngineParams struct { DB *database.Database DBManager *database.WebhookDBManager Logger *logger.Logger + SSRFGuard *Guard } // Engine processes queued deliveries in the background @@ -121,12 +167,28 @@ type Engine struct { retryCh chan Task workers int + // mtr is the delivery metric set. Production wires the + // process-wide one; a test can substitute a set registered on + // a private registry so its assertions are not disturbed by + // deliveries other tests are making at the same time. + mtr *metrics.Set + // targets maps each target type to its implementation. targets map[database.TargetType]Target // httpTarget is retained so tests can reach the HTTP // target's shared client and circuit breakers. httpTarget *httpTarget + + // dbTarget is retained so the engine can reach the archive + // writer registry for webhook eviction and the idle sweep. + dbTarget *databaseTarget + + // inflight is the set of deliveries this engine currently owns. + // Recovery and the sweeps re-dispatch only what it does not + // hold. Held by value: its zero value works, so no constructor + // can leave it out. See inflight.go. + inflight inflightSet } // New creates and registers the delivery engine with the @@ -142,25 +204,15 @@ func New( deliveryCh: make(chan Task, deliveryChannelSize), retryCh: make(chan Task, retryChannelSize), workers: defaultWorkers, + mtr: metrics.Default(), } e.initTargets(&http.Client{ Timeout: httpClientTimeout, - Transport: NewSSRFSafeTransport(), + Transport: params.SSRFGuard.NewSSRFSafeTransport(), }) - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - e.start(ctx) - - return nil - }, - OnStop: func(_ context.Context) error { - e.stop() - - return nil - }, - }) + e.registerHooks(lc) return e } @@ -169,12 +221,27 @@ func New( // are ready. func (e *Engine) Notify(tasks []Task) { for i := range tasks { + // Owned before it is queued, and until the worker that runs + // it returns. A task can sit in a 10000-deep channel for a + // long time on a healthy system, and nothing may re-send it + // while it waits. See inflight.go. + if !e.inflight.retainIdle(tasks[i].DeliveryID) { + e.log.Warn( + "delivery already in flight, not queued again", + "delivery_id", tasks[i].DeliveryID, + "event_id", tasks[i].EventID, + ) + + continue + } + select { case e.deliveryCh <- tasks[i]: default: + e.inflight.release(tasks[i].DeliveryID) e.log.Warn( "delivery channel full, "+ - "task will be recovered on restart", + "task will be recovered by the sweep", "delivery_id", tasks[i].DeliveryID, "event_id", tasks[i].EventID, ) @@ -182,6 +249,19 @@ func (e *Engine) Notify(tasks []Task) { } } +// EvictWebhook implements WebhookEvictor. It releases the +// engine's per-webhook archiving state: the database target's +// cached archive writer is dropped from the registry and its +// file handle closed. The archive file itself is left on disk +// — it is long-term storage the operator owns. +func (e *Engine) EvictWebhook(webhookID string) { + if e.dbTarget == nil { + return + } + + e.dbTarget.evict(webhookID) +} + // ScheduleRetry schedules a task to be re-enqueued onto the // retry channel after delay. It implements the Scheduler // interface the targets use to own their durable retries. @@ -196,10 +276,20 @@ func (e *Engine) ScheduleRetry( "next_attempt", task.AttemptNum, ) + // The reference is taken here rather than when the timer fires, + // so the delivery stays owned across the whole backoff window. + // Its caller is a target inside Deliver, so the engine already + // owns it; this second reference is what keeps that ownership + // alive after the worker returns and the row sits at retrying + // with nothing running. Without it the sweep finds the row + // orphaned and sends it again. + e.inflight.retain(task.DeliveryID) + time.AfterFunc(delay, func() { select { case e.retryCh <- task: default: + e.inflight.release(task.DeliveryID) e.log.Warn( "retry channel full, delivery "+ "will be recovered by periodic sweep", @@ -210,8 +300,39 @@ func (e *Engine) ScheduleRetry( }) } -func (e *Engine) start(ctx context.Context) { - ctx, cancel := context.WithCancel(ctx) +// registerHooks wires the engine's start and stop into the fx +// lifecycle. The start hook's context is deliberately ignored +// (see start for why the worker pool must not inherit it); the +// stop hook's context is honoured (see stop). +func (e *Engine) registerHooks(lc fx.Lifecycle) { + lc.Append(fx.Hook{ + //nolint:contextcheck // Not inheriting the hook context + // is the point: see start. + OnStart: func(_ context.Context) error { + e.start() + + return nil + }, + OnStop: func(ctx context.Context) error { + return e.stop(ctx) + }, + }) +} + +// start launches the worker pool, restart recovery, and the +// periodic retry sweep. +// +// Their context is derived from context.Background(), NOT from +// the fx OnStart hook context. The hook context carries fx's +// start timeout (15s by default) and is cancelled once the start +// phase completes, so goroutines derived from it stop a few +// seconds into the process: every worker would return and the +// engine would silently stop delivering webhooks entirely. A +// long-lived goroutine must outlive the startup phase, so its +// lifetime is bounded by OnStop instead: stop cancels this +// context and waits on the WaitGroup. +func (e *Engine) start() { + ctx, cancel := context.WithCancel(context.Background()) e.cancel = cancel for range e.workers { @@ -228,17 +349,36 @@ func (e *Engine) start(ctx context.Context) { go e.retrySweep(ctx) + e.wg.Add(1) + + go e.queueDepthSampler(ctx) + e.log.Info( "delivery engine started", "workers", e.workers, ) } -func (e *Engine) stop() { +// stop cancels the worker pool's context and waits for the pool +// to drain, bounded by the stop hook's context: a wedged worker +// must not hang the process past fx's stop timeout. +func (e *Engine) stop(ctx context.Context) error { e.log.Info("delivery engine stopping") - e.cancel() - e.wg.Wait() + + if e.cancel != nil { + e.cancel() + } + + err := lifecycle.WaitForShutdown( + ctx, e.log, "delivery engine", &e.wg, + ) + if err != nil { + return err + } + e.log.Info("delivery engine stopped") + + return nil } func (e *Engine) worker(ctx context.Context) { @@ -249,13 +389,35 @@ func (e *Engine) worker(ctx context.Context) { case <-ctx.Done(): return case task := <-e.deliveryCh: - e.processNewTask(ctx, &task) + e.runTask(ctx, &task, e.processNewTask) case task := <-e.retryCh: - e.processRetryTask(ctx, &task) + e.runTask(ctx, &task, e.processRetryTask) } } } +// runTask runs one task and then drops the reference the queueing +// side took on its delivery. +// +// The release is deferred rather than written after the call because +// every early return inside the processing paths must drop it too: a +// delivery whose database could not be opened is one the engine has +// stopped working on, and leaving it owned would hide it from the +// sweep forever. +// +// Ownership does not necessarily end here. A target that scheduled a +// retry took its own reference before this one is dropped, so the +// delivery stays owned through the backoff window. +func (e *Engine) runTask( + ctx context.Context, + task *Task, + run func(context.Context, *Task), +) { + defer e.inflight.release(task.DeliveryID) + + run(ctx, task) +} + func (e *Engine) recoverPending(ctx context.Context) { defer e.wg.Done() @@ -278,7 +440,7 @@ func (e *Engine) processNewTask( event := buildEventFromTask(task) - event, err = e.resolveEventBody( + event, err = e.hydrateEvent( webhookDB, event, task, ) if err != nil { @@ -344,9 +506,13 @@ func (e *Engine) processRetryTask( return } + if e.abandonRetryForMissingTarget(webhookDB, d, task) { + return + } + event := buildEventFromTask(task) - event, err = e.resolveEventBody( + event, err = e.hydrateEvent( webhookDB, event, task, ) if err != nil { @@ -368,6 +534,64 @@ func (e *Engine) processRetryTask( e.processDelivery(ctx, webhookDB, d, task) } +// abandonRetryForMissingTarget stops a retry chain whose target has +// been deleted, and reports whether it did. +// +// A scheduled retry lives in memory as a time.AfterFunc holding the +// target's configuration as it was when the chain began, and nothing +// else on this path reads the target row. Without this check a +// deletion stops nothing: the timer keeps firing and keeps sending to +// the destination the operator removed, for the whole remaining +// backoff chain. Terminalising in the recovery and sweep paths alone +// is not enough, because those only see the delivery once nothing +// holds it in memory — which is to say after a restart. +// +// The worker already owns this delivery, so the terminal write happens +// here directly, exactly as a target's own Deliver fails one. Claiming +// it again through the recovery gate would only fail against the +// reference the worker itself is holding. +// +// A lookup that fails for any other reason is not a deletion — it is +// the main database being unreadable — and the delivery goes ahead as +// it did before. A guard that terminally failed deliveries on a +// transient fault would be worse than the bug it fixes. +func (e *Engine) abandonRetryForMissingTarget( + webhookDB *gorm.DB, + d *database.Delivery, + task *Task, +) bool { + _, err := e.loadTarget(task.TargetID) + if err == nil { + return false + } + + if !errors.Is(err, gorm.ErrRecordNotFound) { + e.log.Warn( + "could not confirm the target of a retrying "+ + "delivery still exists; attempting anyway", + "delivery_id", task.DeliveryID, + "target_id", task.TargetID, + "error", err, + ) + + return false + } + + targetType, reason := e.missingTargetReason(task.TargetID) + + e.log.Warn( + "abandoning scheduled retry: target is gone", + "webhook_id", task.WebhookID, + "delivery_id", task.DeliveryID, + "target_id", task.TargetID, + "target_type", targetType, + ) + + e.failDelivery(webhookDB, d, targetType, reason) + + return true +} + func (e *Engine) recoverInFlight(ctx context.Context) { var webhookIDs []string @@ -443,7 +667,16 @@ func (e *Engine) recoverRetryingDeliveries( return } + settled := e.reconcileDelivered( + webhookDB, webhookID, retrying, + e.loadTargetMap(retrying), + ) + for i := range retrying { + if _, ok := settled[retrying[i].ID]; ok { + continue + } + e.recoverSingleRetry( webhookDB, webhookID, &retrying[i], ) @@ -453,8 +686,9 @@ func (e *Engine) recoverRetryingDeliveries( // recoverSingleRetry hands an orphaned retrying delivery back // to its target to recompute the remaining backoff, then // reschedules it. Targets that do not own durable retries -// (fire-and-forget) never produce retrying deliveries, so -// they are skipped. +// (fire-and-forget) never produce retrying deliveries, so a +// delivery found in that state has had its target's type +// changed underneath it and is terminally failed. func (e *Engine) recoverSingleRetry( webhookDB *gorm.DB, webhookID string, @@ -462,6 +696,20 @@ func (e *Engine) recoverSingleRetry( ) { target, err := e.loadTarget(d.TargetID) if err != nil { + // A target that is merely gone is an operator action with a + // terminal answer. Any other failure is the main database + // refusing to read, which is transient and must leave the + // delivery alone: failing every retrying delivery of every + // webhook on one bad read would be a far larger fault than + // the strand it is meant to clear. + if errors.Is(err, gorm.ErrRecordNotFound) { + e.failMissingTargetRetry( + webhookDB, webhookID, d, + ) + + return + } + e.log.Error( "failed to load target for retrying "+ "delivery recovery", @@ -475,6 +723,10 @@ func (e *Engine) recoverSingleRetry( rs, ok := e.targets[target.Type].(rescheduler) if !ok { + e.failUnretryableRetry( + webhookDB, webhookID, d, &target, + ) + return } @@ -500,6 +752,10 @@ func (e *Engine) recoverSingleRetry( d, webhookID, &event, &target, attemptNum+1, ) + if !e.rescheduleRecovered(webhookDB, task, remaining) { + return + } + e.log.Info( "recovering retrying delivery", "webhook_id", webhookID, @@ -507,8 +763,6 @@ func (e *Engine) recoverSingleRetry( "attempt", attemptNum, "remaining_backoff", remaining, ) - - e.ScheduleRetry(task, remaining) } func (e *Engine) recoverPendingDeliveries( @@ -518,12 +772,14 @@ func (e *Engine) recoverPendingDeliveries( ) { var deliveries []database.Delivery + // No Preload: event bodies are read one at a time in + // sendRecoveredDeliveries, and only for the deliveries actually + // being sent. result := webhookDB. Where( "status = ?", database.DeliveryStatusPending, ). - Preload("Event"). Find(&deliveries) if result.Error != nil { @@ -546,11 +802,137 @@ func (e *Engine) recoverPendingDeliveries( "count", len(deliveries), ) + e.recoverPendingBatch( + ctx, webhookDB, webhookID, deliveries, + ) +} + +// recoverPendingBatch settles every delivery in the batch that was +// already delivered, and re-dispatches only the rest. Both the +// restart-time recovery and the periodic sweep go through it, so a +// pending delivery is treated the same however it was found. +func (e *Engine) recoverPendingBatch( + ctx context.Context, + webhookDB *gorm.DB, + webhookID string, + deliveries []database.Delivery, +) { targetMap := e.loadTargetMap(deliveries) - e.sendRecoveredDeliveries( - ctx, deliveries, webhookID, targetMap, + settled := e.reconcileDelivered( + webhookDB, webhookID, deliveries, targetMap, ) + + e.sendRecoveredDeliveries( + ctx, webhookDB, deliveries, webhookID, + targetMap, settled, + ) +} + +// reconcileDelivered finds the deliveries in a recovered batch that +// already have a successful DeliveryResult, marks them delivered, and +// returns their ids so the caller does not send them a second time. +// +// This is the state the engine previously had no way to represent. A +// delivery is left in a non-terminal state by a failed bookkeeping +// write, and that covers two different histories: nothing was ever +// sent, or the send reached the receiver and only the status write +// failed. Re-sending was the sole option, so every stranded row +// produced a duplicate at the receiver and an event log that recorded +// one attempt for two POSTs. A successful result row distinguishes +// them: it is written before the status, so its presence means the +// wire I/O happened and was recorded, and all that is missing is the +// status. +// +// Every recovery path runs this, not only the pending one. A delivery +// abandoned at retrying can hold a successful result just as a pending +// one can — a second attempt that reached the receiver and whose status +// write then failed sits at retrying with success recorded — and +// re-sending it is the same duplicate. +// +// Deliveries whose result row itself never landed are not in the +// returned set and are re-sent, recorded as the further attempt they +// are. That is honest at-least-once delivery rather than a silent +// duplicate. +func (e *Engine) reconcileDelivered( + webhookDB *gorm.DB, + webhookID string, + deliveries []database.Delivery, + targetMap map[string]database.Target, +) map[string]struct{} { + settled := make(map[string]struct{}) + + if len(deliveries) == 0 { + return settled + } + + ids := make([]string, 0, len(deliveries)) + for i := range deliveries { + ids = append(ids, deliveries[i].ID) + } + + var deliveredIDs []string + + err := webhookDB. + Model(&database.DeliveryResult{}). + Where( + "delivery_id IN ? AND success = ?", ids, true, + ). + Distinct(). + Pluck("delivery_id", &deliveredIDs).Error + if err != nil { + // Every delivery stays out of the settled set, so the batch + // is re-sent exactly as it was before this check existed. + // That is the safe direction: a duplicate delivery beats + // declaring a delivery successful on a query that failed. + e.log.Error( + "failed to query successful delivery results; "+ + "pending deliveries will be re-sent", + "webhook_id", webhookID, + "error", err, + ) + + return settled + } + + for _, id := range deliveredIDs { + settled[id] = struct{}{} + } + + if len(settled) == 0 { + return settled + } + + e.log.Info( + "settling recovered deliveries that already succeeded", + "webhook_id", webhookID, + "count", len(settled), + ) + + for i := range deliveries { + if _, ok := settled[deliveries[i].ID]; !ok { + continue + } + + // A delivery the engine is working on right now settles + // itself; writing over it from here would race that worker. + if !e.inflight.retainIdle(deliveries[i].ID) { + delete(settled, deliveries[i].ID) + + continue + } + + e.settleStatus( + webhookDB, + &deliveries[i], + targetMap[deliveries[i].TargetID].Type, + database.DeliveryStatusDelivered, + ) + + e.inflight.release(deliveries[i].ID) + } + + return settled } func (e *Engine) retrySweep(ctx context.Context) { @@ -634,6 +1016,11 @@ func (e *Engine) sweepWebhookRetries( return } + settled := e.reconcileDelivered( + webhookDB, webhookID, retrying, + e.loadTargetMap(retrying), + ) + for i := range retrying { select { case <-ctx.Done(): @@ -641,16 +1028,76 @@ func (e *Engine) sweepWebhookRetries( default: } + if _, ok := settled[retrying[i].ID]; ok { + continue + } + e.sweepSingleRetry( webhookDB, webhookID, &retrying[i], ) } + + e.sweepWebhookPending(ctx, webhookDB, webhookID) +} + +// sweepWebhookPending recovers deliveries stranded at pending. +// +// A delivery is created pending and leaves that state only when its +// outcome is written, so a pending row the engine does not own is one +// whose bookkeeping write failed — the state that used to sit there +// until a restart, and then produce a duplicate at the receiver. The +// sweep gives it the same reconcile-then-dispatch treatment restart +// recovery gets, so it costs a minute rather than an operator +// noticing. +// +// What keeps the sweep off live work is ownership, checked per +// delivery in takeForRedispatch, not the age bound in this query. +// A delivery waiting in deliveryCh is pending and arbitrarily old — +// the channel holds 10000 tasks and 10 workers drain it — so +// reasoning from the row's age alone re-sends it. See inflight.go. +func (e *Engine) sweepWebhookPending( + ctx context.Context, + webhookDB *gorm.DB, + webhookID string, +) { + var pending []database.Delivery + + err := webhookDB. + Where( + "status = ? AND updated_at < ?", + database.DeliveryStatusPending, + time.Now().Add(-pendingSweepMinAge), + ). + Limit(pendingSweepBatch). + Find(&pending).Error + if err != nil { + e.log.Error( + "retry sweep: "+ + "failed to query pending deliveries", + "webhook_id", webhookID, + "error", err, + ) + + return + } + + if len(pending) == 0 { + return + } + + e.log.Info( + "retry sweep: recovering stranded pending deliveries", + "webhook_id", webhookID, + "count", len(pending), + ) + + e.recoverPendingBatch(ctx, webhookDB, webhookID, pending) } // sweepSingleRetry re-enqueues an orphaned retrying delivery // whose backoff window has elapsed, delegating the backoff -// decision to the delivery's target. Targets that do not own -// durable retries are skipped. +// decision to the delivery's target. A delivery whose target +// no longer owns durable retries is terminally failed. func (e *Engine) sweepSingleRetry( webhookDB *gorm.DB, webhookID string, @@ -658,6 +1105,16 @@ func (e *Engine) sweepSingleRetry( ) { target, err := e.loadTarget(d.TargetID) if err != nil { + // Deleted is terminal, unreadable is not; see + // recoverSingleRetry. + if errors.Is(err, gorm.ErrRecordNotFound) { + e.failMissingTargetRetry( + webhookDB, webhookID, d, + ) + + return + } + e.log.Error( "retry sweep: failed to load target", "delivery_id", d.ID, @@ -670,6 +1127,10 @@ func (e *Engine) sweepSingleRetry( rs, ok := e.targets[target.Type].(rescheduler) if !ok { + e.failUnretryableRetry( + webhookDB, webhookID, d, &target, + ) + return } @@ -697,17 +1158,196 @@ func (e *Engine) sweepSingleRetry( d, webhookID, &event, &target, attemptNum+1, ) - select { - case e.retryCh <- task: - e.log.Info( - "retry sweep: "+ - "recovered orphaned retrying delivery", - "delivery_id", d.ID, - "webhook_id", webhookID, - "attempt", attemptNum+1, - ) - default: + if !e.redispatch( + e.retryCh, webhookDB, task, + database.DeliveryStatusRetrying, + ) { + return } + + e.log.Info( + "retry sweep: "+ + "recovered orphaned retrying delivery", + "delivery_id", d.ID, + "webhook_id", webhookID, + "attempt", attemptNum+1, + ) +} + +// failUnretryableRetry terminally fails an orphaned retrying +// delivery whose target type no longer supports retries. Both +// restart recovery and the periodic sweep call it, so the +// terminal transition exists once. +// +// This is only reachable when a target's type has been changed +// out from under an in-flight retrying delivery (or the type is +// unknown to the registry): fire-and-forget targets never set +// status retrying themselves. Re-dispatching under the new type +// would be a delivery the operator never asked for, and leaving +// the row retrying strands it forever, so the delivery is +// failed with a recorded reason. The event stays stored, but +// nothing redelivers it today. Logged at warn, not error: this +// is operator-caused state, not a system fault. +func (e *Engine) failUnretryableRetry( + webhookDB *gorm.DB, + webhookID string, + d *database.Delivery, + target *database.Target, +) { + // Terminal, and reached from the recovery paths, so it takes + // ownership like every other write they make: a delivery the + // engine is still attempting must not be failed underneath the + // worker running it. + if !e.inflight.retainIdle(d.ID) { + return + } + + defer e.inflight.release(d.ID) + + e.log.Warn( + "failing orphaned retrying delivery: target "+ + "type no longer supports retries", + "webhook_id", webhookID, + "delivery_id", d.ID, + "target_id", target.ID, + "target_name", target.Name, + "target_type", target.Type, + ) + + reason := fmt.Sprintf( + "target type %q does not support retries; "+ + "delivery was left retrying by a previous "+ + "target type and has been failed terminally", + target.Type, + ) + + e.failDelivery(webhookDB, d, target.Type, reason) +} + +// failMissingTargetRetry terminally fails an orphaned retrying +// delivery whose target row is gone. Both restart recovery and the +// periodic sweep call it, so the transition exists once. +// +// Until it existed both paths logged the failed lookup and returned, +// which left the delivery retrying for the life of the database and +// the sweep repeating the same error every minute forever. Failing it +// with a recorded reason is the treatment the other orphaned-retry +// cases already get, so all of them read alike in the event log. +// +// Logged at warn rather than error: a deleted target is an operator +// action, not a system fault. +func (e *Engine) failMissingTargetRetry( + webhookDB *gorm.DB, + webhookID string, + d *database.Delivery, +) { + // Terminal, and reached from the recovery paths, so it takes + // ownership like every other write they make. + if !e.inflight.retainIdle(d.ID) { + return + } + + defer e.inflight.release(d.ID) + + targetType, reason := e.missingTargetReason(d.TargetID) + + e.log.Warn( + "failing orphaned retrying delivery: "+ + "its target no longer exists", + "webhook_id", webhookID, + "delivery_id", d.ID, + "target_id", d.TargetID, + "target_type", targetType, + ) + + e.failDelivery(webhookDB, d, targetType, reason) +} + +// missingTargetReason describes a target id that no longer resolves, +// and returns the type of the deleted row where there still is one. +// +// The lookup is Unscoped because deletes are soft: the row survives +// with deleted_at set, invisible to loadTarget's default scope. +// Reading it is what separates "you deleted this target" from "this id +// never named a row" — different things to whoever reads the event +// log, and only the first is something an operator did. The widened +// scope is deliberately confined to this terminal path: the engine's +// normal target loading must go on refusing a deleted target, or +// deleting one would stop nothing. +// +// The type comes back so the caller can label the delivery's status +// transition with it. Where the row is gone entirely there is no type +// to give, and updateDeliveryStatus leaves the counter alone rather +// than opening a series named by the empty string. +func (e *Engine) missingTargetReason( + targetID string, +) (database.TargetType, string) { + var target database.Target + + err := e.database.DB().Unscoped(). + First(&target, "id = ?", targetID).Error + if err != nil { + return "", fmt.Sprintf( + "target %s no longer exists; the delivery "+ + "cannot be retried and has been failed "+ + "terminally", + targetID, + ) + } + + return target.Type, fmt.Sprintf( + "target %q (type %s) was deleted; the delivery "+ + "cannot be retried and has been failed terminally", + target.Name, target.Type, + ) +} + +// failDelivery records why a delivery is over and then marks it +// failed. The caller must already own the delivery: every call site is +// either a worker holding the reference runTask took, or a recovery +// path that took one through retainIdle. +// +// The result row is written first and a failure to write it stops the +// transition, which is what keeps a delivery from ending failed with +// an empty event log — the state that leaves an operator with nothing +// but a server log line to work out what happened. A delivery whose +// reason could not be recorded stays in the non-terminal state it +// already holds, where the sweep will find it again; see +// bookkeepingFailed. +// +// The target type is a parameter rather than read off d because the +// orphaned-retry callers deliberately hold a delivery loaded without +// its Target relation: populating d.Target would make GORM's +// SaveBeforeAssociations upsert the whole target row — plaintext +// config, which for a slack target is the credential — into the +// per-webhook event database. See +// https://git.eeqj.de/sneak/webhooker/issues/206. +func (e *Engine) failDelivery( + webhookDB *gorm.DB, + d *database.Delivery, + targetType database.TargetType, + reason string, +) { + err := e.recordResult( + webhookDB, + d, + e.countAttempts(webhookDB, d.ID)+1, + false, + 0, + "", + reason, + 0, + ) + if err != nil { + e.bookkeepingFailed(d, err) + + return + } + + e.settleStatus( + webhookDB, d, targetType, + database.DeliveryStatusFailed, + ) } // processDelivery dispatches a delivery to the target that @@ -726,8 +1366,19 @@ func (e *Engine) processDelivery( "type", d.Target.Type, ) - e.updateDeliveryStatus( - webhookDB, d, database.DeliveryStatusFailed, + // The reason is recorded, not just logged. This branch used + // to fail the delivery with no DeliveryResult at all, which + // showed in the event log as "failed, no attempts recorded + // yet" and left one server log line as the only account of + // why anywhere. + e.failDelivery( + webhookDB, d, d.Target.Type, + fmt.Sprintf( + "unknown target type %q: this build has no "+ + "delivery implementation for it, so no "+ + "attempt was made", + d.Target.Type, + ), ) return @@ -736,9 +1387,35 @@ func (e *Engine) processDelivery( target.Deliver(ctx, webhookDB, d, task, e) } +// observeAttempt counts one delivery attempt that was actually +// dispatched to a target, and records how long it took. +// +// It is called from the dispatch paths rather than from around +// Target.Deliver, because Deliver is also entered for deliveries +// that never reach the wire: a delivery an open circuit breaker +// refuses sends nothing, records no DeliveryResult, and is +// rescheduled. Counting those would climb the attempts counter with +// no traffic behind it and fill the duration histogram with +// microsecond samples, which would make the delivery-duration +// quantiles improve during exactly the outage they exist to reveal. +func (e *Engine) observeAttempt( + t database.TargetType, dur time.Duration, +) { + e.mtr.DeliveryAttempted(t) + e.mtr.ObserveDeliveryDuration(t, dur) +} + // recordResult persists a DeliveryResult row describing a // single attempt. It is a cross-target helper the targets // call. +// +// It returns its error rather than swallowing it. A DeliveryResult +// row is the only record that an attempt happened at all, so a +// caller that ignored a failed write would go on to mark the +// delivery delivered — leaving the event log claiming one attempt +// for a receiver that got two. Every caller must instead stop +// advancing the delivery's status and let it stay in the +// non-terminal state it already holds; see bookkeepingFailed. func (e *Engine) recordResult( webhookDB *gorm.DB, d *database.Delivery, @@ -747,7 +1424,7 @@ func (e *Engine) recordResult( statusCode int, respBody, errMsg string, durationMs int64, -) { +) error { result := &database.DeliveryResult{ DeliveryID: d.ID, AttemptNum: attemptNum, @@ -760,31 +1437,104 @@ func (e *Engine) recordResult( err := webhookDB.Create(result).Error if err != nil { - e.log.Error( - "failed to record delivery result", - "delivery_id", d.ID, - "error", err, + return fmt.Errorf( + "recording delivery result for %s: %w", d.ID, err, ) } + + return nil +} + +// bookkeepingFailed reports that a delivery's own record of what +// happened could not be written, and deliberately writes nothing in +// response. +// +// Leaving the row alone is the whole point. A delivery is created +// pending and only ever leaves that state through +// updateDeliveryStatus, so a delivery whose bookkeeping write failed +// is still pending or retrying — the two non-terminal states, per +// DeliveryStatus.Terminal — and both are swept and recovered. Writing +// anything here would need the very database that just refused a +// write, and would be one more thing to fail; not writing cannot. +// +// The cost is honest at-least-once behaviour: a send that reached the +// receiver but whose result row did not land is attempted again, and +// recorded as the further attempt it is. What no longer happens is the +// silent duplicate — a second POST the event log denies ever +// occurred. Where the result row *did* land and only the status write +// failed, reconcileDelivered settles the row without re-sending. +func (e *Engine) bookkeepingFailed( + d *database.Delivery, err error, +) { + e.log.Error( + "delivery bookkeeping write failed; leaving delivery "+ + "in a recoverable state", + "delivery_id", d.ID, + "event_id", d.EventID, + "target_id", d.TargetID, + "status", d.Status, + "error", err, + ) } // updateDeliveryStatus persists a new status for a delivery. -// It is a cross-target helper the targets call. +// It is a cross-target helper the targets call, and therefore the +// single point where a delivery's outcome — delivered, terminally +// failed, or put back into retry — is counted. +// +// The target type is a parameter rather than read off d.Target +// because one caller — failUnretryableRetry — deliberately holds a +// delivery loaded without its target relation, and must keep it that +// way: a populated d.Target makes GORM upsert the target row, config +// and all, into the per-webhook database. +// +// The counter moves only after the row is written, so a transition +// the database rejected is not claimed as an outcome that happened. +// For the same reason the error is returned rather than logged and +// dropped: a delivery whose status write failed has not reached that +// status, and its caller must not act as though it had. func (e *Engine) updateDeliveryStatus( webhookDB *gorm.DB, d *database.Delivery, + targetType database.TargetType, status database.DeliveryStatus, -) { +) error { err := webhookDB.Model(d). Update("status", status).Error if err != nil { - e.log.Error( - "failed to update delivery status", - "delivery_id", d.ID, - "status", status, - "error", err, + return fmt.Errorf( + "updating delivery %s to status %s: %w", + d.ID, status, err, ) } + + // An empty type means the target row is gone — a delivery being + // settled long after its target was deleted. The row still has to + // be settled, but the counter is left alone rather than given a + // series labelled with the empty string. + if targetType != "" { + e.mtr.DeliveryStatusChanged(targetType, status) + } + + return nil +} + +// settleStatus moves a delivery to its outcome status and reports a +// failed write through bookkeepingFailed, which leaves the row +// recoverable. It exists so the target call sites read as one +// statement rather than four lines of identical error handling. +func (e *Engine) settleStatus( + webhookDB *gorm.DB, + d *database.Delivery, + targetType database.TargetType, + status database.DeliveryStatus, +) { + err := e.updateDeliveryStatus( + webhookDB, d, targetType, status, + ) + if err != nil { + e.bookkeepingFailed(d, err) + } } func truncate(s string, maxLen int) string { @@ -797,6 +1547,11 @@ func truncate(s string, maxLen int) string { // --- Helper functions --- +// buildEventFromTask reconstructs the event a Task describes, as far +// as the Task itself goes. The fields it cannot fill — the body when +// it was too large to inline, and the receipt time, which no Task +// carries — come from the stored row in hydrateEvent, which every +// caller of this function runs next. func buildEventFromTask(task *Task) database.Event { event := database.Event{ EntrypointID: task.EntrypointID, @@ -824,29 +1579,67 @@ func buildTargetFromTask(task *Task) database.Target { return target } -func (e *Engine) resolveEventBody( +// hydrateEvent fills in the event fields a Task does not carry, by +// reading the stored event row. +// +// CreatedAt is the event's receipt time and lives only in that row. +// The Slack target renders it into every message it sends, so an +// unhydrated event puts the zero time in front of a human on every +// notification the product delivers. See +// https://git.eeqj.de/sneak/webhooker/issues/257. +// +// The body comes from the same row when the Task did not inline it, +// which is the case for a body at or above MaxInlineBodySize. +// +// A read failure is fatal to the delivery only when the body depended +// on it. When the Task inlined the body, the delivery has everything +// it needs to be sent and goes ahead with the timestamp unset: the row +// can be gone under a retention reap while a queued delivery still +// holds its body, and dropping a deliverable event to protect one +// metadata field would be a worse failure than the one it prevents. +func (e *Engine) hydrateEvent( webhookDB *gorm.DB, event database.Event, task *Task, ) (database.Event, error) { - if task.Body != nil { + columns := []string{"created_at"} + + if task.Body == nil { + columns = append(columns, "body") + } + + var dbEvent database.Event + + err := webhookDB.Select(columns). + First(&dbEvent, "id = ?", task.EventID).Error + if err != nil { + if task.Body == nil { + return event, fmt.Errorf( + "fetching event body: %w", err, + ) + } + + e.log.Warn( + "could not read the stored event; delivering "+ + "the inlined body without its receipt time", + "event_id", task.EventID, + "delivery_id", task.DeliveryID, + "error", err, + ) + event.Body = *task.Body return event, nil } - var dbEvent database.Event + event.CreatedAt = dbEvent.CreatedAt - err := webhookDB.Select("body"). - First(&dbEvent, "id = ?", task.EventID).Error - if err != nil { - return event, fmt.Errorf( - "fetching event body: %w", err, - ) + if task.Body != nil { + event.Body = *task.Body + } else { + event.Body = dbEvent.Body } - event.Body = dbEvent.Body - return event, nil } @@ -878,6 +1671,184 @@ func (e *Engine) countAttempts( return int(resultCount) } +// takeForRedispatch decides whether a recovered delivery may be sent +// again, and takes it if so. It is the single gate every re-dispatch +// path goes through, and it asks two separate questions in order. +// +// First, does the engine already own this delivery? Ownership is +// exact and mutually exclusive, so a delivery queued, being attempted, +// or waiting out a retry backoff is refused here, and two dispatchers +// racing for the same delivery cannot both win. See inflight.go. +// +// Second, is the row still in the status that made it eligible? The +// batch was read some time ago and a worker may have settled a row +// since. The check is a conditional update rather than a read so the +// answer cannot go stale between asking and acting. +// +// Stamping updated_at is the same statement, and it is a cadence +// control rather than a claim: the pending sweep selects on that +// column, so a delivery handed out now is not selected again on the +// next tick a minute later but after pendingSweepMinAge. A delivery +// the database refuses to settle is therefore retried on that +// interval instead of every tick. +// +// A failed write is a refusal. It means the database is not accepting +// writes, which is the condition that stranded the delivery in the +// first place; an attempt that cannot be recorded is exactly the +// unlogged duplicate this is all here to prevent. +// +// The caller must release ownership if it then fails to queue the +// task. +func (e *Engine) takeForRedispatch( + webhookDB *gorm.DB, + deliveryID string, + eligible database.DeliveryStatus, +) bool { + if !e.inflight.retainIdle(deliveryID) { + return false + } + + res := webhookDB. + Model(&database.Delivery{}). + Where( + "id = ? AND status = ?", deliveryID, eligible, + ). + UpdateColumn("updated_at", time.Now()) + + if res.Error != nil { + e.log.Error( + "failed to mark delivery for re-dispatch; "+ + "leaving it for a later sweep", + "delivery_id", deliveryID, + "error", res.Error, + ) + e.inflight.release(deliveryID) + + return false + } + + if res.RowsAffected != 1 { + // Settled underneath us between the query and here. + e.inflight.release(deliveryID) + + return false + } + + return true +} + +// queueRecovered puts an owned delivery's task on a worker channel, +// dropping the ownership the gate took if it does not fit. +func (e *Engine) queueRecovered( + ch chan<- Task, task Task, +) bool { + select { + case ch <- task: + return true + default: + e.inflight.release(task.DeliveryID) + e.log.Warn( + "worker channel full during recovery; "+ + "delivery will be recovered by a later sweep", + "delivery_id", task.DeliveryID, + "webhook_id", task.WebhookID, + ) + + return false + } +} + +// redispatch hands a recovered delivery to a worker channel through +// takeForRedispatch, and reports whether the task was queued. +func (e *Engine) redispatch( + ch chan<- Task, + webhookDB *gorm.DB, + task Task, + eligible database.DeliveryStatus, +) bool { + if !e.takeForRedispatch( + webhookDB, task.DeliveryID, eligible, + ) { + return false + } + + return e.queueRecovered(ch, task) +} + +// rescheduleRecovered hands an orphaned retrying delivery back to the +// retry timer, through the same gate. It reports whether the delivery +// was rescheduled. +// +// The reference taken by the gate is dropped as soon as ScheduleRetry +// has taken its own, which it does before returning: what keeps the +// delivery owned through the backoff window is ScheduleRetry's +// reference, not this one. +func (e *Engine) rescheduleRecovered( + webhookDB *gorm.DB, task Task, delay time.Duration, +) bool { + if !e.takeForRedispatch( + webhookDB, task.DeliveryID, + database.DeliveryStatusRetrying, + ) { + return false + } + + defer e.inflight.release(task.DeliveryID) + + e.ScheduleRetry(task, delay) + + return true +} + +// countAttemptsBatch counts the recorded attempts of every delivery +// in a batch with one grouped query, keyed by delivery id. Deliveries +// with no attempts are simply absent from the result, which reads back +// as the zero this caller wants. +// +// One query rather than one per delivery: this runs on the recovery +// path, which is a burst of writes against a database that has just +// been under enough contention to strand these rows in the first +// place. See https://git.eeqj.de/sneak/webhooker/issues/256. +func (e *Engine) countAttemptsBatch( + webhookDB *gorm.DB, deliveries []database.Delivery, +) map[string]int { + counts := make(map[string]int, len(deliveries)) + + if len(deliveries) == 0 { + return counts + } + + ids := make([]string, 0, len(deliveries)) + for i := range deliveries { + ids = append(ids, deliveries[i].ID) + } + + // One delivery id per recorded attempt, tallied here rather than + // grouped in SQL: internal/gormlog forbids (*gorm.DB).Scan, which + // a GROUP BY into a struct would need, and an attempt row per + // delivery is bounded by the target's MaxRetries. + var attemptIDs []string + + err := webhookDB. + Model(&database.DeliveryResult{}). + Where("delivery_id IN ?", ids). + Pluck("delivery_id", &attemptIDs).Error + if err != nil { + e.log.Error( + "failed to count delivery attempts for recovery", + "error", err, + ) + + return counts + } + + for _, id := range attemptIDs { + counts[id]++ + } + + return counts +} + func (e *Engine) loadEvent( webhookDB *gorm.DB, eventID string, ) (database.Event, error) { @@ -945,6 +1916,10 @@ func buildRecoveryTask( func (e *Engine) loadTargetMap( deliveries []database.Delivery, ) map[string]database.Target { + if len(deliveries) == 0 { + return nil + } + seen := make(map[string]bool) targetIDs := make([]string, 0, len(deliveries)) @@ -981,12 +1956,33 @@ func (e *Engine) loadTargetMap( return targetMap } +// sendRecoveredDeliveries re-dispatches pending deliveries, skipping +// the ids in settled — those already reached their receiver and have +// been marked delivered by reconcileDelivered. +// +// The skip and takeForRedispatch's status check answer different +// questions and neither replaces the other. This one is "did this +// delivery already succeed", which is what settles the row to +// delivered instead of sending it, and which is the only thing that +// keeps the retrying paths from terminally failing a delivery that +// reconcile just settled. The status check is "is the row still what +// the batch query said it was", which catches a worker settling it to +// anything at all in between. func (e *Engine) sendRecoveredDeliveries( ctx context.Context, + webhookDB *gorm.DB, deliveries []database.Delivery, webhookID string, targetMap map[string]database.Target, + settled map[string]struct{}, ) { + // The attempt number continues each delivery's own history + // rather than restarting at 1. A recovered delivery may already + // have recorded attempts, and numbering the next one 1 again + // both collides in the event log and hands the retry path a + // backoff computed from the wrong attempt. + attempts := e.countAttemptsBatch(webhookDB, deliveries) + for i := range deliveries { select { case <-ctx.Done(): @@ -994,6 +1990,10 @@ func (e *Engine) sendRecoveredDeliveries( default: } + if _, ok := settled[deliveries[i].ID]; ok { + continue + } + target, ok := targetMap[deliveries[i].TargetID] if !ok { e.log.Error( @@ -1005,22 +2005,40 @@ func (e *Engine) sendRecoveredDeliveries( continue } + if !e.takeForRedispatch( + webhookDB, deliveries[i].ID, + database.DeliveryStatusPending, + ) { + continue + } + + // The body is read here, one delivery at a time and only for + // deliveries that are actually being sent, rather than + // preloaded across the whole batch. A batch is up to + // pendingSweepBatch rows at up to the 1 MB ingest cap, and + // most of a sweep's batch is refused by the gate above — so + // preloading would hold hundreds of megabytes per webhook per + // tick to build tasks it then discards. + event, err := e.loadEvent( + webhookDB, deliveries[i].EventID, + ) + if err != nil { + e.log.Error( + "failed to load event for recovered delivery", + "delivery_id", deliveries[i].ID, + "event_id", deliveries[i].EventID, + "error", err, + ) + e.inflight.release(deliveries[i].ID) + + continue + } + task := buildRecoveryTask( - &deliveries[i], webhookID, - &deliveries[i].Event, &target, 1, + &deliveries[i], webhookID, &event, &target, + attempts[deliveries[i].ID]+1, ) - select { - case e.deliveryCh <- task: - default: - e.log.Warn( - "delivery channel full during "+ - "recovery, remaining deliveries "+ - "will be recovered on next restart", - "delivery_id", deliveries[i].ID, - ) - - return - } + e.queueRecovered(e.deliveryCh, task) } } diff --git a/internal/delivery/engine_integration_test.go b/internal/delivery/engine_integration_test.go index 6b07c43..90d028e 100644 --- a/internal/delivery/engine_integration_test.go +++ b/internal/delivery/engine_integration_test.go @@ -2,7 +2,6 @@ package delivery_test import ( "context" - "database/sql" "encoding/json" "fmt" "io" @@ -70,11 +69,12 @@ func iMainDB(t *testing.T) *gorm.DB { t.TempDir(), "main-test.db", ) - dsn := fmt.Sprintf( - "file:%s?cache=shared&mode=rwc", dbPath, + // Opened the way the service opens the main database, so these + // tests cannot pass against journal and locking settings + // production does not use. + sqlDB, err := database.OpenSQLite( + dbPath, database.SQLiteModeCreate, ) - - sqlDB, err := sql.Open("sqlite", dsn) require.NoError(t, err) t.Cleanup(func() { _ = sqlDB.Close() }) @@ -377,6 +377,17 @@ func TestProcessRetryTask_SuccessfulRetry(t *testing.T) { bodyStr := event.Body cfg := iHTTPConfig(ts.URL) + + // The target row exists because the engine confirms a scheduled + // retry's target has not been deleted before it runs it. A retry + // task whose target id names no row at all is a state the service + // does not produce: the handler read that target to build the + // task. See https://git.eeqj.de/sneak/webhooker/issues/107. + iCreateTarget( + t, s.MainDB, targetID, s.WebhookID, "retry-target", + database.TargetTypeHTTP, cfg, 5, + ) + task := iTask( d, event, s.WebhookID, targetID, "retry-target", cfg, 5, 2, &bodyStr, @@ -456,6 +467,12 @@ func TestProcessRetryTask_LargeBody_FetchFromDB( ) cfg := iHTTPConfig(ts.URL) + + iCreateTarget( + t, s.MainDB, targetID, s.WebhookID, "retry-large", + database.TargetTypeHTTP, cfg, 5, + ) + task := iTask( d, event, s.WebhookID, targetID, "retry-large", cfg, 5, 2, nil, @@ -476,7 +493,7 @@ func TestWorkerLifecycle_StartStop(t *testing.T) { t.Parallel() s := newISetup(t) - s.Engine.ExportStart(context.Background()) + s.Engine.ExportStart() event := iSeedEvent( t, s.WebhookDB, s.WebhookID, @@ -499,21 +516,17 @@ func TestWorkerLifecycle_StartStop(t *testing.T) { s.Engine.Notify([]delivery.Task{task}) - iWaitForStatus( - t, s.WebhookDB, d.ID, - database.DeliveryStatusDelivered, - ) + iWaitForDelivered(t, s.WebhookDB, d.ID) - s.Engine.ExportStop() + require.NoError(t, s.Engine.ExportStop(context.Background())) } -// iWaitForStatus polls until the delivery reaches the -// expected status. -func iWaitForStatus( +// iWaitForDelivered polls until the delivery reaches the +// delivered status. +func iWaitForDelivered( t *testing.T, db *gorm.DB, deliveryID string, - expected database.DeliveryStatus, ) { t.Helper() @@ -527,7 +540,7 @@ func iWaitForStatus( return false } - return d.Status == expected + return d.Status == database.DeliveryStatusDelivered }, 5*time.Second, 50*time.Millisecond) } @@ -558,10 +571,16 @@ func TestWorkerLifecycle_ProcessesRetryChannel( database.DeliveryStatusRetrying, ) - s.Engine.ExportStart(context.Background()) + s.Engine.ExportStart() bodyStr := event.Body cfg := iHTTPConfig(ts.URL) + + iCreateTarget( + t, s.MainDB, targetID, s.WebhookID, "retry-chan-test", + database.TargetTypeHTTP, cfg, 5, + ) + task := iTask( d, event, s.WebhookID, targetID, "retry-chan-test", cfg, 5, 2, &bodyStr, @@ -569,12 +588,9 @@ func TestWorkerLifecycle_ProcessesRetryChannel( s.Engine.ExportRetryCh() <- task - iWaitForStatus( - t, s.WebhookDB, d.ID, - database.DeliveryStatusDelivered, - ) + iWaitForDelivered(t, s.WebhookDB, d.ID) - s.Engine.ExportStop() + require.NoError(t, s.Engine.ExportStop(context.Background())) } // --- processDelivery: unknown target type --- @@ -748,6 +764,269 @@ func TestRecoverWebhookDeliveries_RetryingDeliveries( case <-time.After(5 * time.Second): t.Fatal("expected retry task from recovery") } + + // Regression guard: a target that still supports retries + // must be rescheduled, never terminally failed, and must + // not gain a synthetic result row. + iAssertStatus( + t, s.WebhookDB, d.ID, + database.DeliveryStatusRetrying, + ) + + assert.Len(t, iResults(t, s.WebhookDB, d.ID), 1) +} + +// --- Retrying deliveries whose target type changed --- + +// iSeedRetryingWithType seeds a retrying delivery with one +// recorded failed attempt against a target of the given type, +// standing in for a target whose type was edited in the main +// database while the delivery was still retrying. +func iSeedRetryingWithType( + t *testing.T, + s iSetup, + targetType database.TargetType, +) string { + t.Helper() + + targetID := uuid.New().String() + + iCreateTarget(t, s.MainDB, targetID, + s.WebhookID, "mutated-target", targetType, + iHTTPConfig("http://example.com/hook"), 5, + ) + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, + `{"orphaned":"retry"}`, + ) + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusRetrying, + ) + + iSeedFailedResult(t, s.WebhookDB, d.ID) + + return d.ID +} + +// iResults loads a delivery's results in attempt order. +func iResults( + t *testing.T, db *gorm.DB, deliveryID string, +) []database.DeliveryResult { + t.Helper() + + var results []database.DeliveryResult + + require.NoError(t, db. + Where("delivery_id = ?", deliveryID). + Order("attempt_num"). + Find(&results).Error) + + return results +} + +// iAssertTerminallyFailed asserts the delivery ended failed +// with a result row recording why, and was not rescheduled. +func iAssertTerminallyFailed( + t *testing.T, + s iSetup, + deliveryID string, + targetType database.TargetType, +) { + t.Helper() + + iAssertStatus( + t, s.WebhookDB, deliveryID, + database.DeliveryStatusFailed, + ) + + results := iResults(t, s.WebhookDB, deliveryID) + require.Len(t, results, 2) + + last := results[1] + + assert.False(t, last.Success) + assert.Equal(t, 2, last.AttemptNum) + + assert.Contains( + t, last.Error, string(targetType), + ) + + assert.Contains( + t, last.Error, "does not support retries", + ) + + assert.Empty(t, s.Engine.ExportRetryCh()) +} + +func TestRecoverSingleRetry_TypeNoLongerRetries( + t *testing.T, +) { + t.Parallel() + + s := newISetup(t) + + iCreateWebhook( + t, s.MainDB, s.WebhookID, "mutated-type", + ) + + deliveryID := iSeedRetryingWithType( + t, s, database.TargetTypeLog, + ) + + s.Engine.ExportRecoverWebhookDeliveries( + context.Background(), s.WebhookID, + ) + + iAssertTerminallyFailed( + t, s, deliveryID, database.TargetTypeLog, + ) +} + +func TestSweepSingleRetry_TypeNoLongerRetries( + t *testing.T, +) { + t.Parallel() + + s := newISetup(t) + + iCreateWebhook( + t, s.MainDB, s.WebhookID, "mutated-type-sweep", + ) + + deliveryID := iSeedRetryingWithType( + t, s, database.TargetTypeDatabase, + ) + + s.Engine.ExportSweepWebhookRetries( + context.Background(), s.WebhookID, + ) + + iAssertTerminallyFailed( + t, s, deliveryID, database.TargetTypeDatabase, + ) +} + +// TestFailUnretryableRetry_WritesNoTargetRow proves the +// orphaned-retry terminal path leaves no target row — and so no +// plaintext target config — in the per-webhook event database. +// +// That path loads the delivery without its Target relation on +// purpose. Populating d.Target makes GORM's SaveBeforeAssociations +// upsert the whole target row on the status UPDATE, which for a slack +// target writes the incoming-webhook credential into events-*.db. +// See https://git.eeqj.de/sneak/webhooker/issues/206. +func TestFailUnretryableRetry_WritesNoTargetRow( + t *testing.T, +) { + t.Parallel() + + s := newISetup(t) + + iCreateWebhook( + t, s.MainDB, s.WebhookID, "no-target-row", + ) + + targetID := uuid.New().String() + + // A Slack incoming-webhook URL: the target config IS the + // credential, which is what makes a leaked target row a + // disclosure rather than a curiosity. + hookURL := "https://hooks.slack.com/services/T00/B00/x" + + iCreateTarget(t, s.MainDB, targetID, + s.WebhookID, "credential-bearing", + database.TargetTypeLog, iHTTPConfig(hookURL), 5, + ) + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"orphaned":"retry"}`, + ) + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusRetrying, + ) + + iSeedFailedResult(t, s.WebhookDB, d.ID) + + s.Engine.ExportSweepWebhookRetries( + context.Background(), s.WebhookID, + ) + + iAssertStatus(t, s.WebhookDB, d.ID, + database.DeliveryStatusFailed, + ) + + // The table exists in the per-webhook database because GORM + // migrates the Delivery relation's model alongside it. It must + // stay empty. + var targetRows int64 + + require.NoError(t, s.WebhookDB. + Table("targets"). + Count(&targetRows).Error) + + assert.Zero(t, targetRows, + "orphaned-retry terminal failure wrote a target row "+ + "into the per-webhook event database", + ) + + var configs []string + + require.NoError(t, s.WebhookDB. + Table("targets"). + Pluck("config", &configs).Error) + + assert.NotContains( + t, strings.Join(configs, " "), hookURL, + ) +} + +func TestRecoverSingleRetry_UnknownTargetType( + t *testing.T, +) { + t.Parallel() + + s := newISetup(t) + + iCreateWebhook( + t, s.MainDB, s.WebhookID, "unknown-type", + ) + + unknown := database.TargetType("not-a-target-type") + + deliveryID := iSeedRetryingWithType(t, s, unknown) + + s.Engine.ExportRecoverWebhookDeliveries( + context.Background(), s.WebhookID, + ) + + iAssertTerminallyFailed(t, s, deliveryID, unknown) +} + +func TestSweepSingleRetry_UnknownTargetType( + t *testing.T, +) { + t.Parallel() + + s := newISetup(t) + + iCreateWebhook( + t, s.MainDB, s.WebhookID, "unknown-type-sweep", + ) + + unknown := database.TargetType("not-a-target-type") + + deliveryID := iSeedRetryingWithType(t, s, unknown) + + s.Engine.ExportSweepWebhookRetries( + context.Background(), s.WebhookID, + ) + + iAssertTerminallyFailed(t, s, deliveryID, unknown) } // iSeedFailedResult creates a failed delivery result. @@ -945,6 +1224,83 @@ func TestDeliverHTTP_TargetTimeout(t *testing.T) { iAssertResultFailed(t, db, del.ID) } +// TestDeliverHTTP_CutsStoredResponseAtMaxBodyLog pins the size +// this engine stores for an oversized response, because the +// event log's redaction is written against it: the row holds +// exactly maxBodyLog bytes and records nothing about how much +// more the remote sent, so a credential echoed across that +// boundary reaches the database already severed and no reader +// of the row can tell the cut happened. +func TestDeliverHTTP_CutsStoredResponseAtMaxBodyLog( + t *testing.T, +) { + t.Parallel() + + // Padded so the cut falls five bytes before the end of the + // echoed webhook URL. + const ( + severedTail = 5 + overshoot = 100000 + ) + + sent := strings.Repeat( + "A", + delivery.ExportMaxBodyLog-len(slackWebhookURL)+ + severedTail, + ) + slackWebhookURL + strings.Repeat("Z", overshoot) + + s := newISetup(t) + + ts := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadGateway) + _, _ = io.WriteString(w, sent) + }, + )) + defer ts.Close() + + cfgJSON := iHTTPConfig(ts.URL) + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"cut":"test"}`, + ) + targetID := uuid.New().String() + + del := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + + bodyStr := event.Body + task := iTask( + del, event, s.WebhookID, targetID, + "cut-target", cfgJSON, 0, 1, &bodyStr, + ) + + s.Engine.ExportProcessNewTask(context.TODO(), &task) + + results := iResults(t, s.WebhookDB, del.ID) + require.Len(t, results, 1) + + stored := results[0].ResponseBody + + assert.Len( + t, stored, delivery.ExportMaxBodyLog, + "an oversized response is stored at exactly the cap", + ) + assert.Equal( + t, sent[:delivery.ExportMaxBodyLog], stored, + ) + assert.NotContains( + t, stored, slackWebhookURL, + "the echoed URL is severed by the cut", + ) + assert.Contains( + t, stored, "T00000000", + "the severed prefix still carries the credential", + ) +} + // iSeedEventAndDelivery creates event + delivery // for standalone tests. func iSeedEventAndDelivery( diff --git a/internal/delivery/engine_lifecycle_test.go b/internal/delivery/engine_lifecycle_test.go new file mode 100644 index 0000000..6e7ef2e --- /dev/null +++ b/internal/delivery/engine_lifecycle_test.go @@ -0,0 +1,271 @@ +package delivery_test + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "go.uber.org/fx" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" +) + +const ( + // hookStopTimeout bounds how long a lifecycle test waits for + // the engine's OnStop hook to return before declaring the + // shutdown hung. + hookStopTimeout = 10 * time.Second + + // hookSettleDelay is how long startEngineViaHook waits after + // OnStart before the caller may enqueue work. A worker pool + // wrongly rooted in the already-done hook context has nothing + // but ctx.Done() ready in its select, so it is deterministically + // gone by the end of this window. Without the wait, Notify would + // race the pool's very first select, in which a ready ctx.Done() + // and a ready deliveryCh are chosen between at random and a + // doomed pool still delivers. + hookSettleDelay = 250 * time.Millisecond + + // wedgeStopTimeout is the stop timeout a wedged-shutdown test + // hands OnStop, standing in for fx's StopTimeout. The test + // asserts only that the hook returns at all, and allows it + // hookStopTimeout — forty times this budget — to do so, so no + // assertion here races the wall clock. + wedgeStopTimeout = 250 * time.Millisecond +) + +// recordingLifecycle is a minimal fx.Lifecycle that records the +// hooks a component registers, so a test can invoke the real +// OnStart/OnStop functions with a context of its choosing. +type recordingLifecycle struct { + hooks []fx.Hook +} + +func (l *recordingLifecycle) Append(h fx.Hook) { + l.hooks = append(l.hooks, h) +} + +// requireStopHookExpires drives hook.OnStop with a stop context +// that expires while a wedged goroutine is still running, and +// requires the hook to return the deadline error naming +// component instead of blocking on the WaitGroup forever. +func requireStopHookExpires( + t *testing.T, hook fx.Hook, component string, +) { + t.Helper() + + stopCtx, cancel := context.WithTimeout( + context.Background(), wedgeStopTimeout, + ) + defer cancel() + + var stopErr error + + stopped := make(chan struct{}) + + go func() { + defer close(stopped) + + stopErr = hook.OnStop(stopCtx) + }() + + select { + case <-stopped: + case <-time.After(hookStopTimeout): + t.Fatal( + "OnStop did not return: it discarded the stop " + + "context and is waiting on a wedged goroutine " + + "that will never observe cancellation", + ) + } + + require.ErrorIs(t, stopErr, context.DeadlineExceeded) + require.ErrorContains(t, stopErr, component) +} + +// startEngineViaHook drives the genuine fx hooks the application +// registers for the engine, handing OnStart a context that is +// already done, and returns only once a pool that inherited that +// context would have exited. It returns the recorded lifecycle so +// the caller can drive OnStop too. +// +// Callers must not seed pending or retrying deliveries before +// calling this: restart recovery enqueues those during startup, +// which would put work in the queue while the pool is still +// racing its first select. +func startEngineViaHook( + t *testing.T, eng *delivery.Engine, +) *recordingLifecycle { + t.Helper() + + lc := &recordingLifecycle{} + eng.ExportRegisterHooks(lc) + require.Len(t, lc.hooks, 1) + + // fx hands OnStart a context carrying the application start + // timeout, and cancels it when the start phase ends. An + // already-cancelled context is that same defect taken to its + // limit, and unlike a plain context.Background() it actually + // distinguishes a correctly rooted loop from a broken one. + hookCtx, cancel := context.WithCancel(context.Background()) + cancel() + + require.NoError(t, lc.hooks[0].OnStart(hookCtx)) + + time.Sleep(hookSettleDelay) + + return lc +} + +// seedLogTask seeds a pending delivery for a log target and +// returns its ID together with the task that drives it. The log +// target needs no network, so a delivery completing proves only +// that a worker picked the task up. +func seedLogTask( + t *testing.T, s iSetup, +) (string, delivery.Task) { + t.Helper() + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, + `{"lifecycle":"hook-context"}`, + ) + targetID := uuid.New().String() + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + + bodyStr := event.Body + task := iTask( + d, event, s.WebhookID, targetID, + "hook-context-test", "", 0, 1, &bodyStr, + ) + task.TargetType = database.TargetTypeLog + + return d.ID, task +} + +// TestEngine_WorkersOutliveStartHookContext is the regression +// test for a delivery engine that stopped delivering roughly +// fifteen seconds after boot. fx calls OnStart with a context +// carrying the application's start timeout (15s by default) and +// cancels it when the start phase ends, so a worker pool rooted +// in it exits shortly after startup: the process keeps accepting +// and persisting events while nothing at all forwards them. +// +// Driving OnStart with an already-cancelled context is that +// defect taken to its limit. A pool that inherits the hook +// context is gone before the task is even enqueued; a correctly +// rooted pool keeps working for as long as the process lives. +func TestEngine_WorkersOutliveStartHookContext(t *testing.T) { + t.Parallel() + + s := newISetup(t) + + lc := startEngineViaHook(t, s.Engine) + t.Cleanup(func() { + _ = lc.hooks[0].OnStop(context.Background()) + }) + + // Seeded only after the pool has settled, so restart recovery + // cannot enqueue it during startup. + deliveryID, task := seedLogTask(t, s) + + s.Engine.Notify([]delivery.Task{task}) + + iWaitForDelivered(t, s.WebhookDB, deliveryID) +} + +// TestEngine_StopHookStopsWorkers proves the fix did not trade a +// startup bug for a shutdown hang: now that the worker pool no +// longer observes the start hook's cancellation, OnStop is the +// only thing that can stop it, and it must both return promptly +// and actually leave the pool drained. +func TestEngine_StopHookStopsWorkers(t *testing.T) { + t.Parallel() + + s := newISetup(t) + + lc := startEngineViaHook(t, s.Engine) + + // Let the pool prove it is running before stopping it, so a + // fast OnStop cannot pass by stopping something already dead. + firstID, firstTask := seedLogTask(t, s) + s.Engine.Notify([]delivery.Task{firstTask}) + iWaitForDelivered(t, s.WebhookDB, firstID) + + var stopErr error + + stopped := make(chan struct{}) + + go func() { + defer close(stopped) + + // stop blocks on the workers' WaitGroup, so returning at + // all proves every goroutine observed the cancellation. + stopErr = lc.hooks[0].OnStop(context.Background()) + }() + + select { + case <-stopped: + case <-time.After(hookStopTimeout): + t.Fatal( + "OnStop did not return: the delivery engine's " + + "WaitGroup is still waiting on a goroutine that " + + "never observed cancellation", + ) + } + + require.NoError(t, stopErr) + + // With every worker gone, a freshly notified task must sit + // untouched in the queue rather than being delivered. + secondID, secondTask := seedLogTask(t, s) + s.Engine.Notify([]delivery.Task{secondTask}) + + time.Sleep(200 * time.Millisecond) + + var after database.Delivery + + require.NoError( + t, + s.WebhookDB.First(&after, "id = ?", secondID).Error, + ) + require.Equal( + t, + database.DeliveryStatusPending, + after.Status, + "a stopped engine must not deliver anything", + ) +} + +// TestEngine_StopHookHonoursStopTimeout is the regression test +// for a shutdown that could never complete. fx hands OnStop a +// context carrying the application's stop timeout; an OnStop +// that discards it and calls wg.Wait() bare hangs the process +// forever on a single worker stuck inside a delivery target that +// never returns — precisely when a bounded shutdown matters +// most. +// +// The wedged goroutine here never observes cancellation, so the +// hook can only return by honouring its context, and it must say +// so rather than reporting a clean stop. +func TestEngine_StopHookHonoursStopTimeout(t *testing.T) { + t.Parallel() + + s := newISetup(t) + + lc := startEngineViaHook(t, s.Engine) + + release := make(chan struct{}) + + t.Cleanup(func() { close(release) }) + + s.Engine.ExportWedgeWorker(release) + + requireStopHookExpires(t, lc.hooks[0], "delivery engine") +} diff --git a/internal/delivery/engine_test.go b/internal/delivery/engine_test.go index 2376de7..6c2e0a2 100644 --- a/internal/delivery/engine_test.go +++ b/internal/delivery/engine_test.go @@ -3,7 +3,6 @@ package delivery_test import ( "bytes" "context" - "database/sql" "encoding/json" "fmt" "log/slog" @@ -37,11 +36,12 @@ func testWebhookDB(t *testing.T) *gorm.DB { t.TempDir(), "events-test.db", ) - dsn := fmt.Sprintf( - "file:%s?cache=shared&mode=rwc", dbPath, + // Opened the way the service opens a per-webhook database, so + // these tests cannot pass against journal and locking settings + // production does not use. + sqlDB, err := database.OpenSQLite( + dbPath, database.SQLiteModeCreate, ) - - sqlDB, err := sql.Open("sqlite", dsn) require.NoError(t, err) t.Cleanup(func() { _ = sqlDB.Close() }) diff --git a/internal/delivery/event_db_isolation_test.go b/internal/delivery/event_db_isolation_test.go new file mode 100644 index 0000000..4bcd183 --- /dev/null +++ b/internal/delivery/event_db_isolation_test.go @@ -0,0 +1,165 @@ +package delivery_test + +import ( + "context" + "database/sql" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + _ "modernc.org/sqlite" + "sneak.berlin/go/webhooker/internal/database" +) + +// assertNoTargetRows opens the per-webhook database file directly, +// outside GORM, and fails if its targets table holds any rows. +// Target config is the credential for slack and http targets, and +// event databases are the files that get backed up and handed +// around. +func assertNoTargetRows(t *testing.T, dbPath string) { + t.Helper() + + sqlDB, err := sql.Open( + "sqlite", fmt.Sprintf("file:%s?mode=ro", dbPath), + ) + require.NoError(t, err) + + defer func() { _ = sqlDB.Close() }() + + var tables int + + require.NoError(t, sqlDB.QueryRowContext( + t.Context(), + "SELECT count(*) FROM sqlite_master "+ + "WHERE type = 'table' AND name = 'targets'", + ).Scan(&tables)) + + if tables == 0 { + return + } + + var rows int + + require.NoError(t, sqlDB.QueryRowContext( + t.Context(), "SELECT count(*) FROM targets", + ).Scan(&rows)) + + assert.Zero( + t, rows, + "per-webhook event database must hold no target rows", + ) +} + +// TestEventDBHoldsNoTargetRows drives a delivery and then a retry +// through the real engine write paths and asserts neither leaves a +// target row behind in events-*.db. +func TestEventDBHoldsNoTargetRows(t *testing.T) { + t.Parallel() + + s := newISetup(t) + + ts := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }, + )) + defer ts.Close() + + cfg := iHTTPConfig(ts.URL) + targetID := uuid.New().String() + dbPath := s.DBMgr.DBPath(s.WebhookID) + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"leak":"none"}`, + ) + body := event.Body + + // A new delivery. + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + task := iTask( + d, event, s.WebhookID, targetID, + "leaky-target", cfg, 5, 1, &body, + ) + + s.Engine.ExportProcessNewTask(context.TODO(), &task) + + iAssertStatus( + t, s.WebhookDB, d.ID, + database.DeliveryStatusDelivered, + ) + assertNoTargetRows(t, dbPath) + + // A retry. Its target exists in the main database, because the + // engine confirms a scheduled retry's target has not been + // deleted before running it; see + // https://git.eeqj.de/sneak/webhooker/issues/107. + iCreateTarget( + t, s.MainDB, targetID, s.WebhookID, "leaky-target", + database.TargetTypeHTTP, cfg, 5, + ) + + rd := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusRetrying, + ) + rTask := iTask( + rd, event, s.WebhookID, targetID, + "leaky-target", cfg, 5, 2, &body, + ) + + s.Engine.ExportProcessRetryTask(context.TODO(), &rTask) + + iAssertStatus( + t, s.WebhookDB, rd.ID, + database.DeliveryStatusDelivered, + ) + assertNoTargetRows(t, dbPath) +} + +// TestEventDBHoldsNoTargetRowsOnFailedDelivery covers the failure +// write path, which updates the delivery to failed and records a +// result, rather than the success path above. +func TestEventDBHoldsNoTargetRowsOnFailedDelivery(t *testing.T) { + t.Parallel() + + s := newISetup(t) + + ts := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + }, + )) + defer ts.Close() + + cfg := iHTTPConfig(ts.URL) + targetID := uuid.New().String() + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"leak":"none"}`, + ) + body := event.Body + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + task := iTask( + d, event, s.WebhookID, targetID, + "leaky-target", cfg, 0, 1, &body, + ) + + s.Engine.ExportProcessNewTask(context.TODO(), &task) + + iAssertStatus( + t, s.WebhookDB, d.ID, + database.DeliveryStatusFailed, + ) + assertNoTargetRows(t, s.DBMgr.DBPath(s.WebhookID)) +} diff --git a/internal/delivery/event_timestamp_test.go b/internal/delivery/event_timestamp_test.go new file mode 100644 index 0000000..09e70cf --- /dev/null +++ b/internal/delivery/event_timestamp_test.go @@ -0,0 +1,442 @@ +package delivery_test + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" +) + +// tsEventCreatedAt is the receipt time seeded on the events these +// tests deliver. It is far enough from both the zero time and from +// now that neither can be mistaken for it. +func tsEventCreatedAt() time.Time { + return time.Date( + 2026, time.March, 4, 5, 6, 7, 0, time.UTC, + ) +} + +// tsZeroStamp is what a Slack message renders when the event handed +// to FormatSlackMessage carries no CreatedAt. +const tsZeroStamp = "*Timestamp:* `0001-01-01T00:00:00Z`" + +// tsEventBody is the body seeded on every event in this file. It is +// small enough that a Task can inline it. +const tsEventBody = `{"hello":"world"}` + +// tsUndeliverableHook stands in for a Slack incoming webhook on the +// tests that never send: the config parser requires a URL, but no +// request is made. +const tsUndeliverableHook = "https://hooks.slack.com/services/T/B/x" + +// tsSink is a stand-in Slack incoming webhook that records the raw +// body posted to it. +type tsSink struct { + *httptest.Server + + bodies chan []byte +} + +func newTSSink(t *testing.T) *tsSink { + t.Helper() + + s := &tsSink{bodies: make(chan []byte, 8)} + + s.Server = httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + + select { + case s.bodies <- body: + default: + } + + w.WriteHeader(http.StatusOK) + }, + )) + + t.Cleanup(s.Close) + + return s +} + +// text returns the Slack message text from the single payload the +// sink received. +func (s *tsSink) text(t *testing.T) string { + t.Helper() + + select { + case raw := <-s.bodies: + t.Logf("raw slack payload: %s", raw) + + var payload struct { + Text string `json:"text"` + } + + require.NoError(t, json.Unmarshal(raw, &payload)) + + return payload.Text + case <-time.After(5 * time.Second): + t.Fatal("slack sink received no payload") + + return "" + } +} + +func tsSlackConfig(t *testing.T, url string) string { + t.Helper() + + data, err := json.Marshal( + delivery.SlackTargetConfig{WebhookURL: url}, + ) + require.NoError(t, err) + + return string(data) +} + +// tsSeedEvent writes an event whose CreatedAt is tsEventCreatedAt +// rather than the write time, so an assertion on the rendered +// timestamp cannot pass by accident against "roughly now". +func tsSeedEvent( + t *testing.T, db *gorm.DB, webhookID string, +) database.Event { + t.Helper() + + event := database.Event{ + WebhookID: webhookID, + EntrypointID: uuid.New().String(), + Method: http.MethodPost, + Headers: `{}`, + Body: tsEventBody, + ContentType: "application/json", + } + event.ID = uuid.New().String() + event.CreatedAt = tsEventCreatedAt() + event.UpdatedAt = tsEventCreatedAt() + + require.NoError(t, db.Create(&event).Error) + + var stored database.Event + + require.NoError(t, + db.First(&stored, "id = ?", event.ID).Error, + ) + require.Equal(t, + tsEventCreatedAt().UTC(), stored.CreatedAt.UTC(), + "seeded created_at did not round-trip", + ) + + return event +} + +// tsSeedTarget writes the slack target row into the main database. +// The retry path confirms the target still exists before sending. +func tsSeedTarget( + t *testing.T, mainDB *gorm.DB, webhookID, config string, +) database.Target { + t.Helper() + + target := database.Target{ + WebhookID: webhookID, + Name: "slack-sink", + Type: database.TargetTypeSlack, + Config: config, + Active: true, + } + + require.NoError(t, mainDB.Create(&target).Error) + + return target +} + +func tsTask( + d database.Delivery, + event database.Event, + webhookID string, + target database.Target, + attemptNum int, + body *string, +) delivery.Task { + return delivery.Task{ + DeliveryID: d.ID, + EventID: event.ID, + WebhookID: webhookID, + EntrypointID: event.EntrypointID, + TargetID: target.ID, + TargetName: target.Name, + TargetType: database.TargetTypeSlack, + TargetConfig: target.Config, + MaxRetries: 0, + Method: event.Method, + Headers: event.Headers, + ContentType: event.ContentType, + Body: body, + AttemptNum: attemptNum, + } +} + +func tsAssertRealTimestamp(t *testing.T, text string) { + t.Helper() + + assert.NotContains(t, text, tsZeroStamp, + "slack message carries the zero timestamp", + ) + assert.Contains(t, text, + "*Timestamp:* `"+ + tsEventCreatedAt().UTC().Format(time.RFC3339)+"`", + "slack message does not carry the event's receipt time", + ) +} + +// tsCase is one end-to-end delivery of a seeded event to a slack +// sink, over whichever engine path `process` names. +type tsCase struct { + // status is the delivery row's status before the engine runs. + // The retry path refuses a delivery that is not retrying. + status database.DeliveryStatus + + // inlineBody mirrors a Task built for a body under + // MaxInlineBodySize. When false the engine reads the body back + // from the stored row. + inlineBody bool + + attemptNum int + + process func( + ctx context.Context, e *delivery.Engine, task *delivery.Task, + ) +} + +// run delivers one event through the named path and returns the +// Slack message text the sink received. +func (c tsCase) run(t *testing.T) (iSetup, database.Delivery, string) { + t.Helper() + + s := newISetup(t) + sink := newTSSink(t) + + cfg := tsSlackConfig(t, sink.URL) + target := tsSeedTarget(t, s.MainDB, s.WebhookID, cfg) + event := tsSeedEvent(t, s.WebhookDB, s.WebhookID) + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, target.ID, c.status, + ) + + var body *string + + if c.inlineBody { + bodyStr := event.Body + body = &bodyStr + } + + task := tsTask( + d, event, s.WebhookID, target, c.attemptNum, body, + ) + + c.process(context.TODO(), s.Engine, &task) + + return s, d, sink.text(t) +} + +// TestSlackFirstAttemptCarriesEventTimestamp covers the path an +// event takes on its first delivery: the task comes from the +// receiver and the engine reconstructs the event from it. +func TestSlackFirstAttemptCarriesEventTimestamp(t *testing.T) { + t.Parallel() + + s, d, text := tsCase{ + status: database.DeliveryStatusPending, + inlineBody: true, + attemptNum: 1, + process: func( + ctx context.Context, + e *delivery.Engine, + task *delivery.Task, + ) { + e.ExportProcessNewTask(ctx, task) + }, + }.run(t) + + tsAssertRealTimestamp(t, text) + + iAssertStatus(t, s.WebhookDB, d.ID, + database.DeliveryStatusDelivered, + ) +} + +// TestSlackFirstAttemptLargeBodyCarriesEventTimestamp covers the +// first-attempt path for an event whose body exceeded +// MaxInlineBodySize, so the task carries no body and the engine +// reads it back from the stored row. +func TestSlackFirstAttemptLargeBodyCarriesEventTimestamp( + t *testing.T, +) { + t.Parallel() + + _, _, text := tsCase{ + status: database.DeliveryStatusPending, + inlineBody: false, + attemptNum: 1, + process: func( + ctx context.Context, + e *delivery.Engine, + task *delivery.Task, + ) { + e.ExportProcessNewTask(ctx, task) + }, + }.run(t) + + tsAssertRealTimestamp(t, text) +} + +// TestSlackRetryCarriesEventTimestamp covers the retry path, which +// reconstructs the event from the same task the first attempt used. +func TestSlackRetryCarriesEventTimestamp(t *testing.T) { + t.Parallel() + + s, d, text := tsCase{ + status: database.DeliveryStatusRetrying, + inlineBody: true, + attemptNum: 2, + process: func( + ctx context.Context, + e *delivery.Engine, + task *delivery.Task, + ) { + e.ExportProcessRetryTask(ctx, task) + }, + }.run(t) + + tsAssertRealTimestamp(t, text) + + iAssertStatus(t, s.WebhookDB, d.ID, + database.DeliveryStatusDelivered, + ) +} + +// TestFormatSlackMessageOverTaskReconstructedEvent asserts on the +// formatted message directly, over the event the delivery paths +// reconstruct from a Task. It is the unit-level guard under the +// end-to-end tests: revert the CreatedAt population in hydrateEvent +// and this fails on the zero timestamp. +func TestFormatSlackMessageOverTaskReconstructedEvent( + t *testing.T, +) { + t.Parallel() + + s := newISetup(t) + + cfg := tsSlackConfig(t, tsUndeliverableHook) + target := tsSeedTarget(t, s.MainDB, s.WebhookID, cfg) + event := tsSeedEvent(t, s.WebhookDB, s.WebhookID) + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, target.ID, + database.DeliveryStatusPending, + ) + + bodyStr := event.Body + task := tsTask(d, event, s.WebhookID, target, 1, &bodyStr) + + rebuilt, err := s.Engine.ExportEventForTask( + s.WebhookDB, &task, + ) + require.NoError(t, err) + assert.False(t, rebuilt.CreatedAt.IsZero(), + "reconstructed event carries the zero time", + ) + assert.Equal(t, + tsEventCreatedAt().UTC(), rebuilt.CreatedAt.UTC(), + ) + + tsAssertRealTimestamp( + t, delivery.FormatSlackMessage(&rebuilt), + ) +} + +// TestFormatSlackMessageZeroTimestamp asserts the rendering choice +// directly, without going through the engine: a zero CreatedAt (the +// shape a reaped-row fallback produces) renders as "unknown" rather +// than the year-1 zero time, while a real CreatedAt still renders as +// RFC3339. +func TestFormatSlackMessageZeroTimestamp(t *testing.T) { + t.Parallel() + + zeroEvent := database.Event{ + Method: http.MethodPost, + ContentType: testContentType, + Body: tsEventBody, + } + + zeroText := delivery.FormatSlackMessage(&zeroEvent) + + assert.NotContains(t, zeroText, "0001-01-01", + "slack message carries the zero-time year", + ) + assert.Contains(t, zeroText, "*Timestamp:* `unknown`", + "slack message does not mark an unset receipt time as unknown", + ) + + nonZeroEvent := zeroEvent + nonZeroEvent.CreatedAt = tsEventCreatedAt() + + nonZeroText := delivery.FormatSlackMessage(&nonZeroEvent) + + assert.Contains(t, nonZeroText, + "*Timestamp:* `"+ + tsEventCreatedAt().UTC().Format(time.RFC3339)+"`", + "slack message does not render a real receipt time as RFC3339", + ) +} + +// TestEventReconstructionSurvivesAReapedRow pins the fallback: an +// event row reaped by retention while its delivery still holds the +// body inline is still delivered, with the receipt time unset, +// rather than dropped. +func TestEventReconstructionSurvivesAReapedRow(t *testing.T) { + t.Parallel() + + s := newISetup(t) + + cfg := tsSlackConfig(t, tsUndeliverableHook) + target := tsSeedTarget(t, s.MainDB, s.WebhookID, cfg) + event := tsSeedEvent(t, s.WebhookDB, s.WebhookID) + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, target.ID, + database.DeliveryStatusPending, + ) + + bodyStr := event.Body + task := tsTask(d, event, s.WebhookID, target, 1, &bodyStr) + + require.NoError(t, s.WebhookDB.Unscoped().Delete( + &database.Event{}, "id = ?", event.ID, + ).Error) + + rebuilt, err := s.Engine.ExportEventForTask( + s.WebhookDB, &task, + ) + require.NoError(t, err) + assert.Equal(t, bodyStr, rebuilt.Body) + assert.True(t, rebuilt.CreatedAt.IsZero()) + + // A task with no inlined body has nothing left to deliver, so + // the same reaped row is an error there. + noBody := task + noBody.Body = nil + + _, err = s.Engine.ExportEventForTask(s.WebhookDB, &noBody) + require.Error(t, err) +} diff --git a/internal/delivery/export_test.go b/internal/delivery/export_test.go index 739eb03..b71de96 100644 --- a/internal/delivery/export_test.go +++ b/internal/delivery/export_test.go @@ -5,18 +5,39 @@ import ( "log/slog" "net" "net/http" + "net/netip" + "net/url" "time" + "go.uber.org/fx" "gorm.io/gorm" "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/metrics" ) +// ErrExportArchiveWriterEvicted exposes the sentinel returned by +// an evicted archive writer. It carries the Err prefix rather +// than this file's usual Export one because it is a sentinel +// error. +var ErrExportArchiveWriterEvicted = errArchiveWriterEvicted + // Exported constants for test access. const ( ExportDeliveryChannelSize = deliveryChannelSize ExportRetryChannelSize = retryChannelSize ExportDefaultFailureThreshold = defaultFailureThreshold ExportDefaultCooldown = defaultCooldown + + // ExportMaxBodyLog is the cap the engine applies to a + // recorded response body. The event log's handling of a cut + // response is written against this number, so a test has to + // be able to name it. + ExportMaxBodyLog = maxBodyLog + + // ExportPendingSweepMinAge is how long a delivery must sit at + // pending before the sweep treats it as stranded. A test has to + // name it to age a row past the bound. + ExportPendingSweepMinAge = pendingSweepMinAge ) // ExportIsBlockedIP exposes isBlockedIP for testing. @@ -24,6 +45,26 @@ func ExportIsBlockedIP(ip net.IP) bool { return isBlockedIP(ip) } +// NewTestGuard builds an SSRF Guard from an explicit egress +// allowlist, without going through config. Passing no prefixes +// yields the default guard, which blocks every private/reserved +// range. +func NewTestGuard(allowed ...netip.Prefix) *Guard { + return &Guard{allowed: allowed} +} + +// ExportCheckIP exposes the guard's single decision point, so a +// test can assert the policy both the validator and the dialer +// inherit without needing a live destination. +func (g *Guard) ExportCheckIP(ip net.IP) error { + return g.checkIP(ip) +} + +// ExportAlwaysBlockedNetworks exposes alwaysBlockedNetworks. +func ExportAlwaysBlockedNetworks() []*net.IPNet { + return alwaysBlockedNetworks +} + // ExportBlockedNetworks exposes blockedNetworks. func ExportBlockedNetworks() []*net.IPNet { return blockedNetworks @@ -34,6 +75,17 @@ func ExportIsForwardableHeader(name string) bool { return isForwardableHeader(name) } +// ExportApplyRequestHeaders exposes applyRequestHeaders, so a test +// can inspect the header set an outbound delivery actually carries +// and the origin-scoped names it reports for the redirect policy. +func ExportApplyRequestHeaders( + req *http.Request, + event *database.Event, + cfg *HTTPTargetConfig, +) []string { + return applyRequestHeaders(req, event, cfg) +} + // ExportTruncate exposes truncate for testing. func ExportTruncate(s string, maxLen int) string { return truncate(s, maxLen) @@ -99,6 +151,16 @@ func (e *Engine) ExportProcessRetryTask( e.processRetryTask(ctx, task) } +// ExportEventForTask exposes the event reconstruction the delivery +// paths run: buildEventFromTask followed by hydrateEvent. +func (e *Engine) ExportEventForTask( + webhookDB *gorm.DB, task *Task, +) (database.Event, error) { + return e.hydrateEvent( + webhookDB, buildEventFromTask(task), task, + ) +} + // ExportProcessDelivery exposes processDelivery. func (e *Engine) ExportProcessDelivery( ctx context.Context, @@ -141,12 +203,27 @@ func (e *Engine) ExportDoHTTPRequest( return e.httpTarget.doHTTPRequest(ctx, cfg, event) } -// ExportClientForConfig exposes the http target's -// clientForConfig. -func (e *Engine) ExportClientForConfig( +// ExportClientForRequest exposes the http target's +// clientForRequest. +func (e *Engine) ExportClientForRequest( cfg *HTTPTargetConfig, + originScoped []string, ) *http.Client { - return e.httpTarget.clientForConfig(cfg) + return e.httpTarget.clientForRequest(cfg, originScoped) +} + +// ErrExportTooManyRedirects exposes the sentinel the redirect +// policy returns once a chain exceeds the hop cap. It carries the +// Err prefix rather than this file's usual Export one because it +// is a sentinel error. +var ErrExportTooManyRedirects = errTooManyRedirects + +// ExportMaxDeliveryRedirects exposes the redirect hop cap. +const ExportMaxDeliveryRedirects = maxDeliveryRedirects + +// ExportSameDeliveryOrigin exposes sameDeliveryOrigin. +func ExportSameDeliveryOrigin(origin, dest *url.URL) bool { + return sameDeliveryOrigin(origin, dest) } // ExportClient returns the http target's shared HTTP client. @@ -188,14 +265,60 @@ func (e *Engine) ExportRecoverInFlight( e.recoverInFlight(ctx) } +// ExportSweepWebhookRetries exposes sweepWebhookRetries. +func (e *Engine) ExportSweepWebhookRetries( + ctx context.Context, webhookID string, +) { + e.sweepWebhookRetries(ctx, webhookID) +} + // ExportStart exposes start for testing. -func (e *Engine) ExportStart(ctx context.Context) { - e.start(ctx) +func (e *Engine) ExportStart() { + e.start() +} + +// ExportRegisterHooks registers the engine's real fx lifecycle +// hooks on a lifecycle supplied by a test, so a test can drive +// the exact OnStart/OnStop functions the application runs and +// hand OnStart the kind of context fx actually supplies. +func (e *Engine) ExportRegisterHooks(lc fx.Lifecycle) { + e.registerHooks(lc) } // ExportStop exposes stop for testing. -func (e *Engine) ExportStop() { - e.stop() +func (e *Engine) ExportStop(ctx context.Context) error { + return e.stop(ctx) +} + +// ExportWedgeWorker adds a goroutine to the engine's WaitGroup +// that never observes cancellation and returns only when release +// is closed. It stands in for a worker stuck inside a delivery +// target that never returns, which is the only way stop can be +// made to outlast its context. +func (e *Engine) ExportWedgeWorker(release <-chan struct{}) { + e.wg.Go(func() { + <-release + }) +} + +// ExportInflightHeld reports how many deliveries the engine currently +// owns, so a test can prove ownership is released rather than leaked. +func (e *Engine) ExportInflightHeld() int { + return e.inflight.held() +} + +// ExportRetainDelivery takes the first reference on a delivery, as the +// queueing side does. It lets a test put a delivery into the state a +// worker or a full channel would, without running the pool. +func (e *Engine) ExportRetainDelivery(deliveryID string) bool { + return e.inflight.retainIdle(deliveryID) +} + +// ExportRecoverRetryingDeliveries exposes recoverRetryingDeliveries. +func (e *Engine) ExportRecoverRetryingDeliveries( + webhookDB *gorm.DB, webhookID string, +) { + e.recoverRetryingDeliveries(webhookDB, webhookID) } // ExportDeliveryCh returns the delivery channel. @@ -220,6 +343,7 @@ func NewTestEngine( deliveryCh: make(chan Task, deliveryChannelSize), retryCh: make(chan Task, retryChannelSize), workers: workers, + mtr: metrics.Default(), } e.initTargets(client) @@ -234,6 +358,7 @@ func NewTestEngineSmallRetry( e := &Engine{ log: log, retryCh: make(chan Task, 1), + mtr: metrics.Default(), } e.initTargets(nil) @@ -256,12 +381,25 @@ func NewTestEngineWithDB( deliveryCh: make(chan Task, deliveryChannelSize), retryCh: make(chan Task, retryChannelSize), workers: workers, + mtr: metrics.Default(), } e.initTargets(client) return e } +// ExportSetMetrics substitutes the engine's metric set, so a test can +// assert on collectors registered on a private registry instead of +// the process-wide ones every other test is also moving. +func (e *Engine) ExportSetMetrics(mtr *metrics.Set) { + e.mtr = mtr +} + +// ExportSampleQueueDepths runs one queue depth sample synchronously. +func (e *Engine) ExportSampleQueueDepths(ctx context.Context) { + e.sampleQueueDepths(ctx) +} + // NewTestCircuitBreaker creates a CircuitBreaker with // custom settings for testing. func NewTestCircuitBreaker( @@ -328,6 +466,194 @@ func (e *ExportArchiveWriter) DB() *gorm.DB { return e.w.db } +// Path returns the archive file the writer owns. +func (e *ExportArchiveWriter) Path() string { + return e.w.path +} + +// OpenExisting opens the archive without permitting creation, +// the way the idle sweep does. +func (e *ExportArchiveWriter) OpenExisting( + expiry time.Duration, +) error { + return e.w.openMode(archiveModeExisting, expiry) +} + +// SweepExpired runs an idle sweep of the archive. +func (e *ExportArchiveWriter) SweepExpired( + expiry time.Duration, +) error { + return e.w.sweepExpired(expiry) +} + +// Evict marks the writer evicted and closes its handle, exactly +// as leaving the registry does. +func (e *ExportArchiveWriter) Evict() { + e.w.evict() +} + +// HandleOpen reports whether the writer currently holds an open +// archive handle. +func (e *ExportArchiveWriter) HandleOpen() bool { + e.w.mu.Lock() + defer e.w.mu.Unlock() + + return e.w.db != nil +} + +// Same reports whether both wrappers refer to the very same +// underlying archive writer, so a test can prove a registry entry +// is the writer it was handed rather than a replacement. +func (e *ExportArchiveWriter) Same( + other *ExportArchiveWriter, +) bool { + return other != nil && e.w == other.w +} + +// ExportArchiveWriterFor returns the archive writer the registry +// currently caches for a webhook, or nil when none is cached. It +// never creates one, so a test can hold a reference to the very +// writer an eviction is about to detach. +func (e *Engine) ExportArchiveWriterFor( + webhookID string, +) *ExportArchiveWriter { + e.dbTarget.mu.Lock() + defer e.dbTarget.mu.Unlock() + + w, ok := e.dbTarget.writers[webhookID] + if !ok { + return nil + } + + return &ExportArchiveWriter{w: w} +} + +// ExportHasArchiveWriter reports whether the database target +// currently caches an archive writer for a webhook. +func (e *Engine) ExportHasArchiveWriter( + webhookID string, +) bool { + e.dbTarget.mu.Lock() + defer e.dbTarget.mu.Unlock() + + _, ok := e.dbTarget.writers[webhookID] + + return ok +} + +// ExportArchiveHandleOpen reports whether the cached archive +// writer for a webhook holds an open database handle. It +// returns false when no writer is cached. +func (e *Engine) ExportArchiveHandleOpen( + webhookID string, +) bool { + e.dbTarget.mu.Lock() + w, ok := e.dbTarget.writers[webhookID] + e.dbTarget.mu.Unlock() + + if !ok { + return false + } + + w.mu.Lock() + defer w.mu.Unlock() + + return w.db != nil +} + +// ExportEnsureArchiveWriter creates (if needed) and returns the +// archive file path of the cached writer for a webhook, so a +// test can prime the registry the way a delivery would. +func (e *Engine) ExportEnsureArchiveWriter( + webhookID string, +) (string, error) { + w, err := e.dbTarget.writerFor(webhookID) + if err != nil { + return "", err + } + + return w.path, nil +} + +// ExportSweepWriterFor takes a webhook's registry writer exactly +// as the idle sweep does, reporting whether the sweep had to +// create the entry. It lets a test drive the registry through the +// sweep's own entry point instead of choreographing goroutines. +func (e *Engine) ExportSweepWriterFor( + webhookID string, +) (*ExportArchiveWriter, bool, error) { + w, created, err := e.dbTarget.sweepWriterFor(webhookID) + if err != nil { + return nil, false, err + } + + return &ExportArchiveWriter{w: w}, created, nil +} + +// ExportReleaseSweepWriter releases a sweep-created registry entry +// exactly as a finished sweep does. +func (e *Engine) ExportReleaseSweepWriter( + webhookID string, w *ExportArchiveWriter, +) { + e.dbTarget.releaseSweepWriter(webhookID, w.w) +} + +// NewTestArchiveSweeper builds an ArchiveSweeper backed by the +// given main database and engine, without the fx lifecycle. +// Intended for tests. +func NewTestArchiveSweeper( + db *database.Database, + eng *Engine, + log *slog.Logger, +) *ArchiveSweeper { + return &ArchiveSweeper{ + db: db, + eng: eng, + log: log, + interval: time.Hour, + } +} + +// ExportSweep runs a single archive sweep synchronously for +// tests. +func (s *ArchiveSweeper) ExportSweep(ctx context.Context) { + s.sweep(ctx) +} + +// ExportStart starts the sweeper's background loop for tests. +func (s *ArchiveSweeper) ExportStart() { + s.start() +} + +// ExportRegisterHooks registers the sweeper's real fx lifecycle +// hooks on a lifecycle supplied by a test, so a test can drive +// the exact OnStart/OnStop functions the application runs and +// hand OnStart the kind of context fx actually supplies. +func (s *ArchiveSweeper) ExportRegisterHooks(lc fx.Lifecycle) { + s.registerHooks(lc) +} + +// ExportStop stops the sweeper's background loop for tests. +func (s *ArchiveSweeper) ExportStop(ctx context.Context) error { + return s.stop(ctx) +} + +// ExportWedgeLoop adds a goroutine to the sweeper's WaitGroup +// that never observes cancellation and returns only when release +// is closed. It stands in for a prune stuck on a locked archive. +func (s *ArchiveSweeper) ExportWedgeLoop( + release <-chan struct{}, +) { + s.wg.Go(func() { + <-release + }) +} + +// ExportSetInterval overrides the sweep interval for tests. +func (s *ArchiveSweeper) ExportSetInterval(d time.Duration) { + s.interval = d +} + // ExportParseArchiveExpiry exposes parseArchiveExpiry. func ExportParseArchiveExpiry( configJSON string, diff --git a/internal/delivery/inflight.go b/internal/delivery/inflight.go new file mode 100644 index 0000000..9596e30 --- /dev/null +++ b/internal/delivery/inflight.go @@ -0,0 +1,110 @@ +package delivery + +import "sync" + +// inflightSet records which deliveries the engine currently owns. +// +// A delivery is owned from the moment a task for it is handed to a +// channel or to a retry timer until the engine has no further plan for +// it in memory. Restart recovery and both arms of the periodic sweep +// re-dispatch only deliveries the set does not hold, which is what +// makes them exact rather than a guess about how long a row has sat at +// pending. +// +// This replaces reasoning from timestamps. A delivery's row says +// pending from creation until its outcome is written, which covers +// four different situations — never dispatched, waiting in a channel, +// being attempted right now, and genuinely stranded — and no column +// distinguishes them. Only the engine knows which, and it knows +// exactly. `deliveryChannelSize` is 10000 against 10 workers, so a +// perfectly healthy delivery can wait far longer than any age bound +// worth setting before its attempt even begins; an age bound alone +// re-sends it. See +// https://git.eeqj.de/sneak/webhooker/issues/256. +// +// In-memory state is sufficient because a data directory admits one +// process: internal/datadir takes an flock on it at startup and a +// second instance refuses to run. Deliveries owned by a process that +// died are not in any successor's set, and restart recovery is what +// picks those up. +// +// References are counted rather than held as a plain set because +// ownership outlives the worker that took it. A target that schedules +// a retry from inside Deliver adds a reference while the worker still +// holds one, so the delivery stays owned across the gap between the +// worker returning and the timer firing — the window in which a sweep +// would otherwise find the row at retrying and send it again. +// +// The zero value is ready to use, and the Engine holds one by value. +// That is deliberate: an engine built by a constructor that forgot to +// initialise this would not refuse to re-dispatch anything, and the +// symptom would be duplicate deliveries rather than a failure anybody +// notices. +type inflightSet struct { + mu sync.Mutex + ids map[string]int +} + +// retain adds a reference to a delivery the caller already knows the +// engine owns, so that ownership survives the current holder letting +// go. It cannot fail. +func (s *inflightSet) retain(deliveryID string) { + s.mu.Lock() + defer s.mu.Unlock() + + if s.ids == nil { + s.ids = make(map[string]int) + } + + s.ids[deliveryID]++ +} + +// retainIdle takes the first reference on a delivery, and reports +// whether it got it. It fails when the engine already owns the +// delivery, which is what makes two claimants — restart recovery and +// the sweep run concurrently, or two sweep arms — mutually exclusive +// rather than merely atomic. +func (s *inflightSet) retainIdle(deliveryID string) bool { + s.mu.Lock() + defer s.mu.Unlock() + + if s.ids[deliveryID] > 0 { + return false + } + + if s.ids == nil { + s.ids = make(map[string]int) + } + + s.ids[deliveryID] = 1 + + return true +} + +// release drops one reference. The delivery becomes eligible for +// re-dispatch again once the last one goes. +func (s *inflightSet) release(deliveryID string) { + s.mu.Lock() + defer s.mu.Unlock() + + n := s.ids[deliveryID] - 1 + if n <= 0 { + delete(s.ids, deliveryID) + + return + } + + s.ids[deliveryID] = n +} + +// held reports how many deliveries the engine currently owns. It +// exists so a test can assert that ownership is released rather than +// leaked: a reference that is never dropped hides its delivery from +// every sweep for the life of the process, which is the one way this +// mechanism can fail silently. +func (s *inflightSet) held() int { + s.mu.Lock() + defer s.mu.Unlock() + + return len(s.ids) +} diff --git a/internal/delivery/inflight_test.go b/internal/delivery/inflight_test.go new file mode 100644 index 0000000..1c86003 --- /dev/null +++ b/internal/delivery/inflight_test.go @@ -0,0 +1,428 @@ +package delivery_test + +import ( + "context" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" +) + +// These tests pin the rule that decides whether a delivery may be +// handed back to a worker: the engine re-dispatches only what it does +// not already own. Age alone is not that rule — a healthy delivery +// waiting in a 10000-deep channel is old and must not be re-sent. See +// https://git.eeqj.de/sneak/webhooker/issues/256. + +// fSweepSetup seeds the main database with the webhook row the sweep +// enumerates, and returns the setup. +func fSweepSetup( + t *testing.T, targetID, name string, +) iSetup { + t.Helper() + + s := newISetup(t) + + iCreateTarget(t, s.MainDB, targetID, + s.WebhookID, name, + database.TargetTypeLog, "", 0, + ) + + require.NoError(t, s.MainDB.Create(&database.Webhook{ + BaseModel: database.BaseModel{ID: s.WebhookID}, + UserID: uuid.New().String(), + Name: name, + }).Error) + + return s +} + +// fDrain collects every task the engine has queued. +// +// Every caller drives the dispatch paths synchronously and has already +// waited for them to return, so anything they queued is in the channel +// by now. The short grace covers nothing but scheduler jitter, and is +// kept small because one of these tests runs the drain forty times. +func fDrain(e *delivery.Engine) []delivery.Task { + var out []delivery.Task + + for { + select { + case task := <-e.ExportDeliveryCh(): + out = append(out, task) + case task := <-e.ExportRetryCh(): + out = append(out, task) + case <-time.After(25 * time.Millisecond): + return out + } + } +} + +// TestArchiveHandleIsWAL closes the last gap in the durability +// evidence: the main and per-webhook tiers each assert their journal +// mode on a live handle, and the archive tier gets its settings from +// the same code path but nothing checked the running file. +func TestArchiveHandleIsWAL(t *testing.T) { + t.Parallel() + + w := delivery.NewExportArchiveWriter( + filepath.Join(t.TempDir(), "archive-wal.db"), + archiveTestLogger(), 0, + ) + + require.NoError(t, w.Open(0)) + + var mode string + + row := w.DB().Raw("pragma journal_mode").Row() + require.NoError(t, row.Scan(&mode)) + assert.Equal(t, "wal", strings.ToLower(mode)) + + var busy string + + row = w.DB().Raw("pragma busy_timeout").Row() + require.NoError(t, row.Scan(&busy)) + assert.Equal(t, "10000", busy) +} + +// TestSweepLeavesAQueuedDeliveryAlone is the case the age bound cannot +// see. The delivery is queued and untouched, so its row is arbitrarily +// old and still perfectly healthy; only ownership distinguishes it +// from a stranded one. +func TestSweepLeavesAQueuedDeliveryAlone(t *testing.T) { + t.Parallel() + + targetID := uuid.New().String() + s := fSweepSetup(t, targetID, "queued") + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"queued":true}`, + ) + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + rAgePending(t, s.WebhookDB, d.ID) + + // Queued exactly as the receiver queues it, and never dequeued: + // no workers are running in this engine. + s.Engine.Notify([]delivery.Task{{ + DeliveryID: d.ID, + EventID: event.ID, + WebhookID: s.WebhookID, + TargetID: targetID, + }}) + + require.Equal(t, 1, s.Engine.ExportInflightHeld()) + + s.Engine.ExportSweepWebhookRetries( + context.Background(), s.WebhookID, + ) + + tasks := fDrain(s.Engine) + assert.Len( + t, tasks, 1, + "the sweep must not queue a delivery that is "+ + "already waiting for a worker", + ) +} + +// TestRecoveryAndSweepDoNotDoubleDispatch drives the two entry points +// the engine starts concurrently against one aged pending row. Before +// ownership they both dispatched it. +func TestRecoveryAndSweepDoNotDoubleDispatch(t *testing.T) { + t.Parallel() + + targetID := uuid.New().String() + s := fSweepSetup(t, targetID, "racing") + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"racing":true}`, + ) + + ctx := context.Background() + + for range 40 { + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + rAgePending(t, s.WebhookDB, d.ID) + + var wg sync.WaitGroup + + wg.Go(func() { + s.Engine.ExportRecoverPendingDeliveries( + ctx, s.WebhookDB, s.WebhookID, + ) + }) + wg.Go(func() { + s.Engine.ExportSweepWebhookRetries( + ctx, s.WebhookID, + ) + }) + wg.Wait() + + tasks := fDrain(s.Engine) + require.Len( + t, tasks, 1, + "delivery %s dispatched %d times", + d.ID, len(tasks), + ) + + // No worker runs in this engine, so the reference the winner + // took is never released and earlier iterations' deliveries + // stay owned — which is itself the property under test, since + // both paths see them on every subsequent pass. + } +} + +// TestConcurrentClaimsOfOneDeliveryYieldOneOwner exercises the +// exclusion directly, rather than arguing it from a SQL predicate. +func TestConcurrentClaimsOfOneDeliveryYieldOneOwner( + t *testing.T, +) { + t.Parallel() + + eng := newISetup(t).Engine + deliveryID := uuid.New().String() + + var ( + wg sync.WaitGroup + mu sync.Mutex + won int + ) + + for range 64 { + wg.Go(func() { + if eng.ExportRetainDelivery(deliveryID) { + mu.Lock() + won++ + mu.Unlock() + } + }) + } + + wg.Wait() + + assert.Equal(t, 1, won) + assert.Equal(t, 1, eng.ExportInflightHeld()) +} + +// TestOwnershipIsReleasedAfterDelivery guards the other direction: a +// leaked reference hides a delivery from every sweep for the life of +// the process. +func TestOwnershipIsReleasedAfterDelivery(t *testing.T) { + t.Parallel() + + s := newISetup(t) + targetID := uuid.New().String() + + iCreateTarget(t, s.MainDB, targetID, + s.WebhookID, "released", + database.TargetTypeLog, "", 0, + ) + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"released":true}`, + ) + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + + s.Engine.ExportStart() + + defer func() { + require.NoError( + t, s.Engine.ExportStop(context.Background()), + ) + }() + + body := `{"released":true}` + + s.Engine.Notify([]delivery.Task{{ + DeliveryID: d.ID, + EventID: event.ID, + WebhookID: s.WebhookID, + TargetID: targetID, + TargetName: "released", + TargetType: database.TargetTypeLog, + Body: &body, + EntrypointID: event.EntrypointID, + }}) + + iWaitForDelivered(t, s.WebhookDB, d.ID) + + assert.Eventually( + t, + func() bool { + return s.Engine.ExportInflightHeld() == 0 + }, + 2*time.Second, 20*time.Millisecond, + "the delivery stayed owned after it was delivered", + ) +} + +// TestRetryingRecoverySkipsASuccessfulResult is the retrying-side twin +// of the pending reconcile. A second attempt that reached the receiver +// and whose status write then failed sits at retrying holding a +// successful result, and re-sending it is the same duplicate. +func TestRetryingRecoverySkipsASuccessfulResult(t *testing.T) { + t.Parallel() + + targetID := uuid.New().String() + s := fSweepSetup(t, targetID, "retry-settled") + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"retry":true}`, + ) + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusRetrying, + ) + rSeedResult(t, s.WebhookDB, d.ID, 1, false) + rSeedResult(t, s.WebhookDB, d.ID, 2, true) + + s.Engine.ExportRecoverRetryingDeliveries( + s.WebhookDB, s.WebhookID, + ) + + assert.Empty( + t, fDrain(s.Engine), + "a retrying delivery holding a successful result "+ + "must not be sent again", + ) + + iAssertStatus( + t, s.WebhookDB, d.ID, + database.DeliveryStatusDelivered, + ) +} + +// TestRetryingSweepSkipsASuccessfulResult is the same rule on the +// periodic sweep's retrying arm. +func TestRetryingSweepSkipsASuccessfulResult(t *testing.T) { + t.Parallel() + + targetID := uuid.New().String() + s := fSweepSetup(t, targetID, "retry-swept") + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"swept":true}`, + ) + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusRetrying, + ) + rSeedResult(t, s.WebhookDB, d.ID, 1, false) + rSeedResult(t, s.WebhookDB, d.ID, 2, true) + + s.Engine.ExportSweepWebhookRetries( + context.Background(), s.WebhookID, + ) + + assert.Empty(t, fDrain(s.Engine)) + + iAssertStatus( + t, s.WebhookDB, d.ID, + database.DeliveryStatusDelivered, + ) + + var attempts int64 + + require.NoError(t, s.WebhookDB. + Model(&database.DeliveryResult{}). + Where("delivery_id = ?", d.ID). + Count(&attempts).Error) + assert.Equal( + t, int64(2), attempts, + "settling must not invent an attempt", + ) +} + +// TestScheduledRetryIsNotSweptDuringBackoff closes the window between +// a target scheduling a retry and the timer firing. The row says +// retrying and nothing is running, which is exactly what an orphaned +// retry looks like from the database. +func TestScheduledRetryIsNotSweptDuringBackoff(t *testing.T) { + t.Parallel() + + targetID := uuid.New().String() + s := fSweepSetup(t, targetID, "backoff") + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"backoff":true}`, + ) + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusRetrying, + ) + + s.Engine.ExportScheduleRetry(delivery.Task{ + DeliveryID: d.ID, + EventID: event.ID, + WebhookID: s.WebhookID, + TargetID: targetID, + AttemptNum: 2, + }, time.Hour) + + require.Equal(t, 1, s.Engine.ExportInflightHeld()) + + s.Engine.ExportSweepWebhookRetries( + context.Background(), s.WebhookID, + ) + + assert.Empty( + t, fDrain(s.Engine), + "the sweep must not duplicate a retry that is "+ + "already scheduled", + ) +} + +// TestRedispatchStampsTheRow pins the cadence control: a stranded +// delivery that has just been handed out is not selected again by the +// next tick a minute later. +func TestRedispatchStampsTheRow(t *testing.T) { + t.Parallel() + + targetID := uuid.New().String() + s := fSweepSetup(t, targetID, "stamped") + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"stamped":true}`, + ) + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + rAgePending(t, s.WebhookDB, d.ID) + + ctx := context.Background() + + s.Engine.ExportSweepWebhookRetries(ctx, s.WebhookID) + require.Len(t, fDrain(s.Engine), 1) + + var row database.Delivery + + require.NoError(t, s.WebhookDB. + First(&row, "id = ?", d.ID).Error) + assert.WithinDuration( + t, time.Now(), row.UpdatedAt, time.Minute, + "a re-dispatched delivery must be stamped so the "+ + "next tick does not select it again", + ) +} diff --git a/internal/delivery/metrics_test.go b/internal/delivery/metrics_test.go new file mode 100644 index 0000000..48492dd --- /dev/null +++ b/internal/delivery/metrics_test.go @@ -0,0 +1,564 @@ +package delivery_test + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/google/uuid" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" + "sneak.berlin/go/webhooker/internal/metrics" +) + +// Metric names as exposed on /metrics. +const ( + mAttempts = "webhooker_delivery_attempts_total" + mSucceeded = "webhooker_deliveries_succeeded_total" + mFailed = "webhooker_deliveries_failed_total" + mRetries = "webhooker_delivery_retries_total" + mDuration = "webhooker_delivery_duration_seconds" + mPending = "webhooker_deliveries_pending" + mRetrying = "webhooker_deliveries_retrying" + mBreakers = "webhooker_circuit_breakers_open" +) + +const ( + mTypeHTTP = "http" + mTypeLog = "log" + mTypeUnknown = "unknown" +) + +// mIsolate gives the setup's engine a metric set registered on a +// private registry. The process-wide collectors are moved by every +// other delivery test running in parallel, so exact assertions are +// only possible against a registry this test owns. +func mIsolate( + t *testing.T, s iSetup, +) *prometheus.Registry { + t.Helper() + + reg := prometheus.NewRegistry() + s.Engine.ExportSetMetrics(metrics.New(reg)) + + return reg +} + +// mFind returns the series of the named metric carrying the given +// target_type label. +func mFind( + t *testing.T, + reg *prometheus.Registry, + name, targetType string, +) *dto.Metric { + t.Helper() + + families, err := reg.Gather() + require.NoError(t, err) + + for _, fam := range families { + if fam.GetName() != name { + continue + } + + for _, m := range fam.GetMetric() { + for _, label := range m.GetLabel() { + if label.GetName() == "target_type" && + label.GetValue() == targetType { + return m + } + } + } + } + + t.Fatalf( + "metric %s{target_type=%q} not found", + name, targetType, + ) + + return nil +} + +func mCounter( + t *testing.T, + reg *prometheus.Registry, + name, targetType string, +) float64 { + t.Helper() + + return mFind(t, reg, name, targetType). + GetCounter().GetValue() +} + +func mGauge( + t *testing.T, + reg *prometheus.Registry, + name, targetType string, +) float64 { + t.Helper() + + return mFind(t, reg, name, targetType). + GetGauge().GetValue() +} + +// mHTTPDurations returns how many samples the delivery duration +// histogram holds for the http target type, which is the type every +// test here times. +func mHTTPDurations( + t *testing.T, reg *prometheus.Registry, +) uint64 { + t.Helper() + + return mFind(t, reg, mDuration, mTypeHTTP). + GetHistogram().GetSampleCount() +} + +// TestDeliveryMetrics_SuccessAndRetryExhaustion drives one delivery +// that succeeds and one that fails every attempt until its retries +// are exhausted, and asserts every delivery counter across both. +func TestDeliveryMetrics_SuccessAndRetryExhaustion( + t *testing.T, +) { + t.Parallel() + + s := newISetup(t) + reg := mIsolate(t, s) + + mDeliverOK(t, s) + + assert.InDelta(t, 1.0, + mCounter(t, reg, mAttempts, mTypeHTTP), 0) + assert.InDelta(t, 1.0, + mCounter(t, reg, mSucceeded, mTypeHTTP), 0) + assert.InDelta(t, 0.0, + mCounter(t, reg, mFailed, mTypeHTTP), 0) + assert.InDelta(t, 0.0, + mCounter(t, reg, mRetries, mTypeHTTP), 0) + assert.Equal(t, uint64(1), + mHTTPDurations(t, reg)) + + mExhaustRetries(t, s) + + // Two further attempts: the first is retried, the second is + // the last one allowed and fails the delivery terminally. + assert.InDelta(t, 3.0, + mCounter(t, reg, mAttempts, mTypeHTTP), 0) + assert.InDelta(t, 1.0, + mCounter(t, reg, mSucceeded, mTypeHTTP), 0) + assert.InDelta(t, 1.0, + mCounter(t, reg, mRetries, mTypeHTTP), 0) + assert.InDelta(t, 1.0, + mCounter(t, reg, mFailed, mTypeHTTP), 0) + assert.Equal(t, uint64(3), + mHTTPDurations(t, reg)) + + // Two consecutive failures are below the trip threshold. + assert.InDelta(t, 0.0, + mGauge(t, reg, mBreakers, mTypeHTTP), 0) + + // The label is the target type and nothing finer: two http + // targets shared one series, and no other type's moved. + assert.InDelta(t, 0.0, + mCounter(t, reg, mAttempts, mTypeLog), 0) + assert.InDelta(t, 0.0, + mCounter(t, reg, mFailed, mTypeLog), 0) +} + +// mDeliverOK delivers one event to a target that answers 200. +func mDeliverOK(t *testing.T, s iSetup) { + t.Helper() + + ts := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }, + )) + defer ts.Close() + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"ok":true}`, + ) + targetID := uuid.New().String() + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + + body := event.Body + task := iTask( + d, event, s.WebhookID, targetID, + "metrics-ok", iHTTPConfig(ts.URL), 3, 1, &body, + ) + + s.Engine.ExportProcessNewTask(context.TODO(), &task) + + iAssertStatus(t, s.WebhookDB, d.ID, + database.DeliveryStatusDelivered, + ) +} + +// mExhaustRetries delivers to a target that answers 500 with a +// two-attempt budget, driving both attempts so the delivery ends +// terminally failed. +func mExhaustRetries(t *testing.T, s iSetup) { + t.Helper() + + ts := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + }, + )) + defer ts.Close() + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"ok":false}`, + ) + targetID := uuid.New().String() + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + + body := event.Body + cfg := iHTTPConfig(ts.URL) + + // The retry below is only run if its target still exists; see + // https://git.eeqj.de/sneak/webhooker/issues/107. + iCreateTarget( + t, s.MainDB, targetID, s.WebhookID, "metrics-fail", + database.TargetTypeHTTP, cfg, 2, + ) + + first := iTask( + d, event, s.WebhookID, targetID, + "metrics-fail", cfg, 2, 1, &body, + ) + + s.Engine.ExportProcessNewTask(context.TODO(), &first) + + iAssertStatus(t, s.WebhookDB, d.ID, + database.DeliveryStatusRetrying, + ) + + // The engine's own scheduler would re-enqueue this after the + // backoff; driving the second attempt directly keeps the test + // deterministic and off the wall clock. + second := iTask( + d, event, s.WebhookID, targetID, + "metrics-fail", cfg, 2, 2, &body, + ) + + s.Engine.ExportProcessRetryTask( + context.TODO(), &second, + ) + + iAssertStatus(t, s.WebhookDB, d.ID, + database.DeliveryStatusFailed, + ) +} + +// TestDeliveryMetrics_CircuitBreakerGauge proves the open-breaker +// gauge follows a breaker that trips. +func TestDeliveryMetrics_CircuitBreakerGauge(t *testing.T) { + t.Parallel() + + s := newISetup(t) + reg := mIsolate(t, s) + + ts := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + }, + )) + defer ts.Close() + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"trip":true}`, + ) + targetID := uuid.New().String() + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + + body := event.Body + cfg := iHTTPConfig(ts.URL) + + // A retry budget above the failure threshold, so the breaker + // rather than the budget is what stops the delivery. + maxRetries := delivery.ExportDefaultFailureThreshold + 5 + + // The retries below are only run if their target still exists; + // see https://git.eeqj.de/sneak/webhooker/issues/107. + iCreateTarget( + t, s.MainDB, targetID, s.WebhookID, "metrics-trip", + database.TargetTypeHTTP, cfg, maxRetries, + ) + + first := iTask( + d, event, s.WebhookID, targetID, + "metrics-trip", cfg, maxRetries, 1, &body, + ) + + s.Engine.ExportProcessNewTask(context.TODO(), &first) + + assert.InDelta(t, 0.0, + mGauge(t, reg, mBreakers, mTypeHTTP), 0) + + for attempt := 2; attempt <= delivery. + ExportDefaultFailureThreshold; attempt++ { + task := iTask( + d, event, s.WebhookID, targetID, + "metrics-trip", cfg, maxRetries, attempt, &body, + ) + + s.Engine.ExportProcessRetryTask( + context.TODO(), &task, + ) + } + + assert.InDelta(t, 1.0, + mGauge(t, reg, mBreakers, mTypeHTTP), 0) +} + +// TestDeliveryMetrics_BreakerBlockedIsNotAnAttempt proves a delivery +// an open circuit breaker refuses is neither counted as an attempt +// nor observed in the duration histogram. +// +// It sends nothing and records no result row, so counting it would +// climb the attempts counter with no traffic behind it and pull the +// duration quantiles down with near-zero samples for as long as the +// breaker stayed open — the metric moving the wrong way during the +// outage it exists to reveal. +func TestDeliveryMetrics_BreakerBlockedIsNotAnAttempt( + t *testing.T, +) { + t.Parallel() + + s := newISetup(t) + reg := mIsolate(t, s) + + ts := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + }, + )) + defer ts.Close() + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"blocked":true}`, + ) + targetID := uuid.New().String() + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + + body := event.Body + cfg := iHTTPConfig(ts.URL) + maxRetries := delivery.ExportDefaultFailureThreshold + 5 + + iCreateTarget( + t, s.MainDB, targetID, s.WebhookID, "metrics-blocked", + database.TargetTypeHTTP, cfg, maxRetries, + ) + + first := iTask( + d, event, s.WebhookID, targetID, + "metrics-blocked", cfg, maxRetries, 1, &body, + ) + + s.Engine.ExportProcessNewTask(context.TODO(), &first) + + for attempt := 2; attempt <= delivery. + ExportDefaultFailureThreshold; attempt++ { + task := iTask( + d, event, s.WebhookID, targetID, + "metrics-blocked", cfg, maxRetries, attempt, &body, + ) + + s.Engine.ExportProcessRetryTask(context.TODO(), &task) + } + + require.InDelta(t, 1.0, + mGauge(t, reg, mBreakers, mTypeHTTP), 0, + "breaker should be open before the blocked attempt") + + threshold := float64( + delivery.ExportDefaultFailureThreshold, + ) + + assert.InDelta(t, threshold, + mCounter(t, reg, mAttempts, mTypeHTTP), 0) + assert.Equal(t, uint64(threshold), + mHTTPDurations(t, reg)) + + retriesBefore := mCounter(t, reg, mRetries, mTypeHTTP) + + blocked := iTask( + d, event, s.WebhookID, targetID, + "metrics-blocked", cfg, maxRetries, + delivery.ExportDefaultFailureThreshold+1, &body, + ) + + s.Engine.ExportProcessRetryTask(context.TODO(), &blocked) + + // The breaker refused it: rescheduled, so the retry counter + // moved, but nothing was attempted or timed. + assert.InDelta(t, retriesBefore+1, + mCounter(t, reg, mRetries, mTypeHTTP), 0) + assert.InDelta(t, threshold, + mCounter(t, reg, mAttempts, mTypeHTTP), 0) + assert.Equal(t, uint64(threshold), + mHTTPDurations(t, reg)) +} + +// TestDeliveryMetrics_OrphanedRetryFailureLabelled proves the +// terminal failure of a delivery whose target no longer retries is +// counted against the target's real type, not against unknown. The +// type is threaded in as an argument because populating d.Target on +// that path would write the target row into the per-webhook database +// (https://git.eeqj.de/sneak/webhooker/issues/206). +func TestDeliveryMetrics_OrphanedRetryFailureLabelled( + t *testing.T, +) { + t.Parallel() + + s := newISetup(t) + reg := mIsolate(t, s) + + iCreateWebhook( + t, s.MainDB, s.WebhookID, "orphaned-label", + ) + + deliveryID := iSeedRetryingWithType( + t, s, database.TargetTypeLog, + ) + + s.Engine.ExportSweepWebhookRetries( + context.Background(), s.WebhookID, + ) + + iAssertStatus(t, s.WebhookDB, deliveryID, + database.DeliveryStatusFailed, + ) + + assert.InDelta(t, 1.0, + mCounter(t, reg, mFailed, mTypeLog), 0) +} + +// TestDeliveryMetrics_QueueDepthGauges proves the sampler publishes +// the queued deliveries it finds in the per-webhook databases, and +// that a drained queue reads zero rather than keeping its last +// value. +func TestDeliveryMetrics_QueueDepthGauges(t *testing.T) { + t.Parallel() + + s := newISetup(t) + reg := mIsolate(t, s) + + iCreateWebhook( + t, s.MainDB, s.WebhookID, "queue-depth", + ) + + targetID := uuid.New().String() + + iCreateTarget(t, s.MainDB, targetID, s.WebhookID, + "queue-depth-target", database.TargetTypeHTTP, + iHTTPConfig("https://example.com/hook"), 3, + ) + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"queued":true}`, + ) + + pending := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + + iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + + retrying := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusRetrying, + ) + + s.Engine.ExportSampleQueueDepths(context.Background()) + + assert.InDelta(t, 2.0, + mGauge(t, reg, mPending, mTypeHTTP), 0) + assert.InDelta(t, 1.0, + mGauge(t, reg, mRetrying, mTypeHTTP), 0) + assert.InDelta(t, 0.0, + mGauge(t, reg, mPending, mTypeLog), 0) + + require.NoError(t, s.WebhookDB. + Model(&database.Delivery{}). + Where("id IN ?", []string{pending.ID, retrying.ID}). + Update( + "status", database.DeliveryStatusDelivered, + ).Error) + + s.Engine.ExportSampleQueueDepths(context.Background()) + + assert.InDelta(t, 1.0, + mGauge(t, reg, mPending, mTypeHTTP), 0) + assert.InDelta(t, 0.0, + mGauge(t, reg, mRetrying, mTypeHTTP), 0) +} + +// TestDeliveryMetrics_QueueDepthDeletedTarget proves a backlog queued +// against a target that has since been deleted stays visible, in the +// unknown series, instead of being dropped. That backlog is the one +// nobody is watching, so losing it would defeat the queue-depth +// alerting this metric exists for. +func TestDeliveryMetrics_QueueDepthDeletedTarget(t *testing.T) { + t.Parallel() + + s := newISetup(t) + reg := mIsolate(t, s) + + iCreateWebhook( + t, s.MainDB, s.WebhookID, "deleted-target", + ) + + // No target row is created: this is a delivery whose target was + // deleted out from under it. + targetID := uuid.New().String() + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"orphan":true}`, + ) + + iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + + iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusRetrying, + ) + + s.Engine.ExportSampleQueueDepths(context.Background()) + + assert.InDelta(t, 1.0, + mGauge(t, reg, mPending, mTypeUnknown), 0) + assert.InDelta(t, 1.0, + mGauge(t, reg, mRetrying, mTypeUnknown), 0) + assert.InDelta(t, 0.0, + mGauge(t, reg, mPending, mTypeHTTP), 0) +} diff --git a/internal/delivery/queue_depth.go b/internal/delivery/queue_depth.go new file mode 100644 index 0000000..e627650 --- /dev/null +++ b/internal/delivery/queue_depth.go @@ -0,0 +1,196 @@ +package delivery + +import ( + "context" + "fmt" + "time" + + "sneak.berlin/go/webhooker/internal/database" +) + +// queueDepthSampleInterval is how often the pending and retrying +// queue depths are counted and published as gauges. +const queueDepthSampleInterval = 30 * time.Second + +// queueDepthSampler publishes the pending and retrying queue depths +// on a timer for as long as the engine runs. +// +// The depths are counted out of the databases rather than tracked as +// deltas alongside the status transitions. A delta counter would have +// to be seeded correctly at startup from rows written by a previous +// process, and would drift permanently on any transition that failed +// to persist. Counting is the measurement that cannot go wrong, and +// it is the same whole-database walk the retry sweep already makes. +func (e *Engine) queueDepthSampler(ctx context.Context) { + defer e.wg.Done() + + ticker := time.NewTicker(queueDepthSampleInterval) + defer ticker.Stop() + + e.sampleQueueDepths(ctx) + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + e.sampleQueueDepths(ctx) + } + } +} + +// sampleQueueDepths counts every queued delivery across all +// per-webhook databases and publishes the result. +func (e *Engine) sampleQueueDepths(ctx context.Context) { + if e.database == nil || e.dbManager == nil { + return + } + + types, err := e.targetTypesByID() + if err != nil { + e.log.Error( + "queue depth sample: failed to load target types", + "error", err, + ) + + return + } + + var webhookIDs []string + + err = e.database.DB(). + Model(&database.Webhook{}). + Pluck("id", &webhookIDs).Error + if err != nil { + e.log.Error( + "queue depth sample: failed to query webhook IDs", + "error", err, + ) + + return + } + + pending := make(map[database.TargetType]int) + retrying := make(map[database.TargetType]int) + + for _, webhookID := range webhookIDs { + select { + case <-ctx.Done(): + return + default: + } + + if !e.dbManager.DBExists(webhookID) { + continue + } + + e.sampleWebhookQueueDepths( + webhookID, types, pending, retrying, + ) + } + + e.mtr.SetQueueDepths(pending, retrying) +} + +// targetTypesByID maps every configured target id to its type. The +// deliveries live in the per-webhook databases but carry only a +// target id, so the type label has to come from the main database. +// +// Find rather than Scan: see sampleWebhookQueueDepths. +func (e *Engine) targetTypesByID() ( + map[string]database.TargetType, error, +) { + var rows []struct { + ID string + Type database.TargetType + } + + err := e.database.DB(). + Model(&database.Target{}). + Select("id", "type"). + Find(&rows).Error + if err != nil { + return nil, fmt.Errorf("loading targets: %w", err) + } + + types := make(map[string]database.TargetType, len(rows)) + + for _, row := range rows { + types[row.ID] = row.Type + } + + return types, nil +} + +// sampleWebhookQueueDepths adds one webhook's queued deliveries into +// the running totals. +// +// A delivery whose target has since been deleted is not in the type +// map and so counts under the empty target type. Set.SetQueueDepths +// folds that into the unknown series rather than dropping it: a +// backlog stuck behind a deleted target is a backlog that still needs +// to be alertable. +// +// The aggregate is read with Find, not Scan. (*gorm.DB).Scan swaps +// GORM's own trace recorder in for the logging adapter, and that +// recorder does not implement gorm.ParamsFilter, so the statement +// reaches the log with its bound values interpolated — here, the +// status list. Find goes through the normal query callback, which is +// filtered. See internal/gormlog and its scan_guard_test.go. +func (e *Engine) sampleWebhookQueueDepths( + webhookID string, + types map[string]database.TargetType, + pending, retrying map[database.TargetType]int, +) { + webhookDB, err := e.dbManager.GetDB(webhookID) + if err != nil { + e.log.Error( + "queue depth sample: failed to get webhook database", + "webhook_id", webhookID, + "error", err, + ) + + return + } + + var rows []struct { + TargetID string + Status database.DeliveryStatus + Depth int + } + + err = webhookDB. + Model(&database.Delivery{}). + Select("target_id", "status", "count(*) as depth"). + Where("status IN ?", []database.DeliveryStatus{ + database.DeliveryStatusPending, + database.DeliveryStatusRetrying, + }). + Group("target_id, status"). + Find(&rows).Error + if err != nil { + e.log.Error( + "queue depth sample: "+ + "failed to count queued deliveries", + "webhook_id", webhookID, + "error", err, + ) + + return + } + + for _, row := range rows { + targetType := types[row.TargetID] + + switch row.Status { + case database.DeliveryStatusPending: + pending[targetType] += row.Depth + case database.DeliveryStatusRetrying: + retrying[targetType] += row.Depth + case database.DeliveryStatusDelivered, + database.DeliveryStatusFailed: + // Excluded by the query above: a delivery that has + // reached a terminal state is not queued. + } + } +} diff --git a/internal/delivery/queue_depth_gormlog_test.go b/internal/delivery/queue_depth_gormlog_test.go new file mode 100644 index 0000000..8c89734 --- /dev/null +++ b/internal/delivery/queue_depth_gormlog_test.go @@ -0,0 +1,175 @@ +package delivery_test + +import ( + "bytes" + "context" + "log/slog" + "net/http" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" + "sneak.berlin/go/webhooker/internal/gormlog" +) + +// qdAggregateMarker identifies the queue-depth aggregate in the +// captured SQL. It is the one statement in this test that binds +// anything, and the raw count() expression appears in no other. +const qdAggregateMarker = "count(*)" + +// qdSyncBuf collects log output from whichever goroutine GORM writes +// on. +type qdSyncBuf struct { + mu sync.Mutex + b bytes.Buffer +} + +func (q *qdSyncBuf) Write(p []byte) (int, error) { + q.mu.Lock() + defer q.mu.Unlock() + + return q.b.Write(p) +} + +func (q *qdSyncBuf) String() string { + q.mu.Lock() + defer q.mu.Unlock() + + return q.b.String() +} + +// qdMainDB opens a main database whose GORM logger is the service's +// adapter, writing through log. +func qdMainDB(t *testing.T, log *slog.Logger) *gorm.DB { + t.Helper() + + sqlDB, err := database.OpenSQLite( + filepath.Join(t.TempDir(), "main-gormlog.db"), + database.SQLiteModeCreate, + ) + require.NoError(t, err) + + t.Cleanup(func() { _ = sqlDB.Close() }) + + db, err := gorm.Open( + sqlite.Dialector{Conn: sqlDB}, + &gorm.Config{Logger: gormlog.New(log)}, + ) + require.NoError(t, err) + + require.NoError(t, db.AutoMigrate( + &database.Webhook{}, + &database.Target{}, + )) + + return db +} + +// qdLinesContaining returns every captured line carrying marker. +func qdLinesContaining(out, marker string) []string { + var found []string + + for line := range strings.SplitSeq(out, "\n") { + if strings.Contains(line, marker) { + found = append(found, line) + } + } + + return found +} + +// TestQueueDepthSample_LogsNoBoundValue holds the queue-depth sampler +// to the values-off property internal/gormlog exists to provide. +// +// The aggregate binds the delivery status list. Read with +// (*gorm.DB).Scan it was logged with those values interpolated, because +// Scan records the statement through GORM's own traceRecorder, which +// does not implement gorm.ParamsFilter. Read with Find it goes through +// the normal query callback and the adapter's filter applies. Restore +// the Scan call in queue_depth.go and this fails on the status literals +// below; scan_guard_test.go catches the same regression statically. +func TestQueueDepthSample_LogsNoBoundValue(t *testing.T) { + t.Parallel() + + buf := &qdSyncBuf{} + log := slog.New(slog.NewTextHandler( + buf, &slog.HandlerOptions{Level: slog.LevelDebug}, + )) + + mainDB := qdMainDB(t, log) + dbMgr := database.NewTestWebhookDBManagerWithLogger( + t.TempDir(), log, + ) + + webhookID := uuid.New().String() + webhookDB := iSeedWebhookDB(t, dbMgr, webhookID) + + iCreateWebhook(t, mainDB, webhookID, "queue-depth-gormlog") + + targetID := uuid.New().String() + + iCreateTarget(t, mainDB, targetID, webhookID, + "queue-depth-gormlog-target", database.TargetTypeHTTP, + iHTTPConfig("https://example.com/hook"), 3, + ) + + event := iSeedEvent( + t, webhookDB, webhookID, `{"queued":true}`, + ) + + iSeedDelivery( + t, webhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + iSeedDelivery( + t, webhookDB, event.ID, targetID, + database.DeliveryStatusRetrying, + ) + + engine := delivery.NewTestEngineWithDB( + database.NewTestDatabase(mainDB), + dbMgr, + log, + &http.Client{Timeout: 5 * time.Second}, + 2, + ) + + engine.ExportSampleQueueDepths(context.Background()) + + out := buf.String() + + lines := qdLinesContaining(out, qdAggregateMarker) + require.NotEmpty( + t, lines, + "the queue-depth aggregate was never logged, so the "+ + "assertions below are vacuous", + ) + + for _, line := range lines { + assert.Contains( + t, line, "?", + "the aggregate was logged without its placeholders: %s", + line, + ) + + for _, status := range []database.DeliveryStatus{ + database.DeliveryStatusPending, + database.DeliveryStatusRetrying, + } { + assert.NotContains( + t, line, string(status), + "a bound status value was interpolated into the "+ + "logged statement: %s", line, + ) + } + } +} diff --git a/internal/delivery/recovery_durability_test.go b/internal/delivery/recovery_durability_test.go new file mode 100644 index 0000000..a139eeb --- /dev/null +++ b/internal/delivery/recovery_durability_test.go @@ -0,0 +1,378 @@ +package delivery_test + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" +) + +// These tests cover the delivery half of +// https://git.eeqj.de/sneak/webhooker/issues/256: a delivery that +// reached its receiver but whose bookkeeping write failed used to be +// left at pending and re-sent on the next restart, giving the receiver +// a second copy while the event log recorded one attempt. + +// rSeedResult records a DeliveryResult against a delivery, standing in +// for the attempt row the send path writes before the status. +func rSeedResult( + t *testing.T, + db *gorm.DB, + deliveryID string, + attemptNum int, + success bool, +) { + t.Helper() + + require.NoError(t, db.Create(&database.DeliveryResult{ + DeliveryID: deliveryID, + AttemptNum: attemptNum, + Success: success, + }).Error) +} + +// rAgePending backdates a delivery past the sweep's age bound, which is +// what separates a stranded delivery from one a worker still holds. +func rAgePending( + t *testing.T, db *gorm.DB, deliveryID string, +) { + t.Helper() + + old := time.Now().Add( + -2 * delivery.ExportPendingSweepMinAge, + ) + + require.NoError(t, db.Model(&database.Delivery{}). + Where("id = ?", deliveryID). + UpdateColumn("updated_at", old).Error) +} + +func TestRecoverySkipsPendingWithSuccessfulResult( + t *testing.T, +) { + t.Parallel() + + s := newISetup(t) + targetID := uuid.New().String() + + iCreateTarget(t, s.MainDB, targetID, + s.WebhookID, "already-delivered", + database.TargetTypeLog, "", 0, + ) + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"delivered":true}`, + ) + + // The delivery whose send succeeded and whose result row landed: + // only the status write failed, so it sits at pending. + done := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + rSeedResult(t, s.WebhookDB, done.ID, 1, true) + + // A delivery that was genuinely never attempted. + fresh := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + + s.Engine.ExportRecoverPendingDeliveries( + context.Background(), s.WebhookDB, s.WebhookID, + ) + + select { + case task := <-s.Engine.ExportDeliveryCh(): + assert.Equal( + t, fresh.ID, task.DeliveryID, + "only the unattempted delivery may be re-sent", + ) + case <-time.After(2 * time.Second): + t.Fatal("expected the unattempted delivery") + } + + select { + case task := <-s.Engine.ExportDeliveryCh(): + t.Fatalf( + "re-sent an already delivered delivery: %s", + task.DeliveryID, + ) + case <-time.After(200 * time.Millisecond): + } + + // It is settled rather than merely skipped: leaving it pending + // would strand it again on the next sweep. + iAssertStatus( + t, s.WebhookDB, done.ID, + database.DeliveryStatusDelivered, + ) +} + +// TestRecoveryContinuesTheAttemptNumbering pins the audit trail: a +// recovered delivery that already recorded two attempts is re-sent as +// attempt three, not as attempt one again. +func TestRecoveryContinuesTheAttemptNumbering(t *testing.T) { + t.Parallel() + + s := newISetup(t) + targetID := uuid.New().String() + + iCreateTarget(t, s.MainDB, targetID, + s.WebhookID, "numbering", + database.TargetTypeLog, "", 0, + ) + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"numbering":true}`, + ) + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + + rSeedResult(t, s.WebhookDB, d.ID, 1, false) + rSeedResult(t, s.WebhookDB, d.ID, 2, false) + + s.Engine.ExportRecoverPendingDeliveries( + context.Background(), s.WebhookDB, s.WebhookID, + ) + + select { + case task := <-s.Engine.ExportDeliveryCh(): + assert.Equal(t, d.ID, task.DeliveryID) + assert.Equal(t, 3, task.AttemptNum) + case <-time.After(2 * time.Second): + t.Fatal("expected the delivery to be recovered") + } +} + +// TestSweepRecoversStrandedPending is the half that removes the +// restart requirement: a delivery left at pending is picked up by the +// periodic sweep. +func TestSweepRecoversStrandedPending(t *testing.T) { + t.Parallel() + + targetID := uuid.New().String() + s := fSweepSetup(t, targetID, "stranded") + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"stranded":true}`, + ) + + stranded := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + rAgePending(t, s.WebhookDB, stranded.ID) + + // A delivery a worker may still be holding: young, and therefore + // none of the sweep's business. + inFlight := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + + s.Engine.ExportSweepWebhookRetries( + context.Background(), s.WebhookID, + ) + + select { + case task := <-s.Engine.ExportDeliveryCh(): + assert.Equal(t, stranded.ID, task.DeliveryID) + case <-time.After(2 * time.Second): + t.Fatal("expected the stranded delivery") + } + + select { + case task := <-s.Engine.ExportDeliveryCh(): + t.Fatalf( + "swept an in-flight delivery: %s", + task.DeliveryID, + ) + case <-time.After(200 * time.Millisecond): + } + + iAssertStatus( + t, s.WebhookDB, inFlight.ID, + database.DeliveryStatusPending, + ) +} + +// TestSweepClaimsAStrandedDeliveryOnlyOnce guards the repeat the sweep +// would otherwise be: the row stays pending for as long as the attempt +// runs, and a sweep a minute later must not send it a second time. +func TestSweepClaimsAStrandedDeliveryOnlyOnce(t *testing.T) { + t.Parallel() + + targetID := uuid.New().String() + s := fSweepSetup(t, targetID, "claimed") + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"claimed":true}`, + ) + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + rAgePending(t, s.WebhookDB, d.ID) + + ctx := context.Background() + + s.Engine.ExportSweepWebhookRetries(ctx, s.WebhookID) + + select { + case task := <-s.Engine.ExportDeliveryCh(): + assert.Equal(t, d.ID, task.DeliveryID) + case <-time.After(2 * time.Second): + t.Fatal("expected the stranded delivery") + } + + // The delivery is still pending — nothing has run it yet — but + // the claim must keep the next sweep off it. + iAssertStatus( + t, s.WebhookDB, d.ID, + database.DeliveryStatusPending, + ) + + s.Engine.ExportSweepWebhookRetries(ctx, s.WebhookID) + + select { + case task := <-s.Engine.ExportDeliveryCh(): + t.Fatalf( + "sent a claimed delivery again: %s", + task.DeliveryID, + ) + case <-time.After(200 * time.Millisecond): + } +} + +// TestSweepSettlesStrandedPendingWithoutResending is the sweep's own +// version of the reconcile: a stranded delivery holding a successful +// result is settled where it stands, and the receiver hears nothing. +func TestSweepSettlesStrandedPendingWithoutResending( + t *testing.T, +) { + t.Parallel() + + targetID := uuid.New().String() + s := fSweepSetup(t, targetID, "settled") + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"settled":true}`, + ) + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + rSeedResult(t, s.WebhookDB, d.ID, 1, true) + rAgePending(t, s.WebhookDB, d.ID) + + s.Engine.ExportSweepWebhookRetries( + context.Background(), s.WebhookID, + ) + + select { + case task := <-s.Engine.ExportDeliveryCh(): + t.Fatalf( + "re-sent a delivery that already succeeded: %s", + task.DeliveryID, + ) + case <-time.After(200 * time.Millisecond): + } + + iAssertStatus( + t, s.WebhookDB, d.ID, + database.DeliveryStatusDelivered, + ) + + var attempts int64 + + require.NoError(t, s.WebhookDB. + Model(&database.DeliveryResult{}). + Where("delivery_id = ?", d.ID). + Count(&attempts).Error) + assert.Equal( + t, int64(1), attempts, + "settling must not invent an attempt", + ) +} + +// TestFailedResultWriteLeavesDeliveryRecoverable is the rule the +// targets now follow: a bookkeeping write that fails must not advance +// the status, because pending and retrying are the states the sweeps +// recover and delivered is a claim the database refused to record. +func TestFailedResultWriteLeavesDeliveryRecoverable( + t *testing.T, +) { + t.Parallel() + + s := newISetup(t) + targetID := uuid.New().String() + + var hits atomic.Int64 + + ts := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusOK) + }, + )) + defer ts.Close() + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"unwritable":true}`, + ) + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + + // Drop the table the attempt row goes in, so the send succeeds + // and only the bookkeeping write fails. + require.NoError( + t, + s.WebhookDB.Exec("drop table delivery_results").Error, + ) + + full := &database.Delivery{ + EventID: event.ID, + TargetID: targetID, + Status: database.DeliveryStatusPending, + Event: event, + Target: database.Target{ + Name: "unwritable", + Type: database.TargetTypeHTTP, + Config: iHTTPConfig(ts.URL), + }, + } + full.ID = d.ID + + s.Engine.ExportDeliverHTTP( + context.Background(), s.WebhookDB, full, + &delivery.Task{DeliveryID: d.ID, AttemptNum: 1}, + ) + + assert.Equal( + t, int64(1), hits.Load(), + "the send itself must still happen", + ) + + iAssertStatus( + t, s.WebhookDB, d.ID, + database.DeliveryStatusPending, + ) +} diff --git a/internal/delivery/redirect.go b/internal/delivery/redirect.go new file mode 100644 index 0000000..af78042 --- /dev/null +++ b/internal/delivery/redirect.go @@ -0,0 +1,107 @@ +package delivery + +import ( + "errors" + "fmt" + "net" + "net/http" + "net/url" + "strings" +) + +// maxDeliveryRedirects caps a redirect chain. Installing a +// CheckRedirect replaces net/http's default policy including its +// own limit, so the limit is restated rather than dropped. +const maxDeliveryRedirects = 10 + +// schemeHTTPS names the scheme the origin comparison treats +// specially: a step down from it is never the same origin. +const schemeHTTPS = "https" + +var errTooManyRedirects = errors.New("too many redirects") + +// offOriginHeaderPolicy returns a CheckRedirect that drops every +// origin-scoped header once a redirect leaves the origin the +// operator configured. names is the set applyRequestHeaders +// reports: the operator's configured headers and the inbound event +// headers this delivery forwarded, under one rule rather than two. +// +// net/http withholds Authorization and Cookie across a host change +// and forwards everything else. A target header is routinely a +// credential under another name — X-Api-Key, PRIVATE-TOKEN, +// X-Auth-Token — and a forwarded inbound header is routinely a +// sender's signature — X-Hub-Signature — so an open redirect at an +// otherwise trusted destination would hand either to a host the +// operator never named. Redirects are still followed: refusing them +// would break every destination that legitimately redirects and +// would record the 3xx as the delivery's result. +// +// The strip is per hop, not permanent: net/http re-copies the +// initial request's headers for every hop, so a chain that returns +// to the configured origin carries them again, exactly as net/http +// treats Authorization. +// +// Each hop is dialled through the same SSRF-safe transport, whose +// guard runs per connection, so a redirect aimed at a private or +// reserved address is still refused at connect time. +func offOriginHeaderPolicy( + names []string, +) func(*http.Request, []*http.Request) error { + return func(req *http.Request, via []*http.Request) error { + if len(via) >= maxDeliveryRedirects { + return fmt.Errorf( + "%w: stopped after %d", + errTooManyRedirects, maxDeliveryRedirects, + ) + } + + if sameDeliveryOrigin(via[0].URL, req.URL) { + return nil + } + + for _, name := range names { + req.Header.Del(name) + } + + return nil + } +} + +// sameDeliveryOrigin reports whether dest is close enough to the +// configured target URL to keep carrying its origin-scoped headers. +// +// This is stricter than the rule net/http applies to Authorization: +// the port is part of the comparison (a different port is a +// different service), and a subdomain of the configured host is not +// the same origin. An https origin stepping down to http is never +// the same origin whatever the hosts are, because that puts the +// header on the wire in clear. +func sameDeliveryOrigin(origin, dest *url.URL) bool { + if origin.Scheme == schemeHTTPS && dest.Scheme != schemeHTTPS { + return false + } + + return originHostPort(origin) == originHostPort(dest) +} + +// originHostPort renders a URL's host for comparison, lowercased +// and with the scheme's default port normalised away so that +// "https://h" and "https://h:443" are one origin. +// +// The port is joined with net.JoinHostPort rather than a bare +// colon: Hostname() unwraps an IPv6 literal's brackets, so +// "[2001:db8::1]:8080" and "[2001:db8::1:8080]" — a different +// address on a different port — would otherwise render the same +// string and pass as one origin. +func originHostPort(u *url.URL) string { + host := strings.ToLower(u.Hostname()) + + port := u.Port() + if port == "" || + (u.Scheme == "http" && port == "80") || + (u.Scheme == schemeHTTPS && port == "443") { + return host + } + + return net.JoinHostPort(host, port) +} diff --git a/internal/delivery/redirect_test.go b/internal/delivery/redirect_test.go new file mode 100644 index 0000000..31e3d66 --- /dev/null +++ b/internal/delivery/redirect_test.go @@ -0,0 +1,383 @@ +package delivery_test + +import ( + "context" + "encoding/json" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" +) + +// The headers these tests drive stand in for the two classes the +// off-origin rule covers: an operator-configured credential and an +// inbound header the delivery path forwards. net/http withholds +// Authorization and Cookie across a host change, and nothing else. +const ( + probeHeaderName = "X-Api-Key" + probeHeaderValue = "QQNEVERONTHEWIREQQ" + inboundHeaderName = "X-Hub-Signature" + inboundHeaderValue = "sha1=QQINBOUNDQQ" +) + +// redirectProbe records what the last hop of a redirect chain +// actually received. +type redirectProbe struct { + mu sync.Mutex + seen http.Header + hits int +} + +func (p *redirectProbe) serve( + w http.ResponseWriter, r *http.Request, +) { + p.mu.Lock() + p.seen = r.Header.Clone() + p.hits++ + p.mu.Unlock() + + w.WriteHeader(http.StatusOK) +} + +func (p *redirectProbe) result() (http.Header, int) { + p.mu.Lock() + defer p.mu.Unlock() + + return p.seen, p.hits +} + +// deliverWithProbeHeaders runs one real delivery of a new task +// through the engine to targetURL, carrying both probe headers — +// probeHeaderName configured on the target, inboundHeaderName +// forwarded from the event — and returns the delivery status the +// engine recorded. +func deliverWithProbeHeaders( + t *testing.T, targetURL string, +) database.DeliveryStatus { + t.Helper() + + s := newISetup(t) + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"hello":"world"}`, + ) + targetID := uuid.New().String() + + inbound, err := json.Marshal(map[string][]string{ + inboundHeaderName: {inboundHeaderValue}, + }) + require.NoError(t, err) + + event.Headers = string(inbound) + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + + cfg, err := json.Marshal(delivery.HTTPTargetConfig{ + URL: targetURL, + Headers: map[string]string{ + probeHeaderName: probeHeaderValue, + }, + }) + require.NoError(t, err) + + body := event.Body + task := iTask( + d, event, s.WebhookID, targetID, + "redirect-target", string(cfg), 0, 1, &body, + ) + + s.Engine.ExportProcessNewTask(context.TODO(), &task) + + var updated database.Delivery + + require.NoError(t, s.WebhookDB.First( + &updated, "id = ?", d.ID, + ).Error) + + return updated.Status +} + +// A 302 to an origin the operator never configured must not carry +// the credential they configured for the one they did, nor the +// inbound header this delivery forwarded — one rule for both +// classes. The chain is still followed, so the delivery is recorded +// from the final hop. +func TestDelivery_CrossOriginRedirectDropsOriginScopedHeaders( + t *testing.T, +) { + t.Parallel() + + var probe redirectProbe + + final := httptest.NewServer( + http.HandlerFunc(probe.serve), + ) + defer final.Close() + + // httptest listens on loopback, so reach the second server + // under loopback's other name: the hop then differs in + // hostname as well as port and is cross-host by any reading. + finalURL, err := url.Parse(final.URL) + require.NoError(t, err) + + finalURL.Host = "localhost:" + finalURL.Port() + finalURL.Path = "/moved" + + origin := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + http.Redirect( + w, r, finalURL.String(), + http.StatusFound, + ) + }, + )) + defer origin.Close() + + status := deliverWithProbeHeaders(t, origin.URL) + + seen, hits := probe.result() + + assert.Equal(t, 1, hits, + "the redirect must still be followed", + ) + assert.Empty(t, seen.Get(probeHeaderName), + "a configured credential header must not reach an "+ + "origin the operator did not configure", + ) + assert.Empty(t, seen.Get(inboundHeaderName), + "a forwarded inbound header must not reach an origin "+ + "the operator did not configure", + ) + assert.Equal(t, + database.DeliveryStatusDelivered, status, + "the final hop's 200 is the delivery's result", + ) +} + +// Stripping must not fire within the configured origin, or every +// destination that redirects its own path would lose its +// credential and start answering 401 — and would lose the inbound +// signature the receiver verifies. +func TestDelivery_SameOriginRedirectKeepsOriginScopedHeaders( + t *testing.T, +) { + t.Parallel() + + var probe redirectProbe + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/moved" { + probe.serve(w, r) + + return + } + + http.Redirect( + w, r, "/moved", http.StatusFound, + ) + }, + )) + defer srv.Close() + + status := deliverWithProbeHeaders(t, srv.URL+"/hook") + + seen, hits := probe.result() + + assert.Equal(t, 1, hits) + assert.Equal(t, probeHeaderValue, seen.Get(probeHeaderName), + "a redirect within the configured origin must keep "+ + "the configured header", + ) + assert.Equal(t, + inboundHeaderValue, seen.Get(inboundHeaderName), + "a redirect within the configured origin must keep "+ + "the forwarded inbound header", + ) + assert.Equal(t, + database.DeliveryStatusDelivered, status, + ) +} + +// The origin comparison is deliberately stricter than the one +// net/http applies to Authorization: the port counts and a +// subdomain does not inherit. Only the default-port spellings of +// one origin are the same origin. +func TestSameDeliveryOrigin(t *testing.T) { + t.Parallel() + + // The configured target URL every case redirects away from. + // Destination paths differ only so that no literal repeats. + const configured = "https://h/a" + + cases := map[string]struct { + origin string + dest string + want bool + }{ + "other path": {configured, "https://h/b", true}, + "default port spelled": {configured, "https://h:443/c", true}, + "host in another case": {configured, "https://H/d", true}, + "http default port": {"http://h:80/a", "http://h/e", true}, + "upgrade to https": {"http://h/a", "https://h/f", true}, + "downgrade to http": {configured, "http://h/g", false}, + "another host": {configured, "https://i/h", false}, + "a subdomain": {configured, "https://x.h/i", false}, + "the parent domain": {"https://x.h/a", "https://h/j", false}, + "another port": {configured, "https://h:8443/k", false}, + + // Hostname() unwraps an IPv6 literal's brackets, so a + // bracketed host whose last group is the origin's port + // renders identically to the origin unless the port is + // re-joined with brackets. Each dest below differs from + // its origin in address AND in port. + "ipv6 port as final group": { + "https://[2001:db8::1]:8080/a", + "https://[2001:db8::1:8080]/l", + false, + }, + "ipv6 loopback port as final group": { + "https://[::1]:8080/a", + "https://[::1:8080]/m", + false, + }, + "ipv6 same origin": { + "https://[2001:db8::1]:8080/a", + "https://[2001:DB8::1]:8080/n", + true, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + origin, err := url.Parse(tc.origin) + require.NoError(t, err) + + dest, err := url.Parse(tc.dest) + require.NoError(t, err) + + assert.Equal(t, tc.want, + delivery.ExportSameDeliveryOrigin( + origin, dest, + ), + ) + }) + } +} + +// Installing a CheckRedirect discards net/http's own redirect +// limit, so the cap this policy restates is the only thing between +// a self-redirecting destination and an unbounded chain. A +// destination that always redirects must be cut off after exactly +// maxDeliveryRedirects requests, with the sentinel surfacing to the +// caller rather than a generic net/http error. +func TestRedirectPolicy_StopsAtHopCap(t *testing.T) { + t.Parallel() + + var hits atomic.Int64 + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + http.Redirect( + w, r, "/loop", http.StatusFound, + ) + }, + )) + defer srv.Close() + + engine := delivery.NewTestEngine( + slog.New(slog.DiscardHandler), + &http.Client{Timeout: 10 * time.Second}, + 1, + ) + + client := engine.ExportClientForRequest( + &delivery.HTTPTargetConfig{URL: srv.URL}, + []string{probeHeaderName}, + ) + require.NotNil(t, client.CheckRedirect) + + req, err := http.NewRequestWithContext( + context.Background(), http.MethodGet, srv.URL, http.NoBody, + ) + require.NoError(t, err) + + resp, doErr := client.Do(req) + if resp != nil { + _ = resp.Body.Close() + } + + require.Error(t, doErr, + "an endless redirect chain must not be followed forever", + ) + require.ErrorIs(t, doErr, delivery.ErrExportTooManyRedirects) + + assert.Equal(t, + int64(delivery.ExportMaxDeliveryRedirects), hits.Load(), + "the chain must stop after exactly %d hops", + delivery.ExportMaxDeliveryRedirects, + ) +} + +// The set the redirect policy strips is whatever the delivery path +// actually put on the wire, so a header added to the forward set is +// covered without a second edit. A header the event never carried +// is not in the set, and the delivery path's own two are deliberately +// excluded: Content-Type describes the body, which a 307 carries +// across hosts, and the inbound User-Agent every real sender supplies +// is overwritten before the request goes out. +func TestApplyRequestHeaders_ReportsOriginScopedNames(t *testing.T) { + t.Parallel() + + inbound, err := json.Marshal(map[string][]string{ + inboundHeaderName: {inboundHeaderValue}, + "Content-Type": {testContentType}, + "User-Agent": {"curl/8.7.1"}, + "Host": {"inbound.example.com"}, + }) + require.NoError(t, err) + + req, err := http.NewRequestWithContext( + context.Background(), + http.MethodPost, + "https://target.example.com/hook", + http.NoBody, + ) + require.NoError(t, err) + + names := delivery.ExportApplyRequestHeaders( + req, + &database.Event{ + Headers: string(inbound), + ContentType: testContentType, + }, + &delivery.HTTPTargetConfig{ + Headers: map[string]string{ + probeHeaderName: probeHeaderValue, + }, + }, + ) + + assert.Equal(t, + []string{probeHeaderName, inboundHeaderName}, names, + "both header classes are reported, and only those: "+ + "Host is never forwarded, Content-Type and "+ + "User-Agent are the delivery path's own", + ) +} diff --git a/internal/delivery/ssrf.go b/internal/delivery/ssrf.go index be23746..a718e6a 100644 --- a/internal/delivery/ssrf.go +++ b/internal/delivery/ssrf.go @@ -6,8 +6,11 @@ import ( "fmt" "net" "net/http" + "net/netip" "net/url" "time" + + "sneak.berlin/go/webhooker/internal/config" ) const ( @@ -25,20 +28,83 @@ var ( errBlockedIP = errors.New( "blocked private/reserved IP range", ) + errBlockedMetadata = errors.New( + "blocked link-local or cloud instance metadata " + + "address: ALLOWED_EGRESS_CIDRS cannot open it", + ) errInvalidScheme = errors.New( "only http and https are allowed", ) ) // blockedNetworks contains all private/reserved IP ranges -// that should be blocked to prevent SSRF attacks. +// that should be blocked to prevent SSRF attacks. An operator +// can permit specific blocks out of this set with +// ALLOWED_EGRESS_CIDRS; see Guard. // //nolint:gochecknoglobals // package-level network list is appropriate here var blockedNetworks []*net.IPNet +// alwaysBlockedNetworks are the ranges no configuration can +// open: the link-local blocks and the cloud instance metadata +// endpoints that live outside them. Reaching one is credential +// or user-data theft rather than delivery to an internal +// service, so a supplied CIDR that covers such an address still +// leaves it blocked. +// +// Inclusion criterion — an address belongs here only if BOTH +// hold, and every entry below satisfies both: +// +// 1. It is a fixed address assigned by the provider, or a +// range reserved by IANA — never one the operator chose. +// That is what makes a host route free: it cannot collide +// with anything the operator runs. +// 2. Reaching it discloses credentials, or user data or +// bootstrap material — something granting onward access, or +// not cheaply rotated. +// +// Both halves are load-bearing, so use them to refuse a +// candidate and say why. An endpoint disclosing only the +// operator's own inventory (instance id, region, disks, NICs) +// fails (2): letting a delivery target reach the operator's own +// infrastructure is the feature ALLOWED_EGRESS_CIDRS exists to +// provide. But (2) is not "IAM credentials only" either — +// fd00:42::42 serves /user_data and /conf rather than tokens, +// and user data routinely carries bootstrap secrets. An address +// stays out if it fails (1) however well it clears (2): a host +// route inside a block operators really assign from, such as +// 10.0.0.0/8, can collide with a real internal service and +// forfeits the justification in (1). +// +// A publicly routable unicast address does not belong here even +// when it clears both halves. Nothing in this list can be +// reopened, so putting a public address here leaves the operator +// no escape hatch at all — the condition ALLOWED_EGRESS_CIDRS +// exists to remove. Default-block it in blockedNetworks instead, +// which an allowlist can override. +// +// This is a criterion, not an enumeration of every metadata +// address in existence. +// +// Every entry is either already in blockedNetworks — this list is +// what makes it unconditional — or an alternate encoding of +// 169.254.169.254 that Contains does not match against +// 169.254.0.0/16. Every entry outside the link-local blocks is a +// /32 or /128 host route, so blocking it costs an operator +// nothing else on the surrounding network. +// +// Derive membership from the address, never from the vendor's +// prose. Several providers call these endpoints "link-local" or +// even "localhost" in their own documentation while the address +// is a ULA outside fe80::/10, so a set derived from the docs +// comes out wrong. +// +//nolint:gochecknoglobals // package-level network list is appropriate here +var alwaysBlockedNetworks []*net.IPNet + //nolint:gochecknoinits // init is the idiomatic way to parse CIDRs once at startup func init() { - cidrs := []string{ + blockedNetworks = mustParseCIDRs([]string{ "127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", @@ -56,7 +122,72 @@ func init() { "::1/128", "fc00::/7", "fe80::/10", - } + }) + + // Every entry is named. The set must not grow or shrink + // without a matching change to + // TestAlwaysBlockedNetworks_PinnedSet. + // + // The IPv4-mapped form ::ffff:169.254.169.254 needs no + // entry: net.IPNet.Contains normalises it via To4() before + // comparing, so 169.254.0.0/16 already matches it. To4() + // does not normalise the IPv4-compatible or NAT64 forms, + // which is why those are listed separately. + alwaysBlockedNetworks = mustParseCIDRs([]string{ + // IPv4 link-local, carrying the 169.254.169.254 + // metadata service used by AWS, Azure, DigitalOcean, + // Hetzner, OpenStack and others. Not Alibaba, which uses + // 100.100.100.200 below exclusively. + "169.254.0.0/16", + // IPv6 link-local, its IPv6 counterpart. + "fe80::/10", + + // IPv6 metadata endpoints in ULA space. Each is a host + // route, and fd00::/8 is an ordinary block for an + // operator to allowlist, so without these entries that + // one allowlist line hands out cloud credentials on + // every provider below. + // + // AWS IPv6 IMDS. + "fd00:ec2::254/128", + // AWS EKS Pod Identity Agent, which issues pod identity + // credentials. A second AWS endpoint, distinct from + // IMDS above. AWS's own docs call it "localhost". + "fd00:ec2::23/128", + // GCP metadata server for IPv6-only instances. + "fd20:ce::254/128", + // Oracle OCI IMDS, serving /opc/v2 instance principals. + "fd00:c1::a9fe:a9fe/128", + // Scaleway metadata, serving /user_data and /conf. + "fd00:42::42/128", + // Linode/Akamai metadata. Akamai's docs call it + // "link-local"; it is not. + "fd00:a9fe:a9fe::1/128", + + // IPv4 metadata endpoints outside link-local. + // + // Alibaba Cloud metadata. It sits in CGNAT + // 100.64.0.0/10, which Tailscale also uses, so an + // operator allowlisting a Tailscale peer's range would + // otherwise reopen it. + "100.100.100.200/32", + // Oracle Cloud Classic metadata. Inside the blocked + // 192.0.0.0/24, so this entry is what stops an + // allowlist from opening it. + "192.0.0.192/32", + + // 169.254.169.254 as an IPv4-compatible IPv6 address. + "::a9fe:a9fe/128", + // 169.254.169.254 behind the NAT64 well-known prefix. + "64:ff9b::a9fe:a9fe/128", + }) +} + +// mustParseCIDRs parses a list of CIDR literals, panicking on a +// bad one. The inputs are compile-time constants, so a failure +// is a programming error rather than a runtime condition. +func mustParseCIDRs(cidrs []string) []*net.IPNet { + networks := make([]*net.IPNet, 0, len(cidrs)) for _, cidr := range cidrs { _, network, err := net.ParseCIDR(cidr) @@ -67,16 +198,15 @@ func init() { )) } - blockedNetworks = append( - blockedNetworks, network, - ) + networks = append(networks, network) } + + return networks } -// isBlockedIP checks whether an IP address falls within -// any blocked private/reserved network range. -func isBlockedIP(ip net.IP) bool { - for _, network := range blockedNetworks { +// matchesAny reports whether ip falls inside any of networks. +func matchesAny(networks []*net.IPNet, ip net.IP) bool { + for _, network := range networks { if network.Contains(ip) { return true } @@ -85,14 +215,50 @@ func isBlockedIP(ip net.IP) bool { return false } +// isBlockedIP checks whether an IP address falls within +// any blocked private/reserved network range, before any +// operator allowlist is considered. +func isBlockedIP(ip net.IP) bool { + return matchesAny(blockedNetworks, ip) +} + +// Guard makes every SSRF decision in the process. +// +// It holds the operator's ALLOWED_EGRESS_CIDRS allowlist and +// applies it in exactly one place, checkIP, which both the +// target-creation validator (ValidateTargetURL) and the delivery +// dialer call. Routing both through the same function is the +// point: when the two paths decided separately they drifted and +// disagreed, which is what made a target creatable but +// undeliverable. +// +// The guard is always on. The allowlist only ever adds specific +// networks to what the default blocklist refuses, and no +// configuration turns the guard off wholesale. +type Guard struct { + // allowed is the operator's ALLOWED_EGRESS_CIDRS. Empty + // (the default) means the default blocklist stands as-is. + allowed []netip.Prefix +} + +// NewGuard builds the process-wide SSRF guard from configuration. +func NewGuard(cfg *config.Config) *Guard { + return &Guard{allowed: cfg.AllowedEgressCIDRs} +} + // ValidateTargetURL checks that an HTTP delivery target // URL is safe from SSRF attacks. -func ValidateTargetURL( +func (g *Guard) ValidateTargetURL( ctx context.Context, targetURL string, ) error { parsed, err := url.Parse(targetURL) if err != nil { - return fmt.Errorf("invalid URL: %w", err) + // url.Parse embeds the whole URL in its error, and + // this one is logged and shown; mask it. Every other + // branch below reports only the hostname. + return fmt.Errorf( + "invalid URL: %w", maskURLError(err), + ) } err = validateScheme(parsed.Scheme) @@ -106,36 +272,79 @@ func ValidateTargetURL( } if ip := net.ParseIP(host); ip != nil { - return checkBlockedIP(ip) + return g.checkIP(ip) } - return validateHostname(ctx, host) + return g.validateHostname(ctx, host) } -func validateScheme(scheme string) error { - if scheme != "http" && scheme != "https" { +// NewSSRFSafeTransport creates an http.Transport with a +// custom DialContext that refuses connections to any address +// this guard blocks. It resolves and checks at dial time, so a +// name that passed validation but now answers with a blocked +// address (DNS rebinding) is still refused. +func (g *Guard) NewSSRFSafeTransport() *http.Transport { + return &http.Transport{ + DialContext: g.ssrfDialContext, + } +} + +// allows reports whether ip falls inside the operator's +// configured egress allowlist. +func (g *Guard) allows(ip net.IP) bool { + if len(g.allowed) == 0 { + return false + } + + addr, ok := netip.AddrFromSlice(ip) + if !ok { + return false + } + + // Config unmaps every parsed prefix, so an IPv4-mapped + // address has to be unmapped too or it would never match. + addr = addr.Unmap() + + for _, prefix := range g.allowed { + if prefix.Contains(addr) { + return true + } + } + + return false +} + +// checkIP is the single point at which SSRF policy is decided. +// +// The order is the policy: +// +// 1. alwaysBlockedNetworks is refused before the allowlist is +// consulted, so no configured CIDR reaches link-local or a +// cloud instance metadata endpoint. +// 2. The allowlist is consulted next, so a listed private +// network becomes reachable. +// 3. Everything else keeps the default blocklist's answer. +func (g *Guard) checkIP(ip net.IP) error { + if matchesAny(alwaysBlockedNetworks, ip) { return fmt.Errorf( - "unsupported URL scheme %q: %w", - scheme, errInvalidScheme, + "target IP %s: %w", ip, errBlockedMetadata, ) } - return nil -} + if g.allows(ip) { + return nil + } -func checkBlockedIP(ip net.IP) error { if isBlockedIP(ip) { return fmt.Errorf( - "target IP %s is in a blocked "+ - "private/reserved range: %w", - ip, errBlockedIP, + "target IP %s: %w", ip, errBlockedIP, ) } return nil } -func validateHostname( +func (g *Guard) validateHostname( ctx context.Context, host string, ) error { dnsCtx, cancel := context.WithTimeout( @@ -160,11 +369,11 @@ func validateHostname( } for _, ipAddr := range ips { - if isBlockedIP(ipAddr.IP) { + err = g.checkIP(ipAddr.IP) + if err != nil { return fmt.Errorf( - "hostname %q resolves to blocked "+ - "IP %s: %w", - host, ipAddr.IP, errBlockedIP, + "hostname %q resolves to a blocked address: %w", + host, err, ) } } @@ -172,16 +381,7 @@ func validateHostname( return nil } -// NewSSRFSafeTransport creates an http.Transport with a -// custom DialContext that blocks connections to -// private/reserved IP addresses. -func NewSSRFSafeTransport() *http.Transport { - return &http.Transport{ - DialContext: ssrfDialContext, - } -} - -func ssrfDialContext( +func (g *Guard) ssrfDialContext( ctx context.Context, network, addr string, ) (net.Conn, error) { @@ -204,11 +404,11 @@ func ssrfDialContext( } for _, ipAddr := range ips { - if isBlockedIP(ipAddr.IP) { + err = g.checkIP(ipAddr.IP) + if err != nil { return nil, fmt.Errorf( - "ssrf: connection to %s (%s) "+ - "blocked: %w", - host, ipAddr.IP, errBlockedIP, + "ssrf: connection to %s blocked: %w", + host, err, ) } } @@ -220,3 +420,14 @@ func ssrfDialContext( net.JoinHostPort(ips[0].IP.String(), port), ) } + +func validateScheme(scheme string) error { + if scheme != "http" && scheme != "https" { + return fmt.Errorf( + "unsupported URL scheme %q: %w", + scheme, errInvalidScheme, + ) + } + + return nil +} diff --git a/internal/delivery/ssrf_allowlist_test.go b/internal/delivery/ssrf_allowlist_test.go new file mode 100644 index 0000000..74f0f35 --- /dev/null +++ b/internal/delivery/ssrf_allowlist_test.go @@ -0,0 +1,562 @@ +package delivery_test + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "net/netip" + "net/url" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/delivery" +) + +// Addresses the SSRF tests in this package share. +const ( + // metadataIP is the cloud instance metadata address, and + // metadataURL an endpoint on it. The guard must never reach + // either, whatever an operator lists. + metadataIP = "169.254.169.254" + metadataURL = "http://" + metadataIP + "/latest/meta-data/" + + // loopbackHookURL is a target on this host: blocked by + // default, reachable only once an operator allowlists + // loopback. + loopbackHookURL = "http://127.0.0.1/hook" + + // publicIP is an ordinary public address, which the guard + // permits with or without an allowlist. + publicIP = "93.184.216.34" + + // allowAllIPv4 and allowAllIPv6 are the widest allowlist + // entries expressible: the whole internet, in each family. + // Nothing unconditionally blocked may be reachable under + // them. + allowAllIPv4 = "0.0.0.0/0" + allowAllIPv6 = "::/0" + + // allowAllULA is the ordinary ULA block an operator lists to + // reach their own IPv6 network. Several providers park a + // metadata endpoint inside it. + allowAllULA = "fd00::/8" + + // metadataRefusalClause is the part of the refusal that only + // alwaysBlockedNetworks produces. Asserting it, rather than + // the bare word "blocked", is what proves the unconditional + // set did the refusing and not the default blocklist. + metadataRefusalClause = "ALLOWED_EGRESS_CIDRS cannot open it" +) + +// TestGuardAllowlist_PermittedCIDRDelivers proves the escape +// hatch actually works end to end: with 127.0.0.0/8 allowed, the +// guard's own transport connects to a loopback server and gets a +// response back. The default guard, given the identical URL, +// refuses it — so the delivery succeeds because of the allowlist +// and nothing else. +func TestGuardAllowlist_PermittedCIDRDelivers(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + }, + )) + t.Cleanup(srv.Close) + + // httptest listens on loopback, which the default blocklist + // covers: exactly the "forward to a service on this host" + // case the allowlist exists for. + requireLoopback(t, srv.URL) + + guard := delivery.NewTestGuard( + netip.MustParsePrefix("127.0.0.0/8"), + ) + + require.NoError(t, + guard.ValidateTargetURL(context.Background(), srv.URL), + "an allowlisted loopback target must pass validation", + ) + + client := &http.Client{ + Timeout: 5 * time.Second, + Transport: guard.NewSSRFSafeTransport(), + } + + req, err := http.NewRequestWithContext( + context.Background(), http.MethodPost, srv.URL, nil, + ) + require.NoError(t, err) + + resp, err := client.Do(req) + require.NoError(t, + err, "an allowlisted loopback target must be deliverable", + ) + + defer func() { _ = resp.Body.Close() }() + + assert.Equal(t, http.StatusNoContent, resp.StatusCode) + + // The same URL through the default guard must still fail, or + // this test would pass without the allowlist doing anything. + assert.Error(t, + delivery.NewTestGuard().ValidateTargetURL( + context.Background(), srv.URL, + ), + "without the allowlist the same target must be refused", + ) +} + +// TestGuardAllowlist_UnlistedPrivateStillRefused proves the +// allowlist grants only what it names. A guard that opens one +// private block must keep refusing every other one, at both the +// validation and the delivery entry point. +func TestGuardAllowlist_UnlistedPrivateStillRefused(t *testing.T) { + t.Parallel() + + // Only 10.1.0.0/16 is open — a narrow block inside a much + // wider private range, so the test can tell "permits the + // listed block" from "permits anything private". + guard := delivery.NewTestGuard( + netip.MustParsePrefix("10.1.0.0/16"), + ) + + refused := []string{ + "http://192.168.1.10/hook", + "http://172.16.0.1/hook", + loopbackHookURL, + "http://[fc00::1]/hook", + "http://100.64.0.1/hook", + // Private, adjacent to the allowed block, outside it. + "http://10.2.0.1/hook", + } + + for _, target := range refused { + t.Run(target, func(t *testing.T) { + t.Parallel() + + err := guard.ValidateTargetURL( + context.Background(), target, + ) + require.Error(t, + err, "%s is not allowlisted and must be refused", + target, + ) + assert.Contains(t, err.Error(), "blocked") + + assertDialRefused(t, guard, target) + }) + } + + // The block that is listed must in fact be permitted, so the + // refusals above are selective rather than a guard that + // ignores its allowlist entirely. + assert.NoError(t, + guard.ValidateTargetURL( + context.Background(), "http://10.1.2.3/hook", + ), + "the allowlisted block must be permitted", + ) +} + +// TestGuardAllowlist_MetadataAlwaysRefused is the load-bearing +// case: cloud instance metadata endpoints are credential theft +// rather than delivery to an internal service, so no allowlist +// reaches one. Every guard below names a CIDR that covers its +// target — including 0.0.0.0/0, ::/0, and the ordinary ULA and +// CGNAT blocks an operator would really list — and the address +// must stay refused anyway, on both the validation and the +// delivery path. +func TestGuardAllowlist_MetadataAlwaysRefused(t *testing.T) { + t.Parallel() + + for _, tt := range metadataAlwaysRefusedCases() { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + guard := delivery.NewTestGuard( + netip.MustParsePrefix(tt.allow), + ) + + err := guard.ValidateTargetURL( + context.Background(), tt.target, + ) + require.Error(t, + err, + "%s must stay blocked even though %s covers it", + tt.target, tt.allow, + ) + assert.Contains(t, + err.Error(), + metadataRefusalClause, + "the refusal must say why it cannot be opened", + ) + + // The metadata clause, not just "blocked": that is + // what distinguishes the unconditional set from the + // ordinary blocklist. + assertDialRefusedWith( + t, guard, tt.target, metadataRefusalClause, + ) + }) + } +} + +// metadataAlwaysRefusedCase is one (allowlist, target) pair that +// must be refused: allow covers target, and target must stay +// blocked regardless. +type metadataAlwaysRefusedCase struct { + name string + allow string + target string +} + +// metadataAlwaysRefusedCases enumerates every unconditionally +// blocked address together with an allowlist entry that would +// otherwise reach it. Split by family of address only to stay +// under the function-length limit. +func metadataAlwaysRefusedCases() []metadataAlwaysRefusedCase { + cases := linkLocalRefusedCases() + cases = append(cases, ulaMetadataRefusedCases()...) + cases = append(cases, ipv4MetadataRefusedCases()...) + + return append(cases, encodedMetadataRefusedCases()...) +} + +// linkLocalRefusedCases covers the link-local blocks, including +// an operator naming the metadata address outright. +func linkLocalRefusedCases() []metadataAlwaysRefusedCase { + return []metadataAlwaysRefusedCase{ + { + name: "exact metadata host", + allow: "169.254.169.254/32", + target: metadataURL, + }, + { + name: "whole link-local block", + allow: "169.254.0.0/16", + target: metadataURL, + }, + { + name: "supernet covering link-local", + allow: "169.0.0.0/8", + target: metadataURL, + }, + { + name: "the entire IPv4 internet", + allow: allowAllIPv4, + target: metadataURL, + }, + { + name: "other link-local address", + allow: allowAllIPv4, + target: "http://169.254.1.1/", + }, + { + name: "IPv6 link-local", + allow: allowAllIPv6, + target: "http://[fe80::1]/", + }, + } +} + +// ulaMetadataRefusedCases covers the metadata endpoints parked +// in ULA space. Every one is opened by the single ordinary +// allowlist entry fd00::/8, which is the whole reason they need +// their own /128 host routes: fe80::/10 does not cover a ULA, +// whatever the vendor's documentation calls the address. +func ulaMetadataRefusedCases() []metadataAlwaysRefusedCase { + return []metadataAlwaysRefusedCase{ + { + name: "AWS IPv6 IMDS under an allowlisted ULA block", + allow: allowAllULA, + target: "http://[fd00:ec2::254]/latest/meta-data/", + }, + { + // A second AWS credential endpoint, distinct from + // IMDS. AWS's own docs call this one "localhost". + name: "AWS EKS Pod Identity under an allowlisted ULA block", + allow: allowAllULA, + target: "http://[fd00:ec2::23]/v1/credentials", + }, + { + name: "GCP IPv6 metadata under an allowlisted ULA block", + allow: allowAllULA, + target: "http://[fd20:ce::254]/computeMetadata/v1/", + }, + { + name: "Oracle OCI IPv6 IMDS under an allowlisted ULA block", + allow: allowAllULA, + target: "http://[fd00:c1::a9fe:a9fe]/opc/v2/instance/", + }, + { + name: "Scaleway IPv6 metadata under an allowlisted ULA block", + allow: allowAllULA, + target: "http://[fd00:42::42]/conf", + }, + { + // Akamai's docs call this "link-local"; it is a ULA, + // so fe80::/10 does not cover it. + name: "Linode IPv6 metadata under an allowlisted ULA block", + allow: allowAllULA, + target: "http://[fd00:a9fe:a9fe::1]/v1/instance", + }, + } +} + +// ipv4MetadataRefusedCases covers the IPv4 metadata endpoints +// that sit outside link-local: one in CGNAT and one in the +// blocked 192.0.0.0/24, each reachable only through an allowlist +// that this set overrides. +func ipv4MetadataRefusedCases() []metadataAlwaysRefusedCase { + return []metadataAlwaysRefusedCase{ + { + // Tailscale uses 100.64.0.0/10, so an operator + // forwarding to a Tailscale peer lists exactly this. + name: "Alibaba metadata under allowlisted CGNAT", + allow: "100.64.0.0/10", + target: "http://100.100.100.200/latest/meta-data/", + }, + { + // Inside the already-blocked 192.0.0.0/24, so only + // an allowlist can reach it — and must not. + name: "Oracle Cloud Classic metadata under 0.0.0.0/0", + allow: allowAllIPv4, + target: "http://192.0.0.192/latest/meta-data/", + }, + } +} + +// encodedMetadataRefusedCases covers the alternate IPv6 +// encodings of 169.254.169.254. +func encodedMetadataRefusedCases() []metadataAlwaysRefusedCase { + return []metadataAlwaysRefusedCase{ + { + // To4() does not normalise the IPv4-compatible form, + // so this needs its own always-blocked entry. + name: "IPv4-compatible IPv6 form of the metadata IP", + allow: allowAllIPv6, + target: "http://[::a9fe:a9fe]/latest/meta-data/", + }, + { + // Nor the NAT64 well-known prefix form. + name: "NAT64 form of the metadata IP", + allow: allowAllIPv6, + target: "http://[64:ff9b::a9fe:a9fe]/latest/meta-data/", + }, + { + // Already refused before this change: IPNet.Contains + // calls To4() first, so the mapped form matches + // 169.254.0.0/16. Pinned so it cannot regress. + // + // Allowed under 0.0.0.0/0 rather than ::/0: allows() + // unmaps before matching, so ::/0 would not cover the + // unmapped v4 address and the case would not prove + // the allowlist was overridden. + name: "IPv4-mapped IPv6 form of the metadata IP", + allow: allowAllIPv4, + target: "http://[::ffff:169.254.169.254]/latest/meta-data/", + }, + } +} + +// TestGuardAllowlist_PublicUnaffected asserts the allowlist does +// not narrow anything: public addresses were reachable before it +// existed and stay reachable, whether or not a list is set. +func TestGuardAllowlist_PublicUnaffected(t *testing.T) { + t.Parallel() + + guards := map[string]*delivery.Guard{ + "default": delivery.NewTestGuard(), + "with allowlist": delivery.NewTestGuard( + netip.MustParsePrefix("10.0.0.0/8"), + ), + } + + for name, guard := range guards { + t.Run(name, func(t *testing.T) { + t.Parallel() + + assert.NoError(t, + guard.ValidateTargetURL( + context.Background(), + "http://"+publicIP+"/webhook", + ), + ) + }) + } +} + +// TestGuardCheckIP_BothPathsShareOneDecision asserts that the +// validator and the dialer are not two policies that happen to +// agree: both are defined in terms of checkIP, so the exported +// decision function is the whole answer for a given address. +func TestGuardCheckIP_BothPathsShareOneDecision(t *testing.T) { + t.Parallel() + + guard := delivery.NewTestGuard( + netip.MustParsePrefix("10.0.0.0/8"), + ) + + tests := []struct { + ip string + allowed bool + }{ + {"10.1.2.3", true}, + {publicIP, true}, + {"192.168.1.1", false}, + {"127.0.0.1", false}, + {metadataIP, false}, + } + + for _, tt := range tests { + t.Run(tt.ip, func(t *testing.T) { + t.Parallel() + + ip := net.ParseIP(tt.ip) + require.NotNil(t, ip) + + decision := guard.ExportCheckIP(ip) + + validation := guard.ValidateTargetURL( + context.Background(), "http://"+hostFor(tt.ip)+"/x", + ) + + if tt.allowed { + require.NoError(t, decision) + require.NoError(t, validation) + + return + } + + require.Error(t, decision) + require.Error(t, validation, + "validation must refuse what checkIP refuses", + ) + }) + } +} + +// TestAlwaysBlockedNetworks_PinnedSet pins the unconditional set +// exactly, so it cannot quietly grow or shrink. +// +// It stays deliberately small. Everything else in the default +// blocklist is an operator's own network and must remain +// openable, or the escape hatch would not work — which is why +// the metadata endpoints outside the link-local range are host +// routes rather than the blocks that contain them. +func TestAlwaysBlockedNetworks_PinnedSet(t *testing.T) { + t.Parallel() + + nets := delivery.ExportAlwaysBlockedNetworks() + + got := make([]string, 0, len(nets)) + for _, n := range nets { + got = append(got, n.String()) + } + + want := []string{ + // IPv4 link-local: the 169.254.169.254 metadata + // service on AWS, Azure and others. + "169.254.0.0/16", + // IPv6 link-local. + "fe80::/10", + // AWS IPv6 IMDS, inside the ULA space an operator may + // legitimately allowlist. + "fd00:ec2::254/128", + // AWS EKS Pod Identity Agent, likewise ULA. + "fd00:ec2::23/128", + // GCP metadata for IPv6-only instances, likewise ULA. + "fd20:ce::254/128", + // Oracle OCI IMDS over IPv6, likewise ULA. + "fd00:c1::a9fe:a9fe/128", + // Scaleway metadata over IPv6, likewise ULA. + "fd00:42::42/128", + // Linode/Akamai metadata over IPv6, likewise ULA. + "fd00:a9fe:a9fe::1/128", + // Alibaba Cloud metadata, inside CGNAT. + "100.100.100.200/32", + // Oracle Cloud Classic metadata, inside the blocked + // 192.0.0.0/24. + "192.0.0.192/32", + // 169.254.169.254 as an IPv4-compatible IPv6 address. + "::a9fe:a9fe/128", + // 169.254.169.254 behind the NAT64 well-known prefix. + "64:ff9b::a9fe:a9fe/128", + } + + assert.Equal(t, want, got) +} + +// requireLoopback fails the test unless rawURL's host is a +// loopback address, so the allowlist test cannot silently stop +// exercising a blocked range. +func requireLoopback(t *testing.T, rawURL string) { + t.Helper() + + parsed, err := url.Parse(rawURL) + require.NoError(t, err) + + ip := net.ParseIP(parsed.Hostname()) + require.NotNil(t, ip, "test server host must be an IP literal") + require.True(t, ip.IsLoopback(), + "test server must listen on loopback, got %s", ip, + ) +} + +// assertDialRefused asserts the guard's transport refuses to +// connect to target, which is the delivery-time half of the +// policy. It never reaches the network: the guard checks the +// resolved address before dialling. +func assertDialRefused( + t *testing.T, guard *delivery.Guard, target string, +) { + t.Helper() + + assertDialRefusedWith(t, guard, target, "blocked") +} + +// assertDialRefusedWith is assertDialRefused with the refusal +// text pinned. Callers testing the unconditional set pass +// metadataRefusalClause so the subtest cannot pass on an +// ordinary blocklist refusal instead. +func assertDialRefusedWith( + t *testing.T, guard *delivery.Guard, target, clause string, +) { + t.Helper() + + client := &http.Client{ + Timeout: 5 * time.Second, + Transport: guard.NewSSRFSafeTransport(), + } + + req, err := http.NewRequestWithContext( + context.Background(), http.MethodPost, target, nil, + ) + require.NoError(t, err) + + resp, err := client.Do(req) + if resp != nil { + _ = resp.Body.Close() + } + + require.Error(t, err, + "delivery to %s must be refused by the dialer", target, + ) + assert.Contains(t, err.Error(), clause, + "the refusal must come from the SSRF guard", + ) +} + +// hostFor renders an IP as it appears in a URL host, bracketing +// IPv6 literals. +func hostFor(ip string) string { + if net.ParseIP(ip).To4() == nil { + return "[" + ip + "]" + } + + return ip +} diff --git a/internal/delivery/ssrf_test.go b/internal/delivery/ssrf_test.go index d919d16..14454e9 100644 --- a/internal/delivery/ssrf_test.go +++ b/internal/delivery/ssrf_test.go @@ -31,10 +31,10 @@ func TestIsBlockedIP_PrivateRanges(t *testing.T) { {"192.168.0.1", "192.168.0.1", true}, {"192.168.255.255", "192.168.255.255", true}, {"169.254.0.1", "169.254.0.1", true}, - {"169.254.169.254", "169.254.169.254", true}, + {metadataIP, metadataIP, true}, {"8.8.8.8", "8.8.8.8", false}, {"1.1.1.1", "1.1.1.1", false}, - {"93.184.216.34", "93.184.216.34", false}, + {publicIP, publicIP, false}, {"::1", "::1", true}, {"fd00::1", "fd00::1", true}, {"fc00::1", "fc00::1", true}, @@ -72,12 +72,12 @@ func TestValidateTargetURL_Blocked(t *testing.T) { t.Parallel() blockedURLs := []string{ - "http://127.0.0.1/hook", + loopbackHookURL, "http://127.0.0.1:8080/hook", "https://10.0.0.1/hook", "http://192.168.1.1/webhook", "http://172.16.0.1/api", - "http://169.254.169.254/latest/meta-data/", + metadataURL, "http://[::1]/hook", "http://[fc00::1]/hook", "http://[fe80::1]/hook", @@ -88,7 +88,7 @@ func TestValidateTargetURL_Blocked(t *testing.T) { t.Run(u, func(t *testing.T) { t.Parallel() - err := delivery.ValidateTargetURL( + err := delivery.NewTestGuard().ValidateTargetURL( context.Background(), u, ) @@ -112,7 +112,7 @@ func TestValidateTargetURL_Allowed(t *testing.T) { t.Run(u, func(t *testing.T) { t.Parallel() - err := delivery.ValidateTargetURL( + err := delivery.NewTestGuard().ValidateTargetURL( context.Background(), u, ) @@ -126,7 +126,7 @@ func TestValidateTargetURL_Allowed(t *testing.T) { func TestValidateTargetURL_InvalidScheme(t *testing.T) { t.Parallel() - err := delivery.ValidateTargetURL( + err := delivery.NewTestGuard().ValidateTargetURL( context.Background(), "ftp://example.com/hook", ) @@ -140,7 +140,7 @@ func TestValidateTargetURL_InvalidScheme(t *testing.T) { func TestValidateTargetURL_EmptyHost(t *testing.T) { t.Parallel() - err := delivery.ValidateTargetURL( + err := delivery.NewTestGuard().ValidateTargetURL( context.Background(), "http:///path", ) @@ -150,7 +150,7 @@ func TestValidateTargetURL_EmptyHost(t *testing.T) { func TestValidateTargetURL_InvalidURL(t *testing.T) { t.Parallel() - err := delivery.ValidateTargetURL( + err := delivery.NewTestGuard().ValidateTargetURL( context.Background(), "://invalid", ) diff --git a/internal/delivery/target.go b/internal/delivery/target.go index 264ce2f..17c3bf5 100644 --- a/internal/delivery/target.go +++ b/internal/delivery/target.go @@ -27,6 +27,12 @@ type Scheduler interface { // own circuit breaker, and reschedules via the injected // Scheduler. Fire-and-forget targets simply record a single // attempt. +// +// An implementation reports each attempt it actually dispatches to +// Engine.observeAttempt, alongside the DeliveryResult it records for +// it. Deliver is also entered for attempts that never happen — an +// open circuit breaker refuses one — so the count cannot be taken +// from around this call. type Target interface { Deliver( ctx context.Context, @@ -74,6 +80,12 @@ type attemptResult struct { errMsg string } +// elapsed returns how long the attempt took. The field is stored in +// milliseconds because that is what DeliveryResult persists. +func (r attemptResult) elapsed() time.Duration { + return time.Duration(r.duration) * time.Millisecond +} + // initTargets builds the target registry, wiring each target // to the engine's persistence helpers and giving the HTTP and // Slack targets the shared SSRF-safe client. It is called by @@ -90,12 +102,15 @@ func (e *Engine) initTargets(client *http.Client) { client: client, } + dbT := &databaseTarget{eng: e} + e.httpTarget = httpT + e.dbTarget = dbT e.targets = map[database.TargetType]Target{ database.TargetTypeHTTP: httpT, database.TargetTypeSlack: slackT, - database.TargetTypeDatabase: &databaseTarget{eng: e}, + database.TargetTypeDatabase: dbT, database.TargetTypeLog: &logTarget{eng: e}, } } diff --git a/internal/delivery/target_config_edit.go b/internal/delivery/target_config_edit.go new file mode 100644 index 0000000..42da682 --- /dev/null +++ b/internal/delivery/target_config_edit.go @@ -0,0 +1,119 @@ +package delivery + +import ( + "encoding/json" + "errors" + "fmt" + + "sneak.berlin/go/webhooker/internal/database" +) + +// errUnknownTargetTypeForEdit is returned when a stored target has a +// type the edit form has no field set for. +var errUnknownTargetTypeForEdit = errors.New( + "unknown target type", +) + +// TargetConfigForm is the UNMASKED projection of a target's stored +// configuration, for pre-filling the target edit form. +// +// It is the deliberate exception to the rule TargetView enforces +// everywhere else: TargetView exists so that no template can render +// a target's stored blob, because a destination URL's path segments +// and a header value are both routinely the credential. An operator +// cannot correct a value they cannot see, so the edit form — and +// only the edit form — is shown the full value. +// +// Everything that keeps that exception narrow lives at the call +// site: the route is behind RequireAuth and the webhook's ownership +// check, and its group sets NoCache so the rendered secret is not +// written to a shared cache. Do not reach for this type from any +// other page. +type TargetConfigForm struct { + // URL is the destination for an HTTP target and the webhook + // URL for a Slack target. + URL string + // Headers is the HTTP target's configured headers in the + // textarea representation, one "Name: value" per line. + Headers string + // Timeout is the HTTP target's per-request timeout in seconds, + // empty when unset. + Timeout string + // Expiry is the database (archive) target's row expiry. + Expiry string +} + +// NewTargetConfigForm parses a target's stored configuration into +// the edit form's fields. +// +// A configuration that does not parse is an error rather than a +// zero-valued form that silently looks like a target with no +// settings. The caller shows the operator that the stored value +// could not be read, so that saving the form is understood as +// replacing it rather than preserving it. +func NewTargetConfigForm( + t *database.Target, +) (TargetConfigForm, error) { + switch t.Type { + case database.TargetTypeHTTP: + cfg, err := parseHTTPConfig(t.Config) + if err != nil { + return TargetConfigForm{}, err + } + + return TargetConfigForm{ + URL: cfg.URL, + Headers: FormatTargetHeaders(cfg.Headers), + Timeout: FormatTargetTimeout(cfg.Timeout), + }, nil + case database.TargetTypeSlack: + cfg, err := parseSlackConfig(t.Config) + if err != nil { + return TargetConfigForm{}, err + } + + return TargetConfigForm{URL: cfg.WebhookURL}, nil + case database.TargetTypeDatabase: + return databaseConfigForm(t.Config) + case database.TargetTypeLog: + // The log target takes no configuration. + return TargetConfigForm{}, nil + default: + return TargetConfigForm{}, fmt.Errorf( + "%w: %q", errUnknownTargetTypeForEdit, t.Type, + ) + } +} + +// databaseConfigForm parses an archive target's optional expiry. +// An absent or empty configuration is the keep-forever default and +// yields an empty field, so re-saving the form unchanged stores the +// same empty configuration it started with. An expiry that is set +// but not a valid duration is an error, not a blank field. +func databaseConfigForm( + configJSON string, +) (TargetConfigForm, error) { + if configJSON == "" { + return TargetConfigForm{}, nil + } + + var cfg databaseTargetConfig + + err := json.Unmarshal([]byte(configJSON), &cfg) + if err != nil { + return TargetConfigForm{}, fmt.Errorf( + "parsing config JSON: %w", err, + ) + } + + if cfg.Expiry == "" || cfg.Expiry == archiveExpiryNever { + return TargetConfigForm{}, nil + } + + err = ValidateArchiveExpiry(cfg.Expiry) + if err != nil { + return TargetConfigForm{}, err + } + + return TargetConfigForm{Expiry: cfg.Expiry}, nil +} diff --git a/internal/delivery/target_config_view.go b/internal/delivery/target_config_view.go new file mode 100644 index 0000000..fc09a40 --- /dev/null +++ b/internal/delivery/target_config_view.go @@ -0,0 +1,239 @@ +package delivery + +import ( + "encoding/json" + "fmt" + "strconv" + + "sneak.berlin/go/webhooker/internal/database" +) + +// configUnavailable is what a target's configuration renders +// as when it is absent, of an unknown type, or does not +// parse. The stored blob is never shown as a fallback: it can +// hold a credential (a Slack incoming webhook URL is a bearer +// token) and a UI that prints it leaks that credential into +// browser history, screenshots and screen shares. +const configUnavailable = "(unavailable)" + +// ConfigField is one labelled, display-safe value derived +// from a target's stored configuration. +type ConfigField struct { + Label string + Value string +} + +// deletedNameSuffix marks the name of a target that no longer +// exists. Deletes are soft and delivery history outlives the +// target, so the event log shows names of targets that are gone; +// an operator reading one needs to know it cannot be delivered +// to, replayed to, or configured. +const deletedNameSuffix = " (deleted)" + +// TargetView is the display-safe projection of a target for +// the UI. It deliberately has no raw configuration field, so +// no template — present or future — can render the stored +// blob. +type TargetView struct { + ID string + Name string + + // Deleted reports that this target's row is soft deleted. + // Only views built for historical display carry it set: + // every other projection is of a live row. + Deleted bool + + Type database.TargetType + Active bool + Config []ConfigField +} + +// DisplayName is the name to render, marked when the target has +// been deleted. Templates showing a name against historical data +// must use it rather than Name, which stays the stored name. +func (v TargetView) DisplayName() string { + if v.Deleted { + return v.Name + deletedNameSuffix + } + + return v.Name +} + +// NewTargetViews projects targets for rendering, replacing +// each stored configuration blob with named, display-safe +// fields. +// +// A soft-deleted row projects exactly as a live one does, minus +// the deleted marker on its name: masking is a property of the +// projection, not of the row's state, so a deleted target's +// credential is as unreachable from a template as a live +// target's. +func NewTargetViews( + targets []database.Target, +) []TargetView { + views := make([]TargetView, 0, len(targets)) + + for i := range targets { + t := &targets[i] + + views = append(views, TargetView{ + ID: t.ID, + Name: t.Name, + Deleted: t.DeletedAt.Valid, + Type: t.Type, + Active: t.Active, + Config: targetConfigFields(t), + }) + } + + return views +} + +// targetConfigFields returns the display-safe fields for a +// target's configuration. Anything it cannot parse becomes +// the neutral placeholder. +func targetConfigFields( + t *database.Target, +) []ConfigField { + switch t.Type { + case database.TargetTypeSlack: + return slackConfigFields(t.Config) + case database.TargetTypeHTTP: + return httpConfigFields(t) + case database.TargetTypeDatabase: + return databaseConfigFields(t.Config) + case database.TargetTypeLog: + // The log target takes no configuration. + return nil + default: + return unavailableConfigFields() + } +} + +// unavailableConfigFields is the neutral placeholder shown +// for a configuration that could not be presented. +func unavailableConfigFields() []ConfigField { + return []ConfigField{{ + Label: "Configuration", + Value: configUnavailable, + }} +} + +// slackConfigFields describes a Slack target. Only the masked +// webhook URL is shown; the full URL is the credential. +func slackConfigFields(configJSON string) []ConfigField { + cfg, err := parseSlackConfig(configJSON) + if err != nil { + return unavailableConfigFields() + } + + return []ConfigField{{ + Label: "Webhook URL", + Value: cfg.MaskedWebhookURL(), + }} +} + +// httpConfigFields describes an HTTP target: its destination +// and its retry settings. Header values are not shown — they +// routinely carry authorization tokens — only how many are +// configured. +// +// The destination is masked to scheme and host by the same +// rule the Slack target uses. An HTTP target's destination is +// commonly a Slack, Discord or Teams incoming-webhook endpoint +// whose path segments are the credential, and the field takes +// an arbitrary URL, so no segment can be assumed non-secret. +func httpConfigFields(t *database.Target) []ConfigField { + cfg, err := parseHTTPConfig(t.Config) + if err != nil { + return unavailableConfigFields() + } + + fields := []ConfigField{{ + Label: "Destination URL", + Value: MaskURL(cfg.URL), + }} + + if cfg.Timeout > 0 { + fields = append(fields, ConfigField{ + Label: "Timeout", + Value: strconv.Itoa(cfg.Timeout) + "s", + }) + } + + if len(cfg.Headers) > 0 { + fields = append(fields, ConfigField{ + Label: "Headers", + Value: fmt.Sprintf( + "%d configured", len(cfg.Headers), + ), + }) + } + + return append(fields, retryFields(t)...) +} + +// retryFields describes a target's retry settings, which live +// on the target row rather than in its configuration blob. +func retryFields(t *database.Target) []ConfigField { + retries := strconv.Itoa(t.MaxRetries) + if t.MaxRetries == 0 { + retries += " (fire-and-forget)" + } + + fields := []ConfigField{{ + Label: "Max Retries", + Value: retries, + }} + + if t.MaxQueueSize > 0 { + fields = append(fields, ConfigField{ + Label: "Max Queue Size", + Value: strconv.Itoa(t.MaxQueueSize), + }) + } + + return fields +} + +// databaseConfigFields describes an archive target. Its +// configuration is optional, and an absent or empty expiry +// means the archive is kept forever. An expiry that is set +// but not a valid duration is reported as unavailable rather +// than echoed back. +func databaseConfigFields(configJSON string) []ConfigField { + expiry := archiveExpiryNever + + if configJSON != "" { + var cfg databaseTargetConfig + + err := json.Unmarshal([]byte(configJSON), &cfg) + if err != nil { + return unavailableConfigFields() + } + + if cfg.Expiry != "" { + if ValidateArchiveExpiry(cfg.Expiry) != nil { + return unavailableConfigFields() + } + + expiry = cfg.Expiry + } + } + + return []ConfigField{{ + Label: "Archive Expiry", + Value: expiry, + }} +} + +// MaskedWebhookURL returns the Slack webhook URL reduced to +// its scheme and host, with the path, query and any userinfo +// elided. The path segments are the credential, so none of +// them is shown: the field accepts an arbitrary URL, so no +// segment can be assumed non-secret. A URL that does not +// parse into a scheme and host yields the neutral +// placeholder, never the raw string. +func (c *SlackTargetConfig) MaskedWebhookURL() string { + return MaskURL(c.WebhookURL) +} diff --git a/internal/delivery/target_config_view_test.go b/internal/delivery/target_config_view_test.go new file mode 100644 index 0000000..41211fa --- /dev/null +++ b/internal/delivery/target_config_view_test.go @@ -0,0 +1,376 @@ +package delivery_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" +) + +const ( + // slackSecretPath is the credential-bearing part of a + // Slack incoming webhook URL: everything after the host. + slackSecretPath = "/services/T00000000/B00000000/" + + "XXXXXXXXXXXXXXXXXXXXXXXX" + slackWebhookURL = "https://hooks.slack.com" + + slackSecretPath + + // slackMaskedURL is what a Slack webhook URL renders as + // once masked: scheme and host, path elided. + slackMaskedURL = "https://hooks.slack.com/..." + + // slackTargetName is the target name the Slack projection + // tests use. + slackTargetName = "slack-target" + + viewExampleOrigin = "https://example.com" + viewExampleHook = viewExampleOrigin + "/hook" + viewMaskedOrigin = viewExampleOrigin + "/..." + viewUnavailable = "(unavailable)" + viewExpiryNever = "never" +) + +func TestMaskedWebhookURL(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + url string + want string + }{ + "slack webhook": { + url: slackWebhookURL, + want: slackMaskedURL, + }, + "query string dropped": { + url: viewExampleOrigin + "/a?token=secret", + want: viewExampleOrigin + "/...", + }, + // Fabricated userinfo in a test URL, not a real + // credential. + //nolint:gosec // G101 + "userinfo dropped": { + url: "https://user:pw@example.com/a/b", + want: viewExampleOrigin + "/...", + }, + "no path": { + url: viewExampleOrigin, + want: viewExampleOrigin, + }, + "root path": { + url: viewExampleOrigin + "/", + want: viewExampleOrigin, + }, + "not a url": { + url: "definitely not a url", + want: viewUnavailable, + }, + "empty": { + url: "", + want: viewUnavailable, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + cfg := &delivery.SlackTargetConfig{ + WebhookURL: tc.url, + } + + assert.Equal( + t, tc.want, cfg.MaskedWebhookURL(), + ) + }) + } +} + +// TestMaskedWebhookURL_NeverLeaksPath is the direct +// expression of the rule: whatever the input, the masked +// value never contains a path segment of it. +func TestMaskedWebhookURL_NeverLeaksPath(t *testing.T) { + t.Parallel() + + cfg := &delivery.SlackTargetConfig{ + WebhookURL: slackWebhookURL, + } + + masked := cfg.MaskedWebhookURL() + + assert.NotContains(t, masked, "T00000000") + assert.NotContains(t, masked, "B00000000") + assert.NotContains( + t, masked, "XXXXXXXXXXXXXXXXXXXXXXXX", + ) + assert.NotContains(t, masked, slackSecretPath) +} + +// fieldMap turns a view's config fields into a lookup so +// assertions read by label. +func fieldMap(fields []delivery.ConfigField) map[string]string { + out := make(map[string]string, len(fields)) + for _, f := range fields { + out[f.Label] = f.Value + } + + return out +} + +// viewFor projects a single target and returns its view. +func viewFor( + t *testing.T, + target database.Target, +) delivery.TargetView { + t.Helper() + + views := delivery.NewTargetViews( + []database.Target{target}, + ) + require.Len(t, views, 1) + + return views[0] +} + +// TestNewTargetViews_DeletedTarget proves the projection marks +// a soft-deleted target's name and masks its configuration by +// the same rules a live target's is. Delivery history outlives +// the target it names, so this projection is what an operator +// reads about a target that no longer exists. +func TestNewTargetViews_DeletedTarget(t *testing.T) { + t.Parallel() + + target := slackTarget() + target.DeletedAt = gorm.DeletedAt{ + Time: time.Now(), + Valid: true, + } + + view := viewFor(t, target) + + assert.True(t, view.Deleted) + assert.Equal(t, slackTargetName, view.Name) + assert.Equal( + t, slackTargetName+" (deleted)", view.DisplayName(), + ) + assert.Equal( + t, + map[string]string{"Webhook URL": slackMaskedURL}, + fieldMap(view.Config), + ) +} + +// slackTarget is the live Slack target the projection tests +// share. +func slackTarget() database.Target { + return database.Target{ + Name: slackTargetName, + Type: database.TargetTypeSlack, + Active: true, + Config: `{"webhookUrl":"` + + slackWebhookURL + `"}`, + } +} + +func TestNewTargetViews_Slack(t *testing.T) { + t.Parallel() + + view := viewFor(t, slackTarget()) + + assert.Equal(t, slackTargetName, view.Name) + + // A live target is never marked, so the marker cannot + // reach a name that still exists. + assert.False(t, view.Deleted) + assert.Equal(t, slackTargetName, view.DisplayName()) + + assert.Equal( + t, + map[string]string{"Webhook URL": slackMaskedURL}, + fieldMap(view.Config), + ) +} + +func TestNewTargetViews_HTTP(t *testing.T) { + t.Parallel() + + view := viewFor(t, database.Target{ + Type: database.TargetTypeHTTP, + Config: `{"url":"` + viewExampleHook + `",` + + `"timeout":30,` + + `"headers":{"Authorization":"Bearer sekrit"}}`, + MaxRetries: 5, + MaxQueueSize: 100, + }) + + fields := fieldMap(view.Config) + + assert.Equal( + t, + map[string]string{ + "Destination URL": viewMaskedOrigin, + "Timeout": "30s", + "Headers": "1 configured", + "Max Retries": "5", + "Max Queue Size": "100", + }, + fields, + ) + + // Header values can be credentials and are never shown. + for _, v := range fields { + assert.NotContains(t, v, "sekrit") + } +} + +func TestNewTargetViews_HTTPFireAndForget(t *testing.T) { + t.Parallel() + + view := viewFor(t, database.Target{ + Type: database.TargetTypeHTTP, + Config: `{"url":"` + viewExampleHook + `"}`, + }) + + assert.Equal( + t, + map[string]string{ + "Destination URL": viewMaskedOrigin, + "Max Retries": "0 (fire-and-forget)", + }, + fieldMap(view.Config), + ) +} + +// TestNewTargetViews_HTTPMasksDestinationURL proves the rule +// holds for the http target too: an http destination is +// routinely an incoming-webhook endpoint whose path segments +// are the credential, so none of them is shown. +func TestNewTargetViews_HTTPMasksDestinationURL(t *testing.T) { + t.Parallel() + + view := viewFor(t, database.Target{ + Type: database.TargetTypeHTTP, + Config: `{"url":"` + slackWebhookURL + `"}`, + }) + + fields := fieldMap(view.Config) + + assert.Equal( + t, + slackMaskedURL, + fields["Destination URL"], + ) + + for _, v := range fields { + assert.NotContains(t, v, slackSecretPath) + assert.NotContains(t, v, "T00000000") + assert.NotContains(t, v, "B00000000") + assert.NotContains(t, v, "XXXXXXXXXXXXXXXXXXXXXXXX") + } +} + +func TestNewTargetViews_Database(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + config string + want string + }{ + "empty config": {config: "", want: viewExpiryNever}, + "empty expiry": {config: `{}`, want: viewExpiryNever}, + "explicit": { + config: `{"expiry":"720h"}`, + want: "720h", + }, + "never literal": { + config: `{"expiry":"` + viewExpiryNever + `"}`, + want: viewExpiryNever, + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + view := viewFor(t, database.Target{ + Type: database.TargetTypeDatabase, + Config: tc.config, + }) + + assert.Equal( + t, + map[string]string{"Archive Expiry": tc.want}, + fieldMap(view.Config), + ) + }) + } +} + +func TestNewTargetViews_Log(t *testing.T) { + t.Parallel() + + view := viewFor(t, database.Target{ + Type: database.TargetTypeLog, + Config: "", + }) + + assert.Empty(t, view.Config) +} + +// TestNewTargetViews_Unpresentable proves that no config the +// view cannot present falls back to the stored blob. +func TestNewTargetViews_Unpresentable(t *testing.T) { + t.Parallel() + + const blob = `{"webhookUrl":"https://hooks.slack.com` + + slackSecretPath + `"` + + tests := map[string]database.Target{ + "unknown target type": { + Type: database.TargetType("carrier-pigeon"), + Config: blob, + }, + "unparseable json": { + Type: database.TargetTypeSlack, + Config: blob, + }, + "empty slack config": { + Type: database.TargetTypeSlack, + }, + "slack config without url": { + Type: database.TargetTypeSlack, + Config: `{}`, + }, + "unparseable http json": { + Type: database.TargetTypeHTTP, + Config: `{"url":`, + }, + "unparseable archive json": { + Type: database.TargetTypeDatabase, + Config: `{"expiry":`, + }, + "invalid archive expiry": { + Type: database.TargetTypeDatabase, + Config: `{"expiry":"a fortnight"}`, + }, + } + + for name, target := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + + view := viewFor(t, target) + + assert.Equal( + t, + map[string]string{ + "Configuration": viewUnavailable, + }, + fieldMap(view.Config), + ) + }) + } +} diff --git a/internal/delivery/target_database.go b/internal/delivery/target_database.go index 0443aaa..d319551 100644 --- a/internal/delivery/target_database.go +++ b/internal/delivery/target_database.go @@ -5,6 +5,7 @@ import ( "fmt" "path/filepath" "sync" + "time" "gorm.io/gorm" "sneak.berlin/go/webhooker/internal/database" @@ -41,7 +42,14 @@ func (t *databaseTarget) Deliver( _ *Task, _ Scheduler, ) { + start := time.Now() + err := t.archive(d) + + elapsed := time.Since(start) + + t.eng.observeAttempt(d.Target.Type, elapsed) + if err != nil { t.eng.log.Error( "failed to archive event to database target", @@ -50,24 +58,37 @@ func (t *databaseTarget) Deliver( "error", err, ) - t.eng.recordResult( + recErr := t.eng.recordResult( webhookDB, d, 1, false, 0, "", - err.Error(), 0, + err.Error(), elapsed.Milliseconds(), ) + if recErr != nil { + t.eng.bookkeepingFailed(d, recErr) - t.eng.updateDeliveryStatus( - webhookDB, d, database.DeliveryStatusFailed, + return + } + + t.eng.settleStatus( + webhookDB, d, d.Target.Type, + database.DeliveryStatusFailed, ) return } - t.eng.recordResult( - webhookDB, d, 1, true, 0, "", "", 0, + recErr := t.eng.recordResult( + webhookDB, d, 1, true, 0, "", "", + elapsed.Milliseconds(), ) + if recErr != nil { + t.eng.bookkeepingFailed(d, recErr) - t.eng.updateDeliveryStatus( - webhookDB, d, database.DeliveryStatusDelivered, + return + } + + t.eng.settleStatus( + webhookDB, d, d.Target.Type, + database.DeliveryStatusDelivered, ) } @@ -111,15 +132,11 @@ func (t *databaseTarget) archive(d *database.Delivery) error { func (t *databaseTarget) writerFor( webhookID string, ) (*archiveWriter, error) { - if t.eng.dbManager == nil { - return nil, errArchiveNoDataDir + path, err := t.archivePath(webhookID) + if err != nil { + return nil, err } - dir := filepath.Dir(t.eng.dbManager.DBPath(webhookID)) - path := filepath.Join( - dir, fmt.Sprintf("archive-%s.db", webhookID), - ) - t.mu.Lock() defer t.mu.Unlock() @@ -133,5 +150,166 @@ func (t *databaseTarget) writerFor( t.writers[webhookID] = w } + // A delivery claims the entry: even if the idle sweep created + // it moments ago, it now belongs to the registry proper and + // the sweep must leave it in place when it finishes. + w.sweepOwned = false + return w, nil } + +// sweepWriterFor returns the archive writer the idle sweep should +// prune a webhook through, together with whether the sweep itself +// created the registry entry. +// +// The sweep must route its prune through the registered writer so +// the writer's mutex orders it against concurrent writes, but it +// must never leave a registry entry behind: a sweep that ran +// concurrently with the webhook's deletion would otherwise +// re-create an entry that nothing will ever evict again, which is +// exactly the leak eviction exists to prevent. An entry the sweep +// creates is therefore marked sweep-owned and handed back to +// releaseSweepWriter when the sweep is done. +func (t *databaseTarget) sweepWriterFor( + webhookID string, +) (*archiveWriter, bool, error) { + path, err := t.archivePath(webhookID) + if err != nil { + return nil, false, err + } + + t.mu.Lock() + defer t.mu.Unlock() + + if t.writers == nil { + t.writers = make(map[string]*archiveWriter) + } + + w, ok := t.writers[webhookID] + if ok { + return w, false, nil + } + + w = newArchiveWriter(path, t.eng.log) + w.sweepOwned = true + t.writers[webhookID] = w + + return w, true, nil +} + +// releaseSweepWriter drops a registry entry that the idle sweep +// created, so a sweep leaves the registry exactly as it found it. +// +// The entry is removed only if it is still the very writer the +// sweep installed and no delivery has claimed it in the meantime +// (writerFor clears sweepOwned when it hands a writer to the +// write path). Both conditions are evaluated under the registry +// lock, so an eviction that raced the sweep — which removes the +// entry outright — simply finds nothing left to do here, and a +// delivery that adopted the writer keeps a registered, evictable +// one. +func (t *databaseTarget) releaseSweepWriter( + webhookID string, w *archiveWriter, +) { + t.mu.Lock() + defer t.mu.Unlock() + + cur, ok := t.writers[webhookID] + if !ok || cur != w || !cur.sweepOwned { + return + } + + delete(t.writers, webhookID) +} + +// archivePath returns the archive file path for a webhook: it +// lives beside the per-webhook event database in the data +// directory. It does not touch the filesystem. +func (t *databaseTarget) archivePath( + webhookID string, +) (string, error) { + if t.eng.dbManager == nil { + return "", errArchiveNoDataDir + } + + dir := filepath.Dir(t.eng.dbManager.DBPath(webhookID)) + + return filepath.Join( + dir, fmt.Sprintf("archive-%s.db", webhookID), + ), nil +} + +// evict drops a webhook's archive writer from the registry and +// closes its handle, so a deleted webhook does not leave a +// writer (and an open archive handle within its debounce +// window) alive for the process lifetime. +// +// The map entry is removed under the registry lock, which is +// then released before the handle is closed under the writer's +// own lock: that ordering keeps the registry available to other +// webhooks while an in-flight write on this one drains, and +// closing under the writer's lock means eviction can never race +// a write. +// +// Eviction is idempotent and silent for a webhook with no +// writer, which is the common case: a webhook with no database +// target never creates one. It never deletes the archive file. +func (t *databaseTarget) evict(webhookID string) { + t.mu.Lock() + + w, ok := t.writers[webhookID] + if ok { + delete(t.writers, webhookID) + } + + t.mu.Unlock() + + if !ok { + return + } + + w.evict() + + t.eng.log.Info( + "evicted archive writer", + "webhook_id", webhookID, + "path", w.path, + ) +} + +// sweepWebhook prunes one webhook's archive of rows older than +// expiry, without requiring a write. It returns nil (nothing to +// do) when the archive file does not exist, so a sweep never +// creates an archive for a webhook that has a database target +// but has never received an event. +// +// It also never leaves a registry entry behind: an entry it had +// to create to reach the writer's mutex is released again once +// the prune is done, so a sweep racing a webhook deletion cannot +// resurrect the writer the eviction just dropped. +func (t *databaseTarget) sweepWebhook( + webhookID string, expiry time.Duration, +) error { + path, err := t.archivePath(webhookID) + if err != nil { + return err + } + + // Check before taking a writer at all: a webhook whose + // archive has never been created gets no writer, no handle, + // and no file. + if !fileExists(path) { + return nil + } + + w, created, err := t.sweepWriterFor(webhookID) + if err != nil { + return err + } + + if created { + defer t.releaseSweepWriter(webhookID, w) + } + + return w.sweepExpired(expiry) +} diff --git a/internal/delivery/target_database_archive.go b/internal/delivery/target_database_archive.go index 547a0af..0937854 100644 --- a/internal/delivery/target_database_archive.go +++ b/internal/delivery/target_database_archive.go @@ -1,7 +1,6 @@ package delivery import ( - "database/sql" "encoding/json" "errors" "fmt" @@ -12,6 +11,8 @@ import ( "gorm.io/driver/sqlite" "gorm.io/gorm" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/gormlog" ) // archiveExpiryNever is the expiry sentinel (and default) that @@ -24,6 +25,20 @@ const archiveExpiryNever = "never" // offline archiving, but never more than once per this window. const archiveReopenDebounce = time.Second +const ( + // archiveModeCreate is the SQLite URI mode used by the write + // path: open the archive file, creating it if missing, so a + // first write (or a write after the operator moved the file + // away) recreates it. + archiveModeCreate = database.SQLiteModeCreate + + // archiveModeExisting is the SQLite URI mode used by the idle + // sweep: open read-write but never create. A sweep must never + // conjure an empty archive file for a webhook that has a + // database target but has never received an event. + archiveModeExisting = database.SQLiteModeExisting +) + var ( // errArchiveMissingWebhookID is returned when an event to // archive has no webhook id to key its archive file on. @@ -44,6 +59,15 @@ var ( errArchiveExpiryNotPositive = errors.New( "expiry must be a positive duration or \"never\"", ) + + // errArchiveWriterEvicted is returned when a writer that has + // been evicted (its webhook was deleted, or its last database + // target was removed) is used again. An evicted writer is no + // longer in the registry, so reopening its file would leak a + // handle nothing owns. + errArchiveWriterEvicted = errors.New( + "archive writer has been evicted", + ) ) // databaseTargetConfig is the optional per-target JSON config @@ -161,6 +185,25 @@ type archiveWriter struct { db *gorm.DB lastReopen time.Time reopens int + + // evicted marks a writer that has been removed from the + // per-webhook registry. Its handle is closed and it must + // never open the file again: nothing holds it any more, so a + // reopen would leak the handle for the process lifetime. + evicted bool + + // sweepOwned marks a registry entry that the idle sweep + // created because no writer was cached for the webhook. The + // sweep removes such an entry again when it is done, so a + // sweep can never leave — or resurrect — a registry entry + // for a webhook that has been deleted. A delivery that adopts + // the writer clears the flag, handing the entry to the + // registry proper. + // + // Unlike every other field here it is guarded by + // databaseTarget.mu, not by this writer's mu: it describes the + // registry entry rather than the file. + sweepOwned bool } // newArchiveWriter builds an archiveWriter for a file path with @@ -185,6 +228,12 @@ func (w *archiveWriter) write( w.mu.Lock() defer w.mu.Unlock() + if w.evicted { + return fmt.Errorf( + "%w: %s", errArchiveWriterEvicted, w.path, + ) + } + if w.db == nil || !fileExists(w.path) { err := w.reopen(expiry) if err != nil { @@ -212,9 +261,23 @@ func (w *archiveWriter) write( // its schema, records the reopen time, and prunes expired rows // when expiry is positive. func (w *archiveWriter) open(expiry time.Duration) error { - dbURL := fmt.Sprintf("file:%s?mode=rwc", w.path) + return w.openMode(archiveModeCreate, expiry) +} - sqlDB, err := sql.Open("sqlite", dbURL) +// openMode opens the archive file with the given SQLite URI +// mode, migrates its schema, records the reopen time, and +// prunes expired rows when expiry is positive. The write path +// passes archiveModeCreate so a missing file is recreated; the +// idle sweep passes archiveModeExisting so a missing file is an +// error rather than a newly conjured empty archive. +func (w *archiveWriter) openMode( + mode string, expiry time.Duration, +) error { + // Opened through database.OpenSQLite so an archive file carries + // the same WAL journaling, busy timeout, immediate-transaction + // locking, and pool bounds as every other database file. See + // internal/database/sqlite_open.go. + sqlDB, err := database.OpenSQLite(w.path, mode) if err != nil { return fmt.Errorf( "opening archive database %s: %w", w.path, err, @@ -222,7 +285,11 @@ func (w *archiveWriter) open(expiry time.Duration) error { } gdb, err := gorm.Open( - sqlite.Dialector{Conn: sqlDB}, &gorm.Config{}, + sqlite.Dialector{Conn: sqlDB}, &gorm.Config{ + // Never leave this at GORM's default. See + // internal/gormlog. + Logger: gormlog.New(w.log), + }, ) if err != nil { _ = sqlDB.Close() @@ -275,11 +342,70 @@ func (w *archiveWriter) close() { w.db = nil } +// sweepExpired prunes an archive that may have gone idle, with +// no write to trigger the usual on-reopen prune. It takes the +// writer's own mutex for the whole operation, so a sweep is +// ordered against concurrent writes rather than reaching around +// them to the file. +// +// It never creates the archive file: a missing file is skipped, +// and the reopen uses archiveModeExisting so SQLite itself +// refuses to create one if the file disappears between the +// check and the open. +// +// The archive is left CLOSED afterwards. An idle archive holding +// no handle is what keeps the operator's move-the-file-away +// workflow working; the next write reopens (and recreates) the +// file as it always has. +func (w *archiveWriter) sweepExpired(expiry time.Duration) error { + w.mu.Lock() + defer w.mu.Unlock() + + if w.evicted { + return fmt.Errorf( + "%w: %s", errArchiveWriterEvicted, w.path, + ) + } + + if !fileExists(w.path) { + return nil + } + + // Drop any live handle first so the prune runs against a + // freshly opened file, matching the write path's semantics. + w.close() + + err := w.openMode(archiveModeExisting, expiry) + if err != nil { + return err + } + + w.close() + + return nil +} + +// evict closes the writer's handle and marks it unusable. It is +// called when the writer leaves the registry, either because the +// webhook was deleted or because its last database target was +// removed. The archive FILE is deliberately left on disk: it is +// long-term storage an operator may still want. +func (w *archiveWriter) evict() { + w.mu.Lock() + defer w.mu.Unlock() + + w.evicted = true + + w.close() +} + // prune deletes archived rows older than expiry, measured from -// each row's archived time. It runs on every (re)open, and -// because the file is reopened after writes this keeps the -// archive swept without a separate background sweeper. Failures -// are logged, not fatal: a prune error must not stop archiving. +// each row's archived time. It runs on every (re)open, so a +// steadily written archive is swept by its own write traffic. An +// archive that goes idle receives no further reopens, which is +// why ArchiveSweeper exists to drive sweepExpired on a timer. +// Failures are logged, not fatal: a prune error must not stop +// archiving. func (w *archiveWriter) prune(expiry time.Duration) { cutoff := time.Now().Add(-expiry) diff --git a/internal/delivery/target_database_archive_gormlog_test.go b/internal/delivery/target_database_archive_gormlog_test.go new file mode 100644 index 0000000..1644f8c --- /dev/null +++ b/internal/delivery/target_database_archive_gormlog_test.go @@ -0,0 +1,155 @@ +package delivery_test + +import ( + "bytes" + "log" + "log/slog" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + gormlogger "gorm.io/gorm/logger" + "sneak.berlin/go/webhooker/internal/delivery" + "sneak.berlin/go/webhooker/internal/middleware" +) + +// archiveGORMTailMarker sits at the far end of the value this file +// drives into an archive lookup. Its presence in a log line means the +// whole value reached the log, so nothing truncated it. +const archiveGORMTailMarker = "ENDOFCLIENTVALUE" + +// archiveGORMFillBytes is how much text the lookup carries. It is far +// past every budget in play. +const archiveGORMFillBytes = 8 << 10 + +// gormDefaultBuf collects what GORM's package-level default logger +// writes, if anything reaches it. +type gormDefaultBuf struct { + mu sync.Mutex + b bytes.Buffer +} + +func (g *gormDefaultBuf) Write(p []byte) (int, error) { + g.mu.Lock() + defer g.mu.Unlock() + + return g.b.Write(p) +} + +func (g *gormDefaultBuf) String() string { + g.mu.Lock() + defer g.mu.Unlock() + + return g.b.String() +} + +// captureArchiveGORMDefault replaces GORM's package-level default +// logger with one configured exactly as GORM configures its own, +// writing to a buffer. +// +// This duplicates the detector in internal/handlers rather than +// sharing it: a test helper cannot cross a package's test boundary +// without exporting production code to carry it, and a logging +// detector is not worth a production symbol. What it detects is the +// third gorm.Open in this service, at +// internal/delivery/target_database_archive.go — the archive writer, +// whose type is unexported, so nothing outside this package can drive +// it. +func captureArchiveGORMDefault(t *testing.T) *gormDefaultBuf { + t.Helper() + + buf := &gormDefaultBuf{} + orig := gormlogger.Default + + gormlogger.Default = gormlogger.New( + log.New(buf, "", log.LstdFlags), + gormlogger.Config{ + SlowThreshold: 200 * time.Millisecond, + LogLevel: gormlogger.Warn, + IgnoreRecordNotFoundError: false, + Colorful: false, + }, + ) + + t.Cleanup(func() { gormlogger.Default = orig }) + + return buf +} + +// TestArchiveWriter_NeverUsesGORMsDefaultLogger pins the archive +// writer's gorm.Open to the adapter. +// +// Restore a bare &gorm.Config{} at +// internal/delivery/target_database_archive.go and this fails: the +// default logger prints the fully interpolated SELECT on every +// ErrRecordNotFound, so the client-chosen event id below arrives whole +// and unbounded on stdout, answering to no level the operator set. +// +// Not parallel: gormlogger.Default is process-global. Go runs every +// non-parallel top-level test to completion before it resumes the +// parallel ones. +// +//nolint:paralleltest // Deliberately sequential; see above. +func TestArchiveWriter_NeverUsesGORMsDefaultLogger(t *testing.T) { + var captured bytes.Buffer + + gormDefault := captureArchiveGORMDefault(t) + + w := delivery.NewExportArchiveWriter( + filepath.Join(t.TempDir(), "archive.db"), + slog.New(slog.NewTextHandler( + &captured, &slog.HandlerOptions{Level: slog.LevelDebug}, + )), + 0, + ) + + require.NoError(t, w.Open(0)) + + t.Cleanup(w.Evict) + + // A lookup that misses, carrying a value the size of an inbound + // event id. Under the default logger this is the line that gets + // interpolated and printed. + value := strings.Repeat("\x01", archiveGORMFillBytes) + + archiveGORMTailMarker + + var row delivery.ExportArchivedEvent + + err := w.DB().Where("event_id = ?", value).First(&row).Error + require.ErrorIs(t, err, gorm.ErrRecordNotFound) + + got := gormDefault.String() + assert.Empty( + t, got, + "GORM's default logger wrote %d bytes, so the archive "+ + "writer's gorm.Open is back on a bare &gorm.Config{}; "+ + "the first of them: %s", + len(got), got[:min(len(got), 300)], + ) + + // The adapter drops a miss, so this should be silent too — and + // whatever it does write stays inside the stated ceiling. + out := captured.String() + + assert.NotContains( + t, out, archiveGORMTailMarker, + "the far end of the client-chosen value reached the log", + ) + + for line := range strings.SplitSeq(strings.TrimRight(out, "\n"), "\n") { + if line == "" { + continue + } + + assert.LessOrEqual( + t, len(line), middleware.MaxAccessLogLineBytes, + "log line exceeded its bound: %s", + line[:min(len(line), 300)], + ) + } +} diff --git a/internal/delivery/target_database_evict_test.go b/internal/delivery/target_database_evict_test.go new file mode 100644 index 0000000..14e7945 --- /dev/null +++ b/internal/delivery/target_database_evict_test.go @@ -0,0 +1,363 @@ +package delivery_test + +import ( + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" +) + +// evictTestEngine builds an engine backed by a temporary data +// directory and returns it along with that directory. +func evictTestEngine(t *testing.T) (*delivery.Engine, string) { + t.Helper() + + dataDir := t.TempDir() + + eng := delivery.NewTestEngineWithDB( + nil, + database.NewTestWebhookDBManager(dataDir), + archiveTestLogger(), + &http.Client{Timeout: 5 * time.Second}, + 1, + ) + + return eng, dataDir +} + +// TestEvictWebhook_ClosesAndRemovesWriter proves that evicting +// a webhook drops its archive writer from the registry and +// closes the open archive handle, rather than leaving both +// alive for the process lifetime. +func TestEvictWebhook_ClosesAndRemovesWriter(t *testing.T) { + t.Parallel() + + eng, dataDir := evictTestEngine(t) + + webhookDB := testWebhookDB(t) + event := seedEvent(t, webhookDB, `{"archived":true}`) + d := seedDatabaseTargetDelivery(t, webhookDB, event, "") + + eng.ExportDeliverDatabase(webhookDB, d) + + webhookID := event.WebhookID + + require.True( + t, eng.ExportHasArchiveWriter(webhookID), + "a delivery should have cached an archive writer", + ) + require.True( + t, eng.ExportArchiveHandleOpen(webhookID), + "the writer should hold an open handle after a write", + ) + + eng.EvictWebhook(webhookID) + + assert.False( + t, eng.ExportHasArchiveWriter(webhookID), + "eviction should remove the registry entry", + ) + assert.False( + t, eng.ExportArchiveHandleOpen(webhookID), + "eviction should close the archive handle", + ) + + archivePath := filepath.Join( + dataDir, fmt.Sprintf("archive-%s.db", webhookID), + ) + assert.FileExists( + t, archivePath, + "eviction must not delete the archive file", + ) +} + +// TestEvictWebhook_UnknownWebhookIsNoOp proves eviction is safe +// for the common case of a webhook that never had a database +// target, and that repeating it does not panic. +func TestEvictWebhook_UnknownWebhookIsNoOp(t *testing.T) { + t.Parallel() + + eng, _ := evictTestEngine(t) + + assert.NotPanics(t, func() { + eng.EvictWebhook("no-such-webhook") + eng.EvictWebhook("no-such-webhook") + }) + + assert.False( + t, eng.ExportHasArchiveWriter("no-such-webhook"), + "eviction must not create a writer", + ) +} + +// evictTestRow builds an archive row for the eviction tests. +func evictTestRow(eventID string) delivery.ExportArchivedEvent { + return delivery.ExportArchivedEvent{ + EventID: eventID, + WebhookID: "wh-evict", + Method: http.MethodPost, + Body: `{"seeded":true}`, + } +} + +// TestEvictedWriter_WriteDoesNotReopenFile is the direct test of +// the evicted guard on the write path. A writer that has left +// the registry is held by nobody, so a handle it opened could +// never be closed again: it must refuse the write outright +// rather than recreate the archive behind the registry's back. +// +// The archive file is removed before the eviction, so an +// unguarded write is unmistakable — it recreates the file. +func TestEvictedWriter_WriteDoesNotReopenFile(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "archive-evicted.db") + + w := delivery.NewExportArchiveWriter( + path, archiveTestLogger(), 0, + ) + + require.NoError(t, w.Write(evictTestRow("ev-1"), 0)) + require.FileExists(t, path) + + // The operator moves the archive away for offline retention, + // which the write path would ordinarily undo on the next + // write by recreating the file. + require.NoError(t, os.Remove(path)) + + w.Evict() + + err := w.Write(evictTestRow("ev-2"), 0) + + require.ErrorIs( + t, err, delivery.ErrExportArchiveWriterEvicted, + "an evicted writer must refuse writes", + ) + assert.NoFileExists( + t, path, + "an evicted writer must not reopen (or recreate) the "+ + "archive file", + ) + assert.False( + t, w.HandleOpen(), + "an evicted writer must hold no handle", + ) +} + +// TestEvictedWriter_SweepDoesNotReopenFile is the same test for +// the sweep path: an idle sweep that reaches a writer already +// evicted underneath it must return the sentinel rather than +// reopen a file nothing owns. +func TestEvictedWriter_SweepDoesNotReopenFile(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "archive-evicted.db") + + w := delivery.NewExportArchiveWriter( + path, archiveTestLogger(), 0, + ) + + require.NoError(t, w.Write(evictTestRow("ev-1"), 0)) + require.FileExists(t, path) + + w.Evict() + + err := w.SweepExpired(time.Hour) + + require.ErrorIs( + t, err, delivery.ErrExportArchiveWriterEvicted, + "an evicted writer must refuse an idle sweep", + ) + assert.False( + t, w.HandleOpen(), + "a refused sweep must not leave a handle open", + ) +} + +// racingWrites drives a pack of goroutines writing to one +// archive writer until each is refused, so an eviction on the +// test goroutine has to take the writer's mutex away from writes +// that are already contending for it. +type racingWrites struct { + wg sync.WaitGroup + mu sync.Mutex + sawEvicted bool + otherErr error + started chan struct{} +} + +// racingWriteGoroutines is how many goroutines contend for the +// writer's mutex while the eviction lands. +const racingWriteGoroutines = 4 + +// startRacingWrites launches the writing goroutines. Each writes +// in a loop and stops at its first error, recording whether that +// error was the eviction sentinel. The deadline is a backstop +// against a hang, not a timing assumption: the first write after +// the eviction is refused. +func startRacingWrites( + w *delivery.ExportArchiveWriter, +) *racingWrites { + r := &racingWrites{ + started: make(chan struct{}, racingWriteGoroutines), + } + + deadline := time.Now().Add(10 * time.Second) + + r.wg.Add(racingWriteGoroutines) + + for i := range racingWriteGoroutines { + go func() { + defer r.wg.Done() + + first := true + + for time.Now().Before(deadline) { + err := w.Write( + evictTestRow(fmt.Sprintf("ev-%d", i)), 0, + ) + + if first { + r.started <- struct{}{} + + first = false + } + + if err == nil { + continue + } + + r.record(err) + + return + } + }() + } + + return r +} + +// record classifies the error that stopped one goroutine. +func (r *racingWrites) record(err error) { + r.mu.Lock() + defer r.mu.Unlock() + + if errors.Is(err, delivery.ErrExportArchiveWriterEvicted) { + r.sawEvicted = true + + return + } + + r.otherErr = err +} + +// awaitFirstWrite blocks until at least one write has run, so +// the eviction that follows is a genuine race. +func (r *racingWrites) awaitFirstWrite() { + <-r.started +} + +// wait joins the goroutines and reports whether any write was +// refused with the eviction sentinel, plus any unexpected error. +func (r *racingWrites) wait() (bool, error) { + r.wg.Wait() + + r.mu.Lock() + defer r.mu.Unlock() + + return r.sawEvicted, r.otherErr +} + +// TestEvictWebhook_RacingWriteDoesNotReopenHandle exercises the +// interleaving the evicted flag exists for: writes already +// contending for the writer's mutex when the eviction takes it. +// The write that wins the mutex after the eviction must abandon +// its work rather than reopen the archive, leaving the writer +// permanently handle-free. Run under -race. +func TestEvictWebhook_RacingWriteDoesNotReopenHandle( + t *testing.T, +) { + t.Parallel() + + eng, _ := evictTestEngine(t) + + webhookDB := testWebhookDB(t) + event := seedEvent(t, webhookDB, `{"archived":true}`) + d := seedDatabaseTargetDelivery(t, webhookDB, event, "") + + // Prime the registry so the test can hold the very writer the + // eviction is about to detach. + eng.ExportDeliverDatabase(webhookDB, d) + + w := eng.ExportArchiveWriterFor(event.WebhookID) + require.NotNil(t, w) + require.True(t, w.HandleOpen()) + + race := startRacingWrites(w) + + // Evict only once writes are genuinely in flight, so the + // eviction has to contend for the writer's mutex. + race.awaitFirstWrite() + + eng.EvictWebhook(event.WebhookID) + + sawEvicted, otherErr := race.wait() + + require.NoError(t, otherErr) + assert.True( + t, sawEvicted, + "a write after eviction must be refused", + ) + assert.False( + t, w.HandleOpen(), + "no write may reopen the archive once the writer has "+ + "been evicted", + ) + assert.False( + t, eng.ExportHasArchiveWriter(event.WebhookID), + "the registry entry must stay gone", + ) +} + +// TestEvictWebhook_LaterDeliveryRecreatesWriter proves eviction +// does not break archiving for a webhook that is still alive: a +// subsequent delivery gets a brand new writer from the registry. +// It says nothing about the evicted writer itself — that is what +// TestEvictedWriter_WriteDoesNotReopenFile covers. +func TestEvictWebhook_LaterDeliveryRecreatesWriter(t *testing.T) { + t.Parallel() + + eng, _ := evictTestEngine(t) + + webhookDB := testWebhookDB(t) + event := seedEvent(t, webhookDB, `{"archived":true}`) + d := seedDatabaseTargetDelivery(t, webhookDB, event, "") + + eng.ExportDeliverDatabase(webhookDB, d) + require.True( + t, eng.ExportHasArchiveWriter(event.WebhookID), + ) + + eng.EvictWebhook(event.WebhookID) + + // A fresh delivery for the same webhook gets a brand new + // writer from the registry, so archiving keeps working. + second := seedDatabaseTargetDelivery( + t, webhookDB, event, "", + ) + eng.ExportDeliverDatabase(webhookDB, second) + + assert.True( + t, eng.ExportHasArchiveWriter(event.WebhookID), + "a later delivery should recreate the writer", + ) +} diff --git a/internal/delivery/target_database_test.go b/internal/delivery/target_database_test.go index 3ee38ac..dc2e65c 100644 --- a/internal/delivery/target_database_test.go +++ b/internal/delivery/target_database_test.go @@ -50,6 +50,13 @@ func openArchiveDBForRead( return gdb } +// archiveFileSuffixes returns the archive file itself and the +// SQLite sidecars that accompany an open database. A test that +// asserts no archive was created has to check all of them. +func archiveFileSuffixes() []string { + return []string{"", "-wal", "-shm"} +} + // removeArchiveFiles simulates an operator moving the archive // away by deleting the SQLite file and its sidecar files. func removeArchiveFiles(t *testing.T, path string) { diff --git a/internal/delivery/target_headers.go b/internal/delivery/target_headers.go new file mode 100644 index 0000000..20e3c9b --- /dev/null +++ b/internal/delivery/target_headers.go @@ -0,0 +1,263 @@ +package delivery + +import ( + "errors" + "fmt" + "net/http" + "slices" + "strconv" + "strings" +) + +// MaxTargetTimeoutSeconds bounds a per-target request timeout. +// A delivery attempt holds a worker for its whole duration, so an +// unbounded timeout lets one misconfigured target stall the queue +// indefinitely. Five minutes is far beyond any healthy webhook +// receiver and still finite. +const MaxTargetTimeoutSeconds = 300 + +// Errors returned when a target's header or timeout form input +// cannot be turned into a configuration. +// +// None of these ever quotes a header VALUE. A target header value +// is routinely an authorization token, and these messages are shown +// to the user in an error page body. +var ( + errHeaderLineMalformed = errors.New( + `each header line must be "Name: value"`, + ) + errHeaderNameInvalid = errors.New( + "header name must be a valid HTTP token", + ) + errHeaderValueInvalid = errors.New( + "header value must not contain control characters", + ) + errHeaderDuplicate = errors.New( + "header given more than once", + ) + errHeaderReserved = errors.New( + "header is set by the delivery engine and cannot be " + + "overridden", + ) + errTimeoutInvalid = errors.New( + "timeout must be a whole number of seconds", + ) + errTimeoutOutOfRange = errors.New( + "timeout is out of range", + ) +) + +// isReservedTargetHeader reports whether name (canonicalised) is a +// header a target configuration may not set, because the delivery +// path or net/http itself writes it regardless. +// +// These are rejected rather than accepted-and-ignored. Storing a +// header that provably never reaches the wire tells the operator +// their configuration took effect when it did not, which is the +// same failure mode as silently substituting a default for an +// invalid value. +func isReservedTargetHeader(name string) bool { + switch name { + case "Host", "Content-Length", "Transfer-Encoding", "Connection": + return true + case "User-Agent": + // applyRequestHeaders sets the User-Agent after it applies + // the configured headers, so a configured one would always + // be overwritten. + return true + case "Trailer": + // net/http strips Trailer from the request it writes + // (reqWriteExcludeHeader), so a configured one is accepted + // and stored and then provably never reaches the wire. + return true + default: + return false + } +} + +// ParseTargetHeaders turns the target form's headers field — one +// "Name: value" pair per line, blank lines ignored — into the map +// stored in HTTPTargetConfig.Headers. Names are canonicalised, so a +// name repeated in a different case is still a duplicate rather than +// one pair silently overwriting the other. +// +// An input with no pairs yields an empty map, which omitempty drops +// from the stored config: a target configured with no headers keeps +// the same config JSON it had before this field existed. +func ParseTargetHeaders(raw string) (map[string]string, error) { + headers := make(map[string]string) + + for i, line := range strings.Split(raw, "\n") { + lineNum := i + 1 + + line = strings.TrimSpace(line) + if line == "" { + continue + } + + name, value, err := parseHeaderLine(line) + if err != nil { + return nil, fmt.Errorf("line %d: %w", lineNum, err) + } + + if _, dup := headers[name]; dup { + return nil, fmt.Errorf( + "line %d: %w: %q", lineNum, + errHeaderDuplicate, name, + ) + } + + headers[name] = value + } + + return headers, nil +} + +// parseHeaderLine splits and validates one "Name: value" line, +// returning the canonicalised name and the trimmed value. +func parseHeaderLine(line string) (string, string, error) { + rawName, value, found := strings.Cut(line, ":") + if !found { + return "", "", errHeaderLineMalformed + } + + rawName = strings.TrimSpace(rawName) + if !validHeaderName(rawName) { + // Quotes nothing. The text before the first colon is only + // a name if it parses as one; when it does not, it is as + // likely to be a pasted value whose own colon split the + // line, and half of a token would be echoed into the 400. + return "", "", errHeaderNameInvalid + } + + name := http.CanonicalHeaderKey(rawName) + if isReservedTargetHeader(name) { + return "", "", fmt.Errorf( + "%w: %q", errHeaderReserved, name, + ) + } + + value = strings.TrimSpace(value) + if !validHeaderValue(value) { + return "", "", fmt.Errorf( + "%w: %q", errHeaderValueInvalid, name, + ) + } + + return name, value, nil +} + +// validHeaderName reports whether name is a non-empty RFC 9110 +// field name. Rejecting anything else here is what keeps a value +// containing CR or LF from being smuggled in as part of a name and +// injecting a second header into the outbound request. +func validHeaderName(name string) bool { + if name == "" { + return false + } + + for i := range len(name) { + if !isTokenByte(name[i]) { + return false + } + } + + return true +} + +// isTokenByte reports whether c is a "tchar" per RFC 9110 5.6.2. +func isTokenByte(c byte) bool { + switch { + case c >= 'a' && c <= 'z', + c >= 'A' && c <= 'Z', + c >= '0' && c <= '9': + return true + } + + return strings.IndexByte("!#$%&'*+-.^_`|~", c) >= 0 +} + +// validHeaderValue reports whether value is a legal field value: +// no control characters, which is the other half of the header +// injection guard. An empty value is legal. +func validHeaderValue(value string) bool { + for i := range len(value) { + c := value[i] + if c < 0x20 || c == 0x7f { + return false + } + } + + return true +} + +// FormatTargetHeaders renders a stored header map back into the +// form's textarea representation, one "Name: value" per line. +// +// Names are sorted so that loading the edit form twice without +// saving produces identical text; Go map iteration order would +// otherwise reshuffle the field on every render. +func FormatTargetHeaders(headers map[string]string) string { + if len(headers) == 0 { + return "" + } + + names := make([]string, 0, len(headers)) + for name := range headers { + names = append(names, name) + } + + slices.Sort(names) + + var b strings.Builder + + for _, name := range names { + b.WriteString(name) + b.WriteString(": ") + b.WriteString(headers[name]) + b.WriteString("\n") + } + + return b.String() +} + +// ParseTargetTimeout interprets the target form's timeout field as +// a whole number of seconds. An empty field means "unset" and yields +// 0, which omitempty drops from the stored config and which the +// delivery path reads as "use the shared client's timeout". +// +// Anything else that is not a whole number in range is an error, not +// a silently substituted default: a target whose timeout was typed +// wrong must say so at the form rather than deliver on a timeout its +// operator did not choose. +func ParseTargetTimeout(raw string) (int, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return 0, nil + } + + v, err := strconv.Atoi(raw) + if err != nil || v < 0 { + return 0, errTimeoutInvalid + } + + if v > MaxTargetTimeoutSeconds { + return 0, fmt.Errorf( + "%w: at most %d seconds", + errTimeoutOutOfRange, MaxTargetTimeoutSeconds, + ) + } + + return v, nil +} + +// FormatTargetTimeout renders a stored timeout for the form field. +// An unset timeout renders as an empty field rather than "0", so the +// placeholder can describe the default the target actually uses. +func FormatTargetTimeout(timeout int) string { + if timeout <= 0 { + return "" + } + + return strconv.Itoa(timeout) +} diff --git a/internal/delivery/target_headers_test.go b/internal/delivery/target_headers_test.go new file mode 100644 index 0000000..1b1c7ae --- /dev/null +++ b/internal/delivery/target_headers_test.go @@ -0,0 +1,292 @@ +package delivery_test + +import ( + "encoding/json" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" +) + +// Literals these tests repeat, named so that the header name and the +// keep-forever archive config each have one definition. +const ( + headerAuthorization = "Authorization" + bearerValue = "Bearer abc" + archiveConfigNever = "{\"expiry\":\"never\"}" +) + +func TestParseTargetHeaders_AcceptsPairs(t *testing.T) { + t.Parallel() + + got, err := delivery.ParseTargetHeaders( + " Authorization: Bearer abc \n\n" + + "x-tenant:acme\r\n" + + "X-Empty:\n", + ) + require.NoError(t, err) + + assert.Equal( + t, + map[string]string{ + headerAuthorization: bearerValue, + "X-Tenant": "acme", + "X-Empty": "", + }, + got, + ) +} + +// A configuration with no headers must stay indistinguishable from +// one written before the field existed, so omitempty drops the key. +func TestParseTargetHeaders_EmptyInputYieldsNoHeaders(t *testing.T) { + t.Parallel() + + got, err := delivery.ParseTargetHeaders("\n \n") + require.NoError(t, err) + assert.Empty(t, got) + + encoded, err := json.Marshal(delivery.HTTPTargetConfig{ + URL: "https://example.com/h", + Headers: got, + }) + require.NoError(t, err) + assert.JSONEq( + t, `{"url":"https://example.com/h"}`, string(encoded), + ) +} + +func TestParseTargetHeaders_Rejects(t *testing.T) { + t.Parallel() + + cases := map[string]string{ + "no colon": "Authorization Bearer abc", + "empty name": ": value", + "space in name": "X Bad: value", + "reserved host": "Host: evil.example", + "reserved ua": "User-Agent: curl/8", + "reserved length": "Content-Length: 0", + "duplicate any case": "X-A: 1\nx-a: 2", + } + + for name, input := range cases { + t.Run(name, func(t *testing.T) { + t.Parallel() + + _, err := delivery.ParseTargetHeaders(input) + require.Error(t, err) + }) + } +} + +// net/http strips Trailer from the request it writes, so accepting +// one would store a header that never reaches the target. +func TestParseTargetHeaders_RejectsTrailer(t *testing.T) { + t.Parallel() + + _, err := delivery.ParseTargetHeaders("Trailer: X-Checksum") + require.Error(t, err) + assert.Contains(t, err.Error(), "Trailer") +} + +// A header value is routinely a bearer token and these errors are +// rendered into a 400 body, so no message may quote one. +func TestParseTargetHeaders_ErrorsNeverQuoteAValue(t *testing.T) { + t.Parallel() + + const secret = "QQNEVERINAMESSAGEQQ" + + inputs := []string{ + // The value, after the colon, in a duplicate name. + "X-A: " + secret + "\nx-a: " + secret, + // The value after the colon of an unusable name. + "X Bad Name: " + secret, + // The line splits on the value's own colon, so the + // secret lands in the text an unusable-name error is + // tempted to quote as the name. + "X-Api-Key " + secret + ":x", + // The same, with nothing before the secret at all. + secret + " and more:x", + // A control character in the value. + "X-A: " + secret + "\x01", + } + + for _, input := range inputs { + _, err := delivery.ParseTargetHeaders(input) + require.Error(t, err, input) + assert.NotContains(t, err.Error(), secret, input) + } +} + +// Loading the edit form twice without saving must not reshuffle +// the textarea, which Go's map iteration order would otherwise do. +func TestFormatTargetHeaders_IsSorted(t *testing.T) { + t.Parallel() + + got := delivery.FormatTargetHeaders(map[string]string{ + "X-Zed": "z", + headerAuthorization: bearerValue, + "X-Alpha": "a", + }) + + assert.Equal( + t, + "Authorization: Bearer abc\nX-Alpha: a\nX-Zed: z\n", + got, + ) + assert.Empty(t, delivery.FormatTargetHeaders(nil)) +} + +func TestFormatTargetHeaders_RoundTripsThroughParse(t *testing.T) { + t.Parallel() + + want := map[string]string{ + headerAuthorization: bearerValue, + "X-Tenant": "acme", + } + + got, err := delivery.ParseTargetHeaders( + delivery.FormatTargetHeaders(want), + ) + require.NoError(t, err) + assert.Equal(t, want, got) +} + +func TestParseTargetTimeout(t *testing.T) { + t.Parallel() + + got, err := delivery.ParseTargetTimeout(" 30 ") + require.NoError(t, err) + assert.Equal(t, 30, got) + + got, err = delivery.ParseTargetTimeout("") + require.NoError(t, err) + assert.Zero(t, got) + + for _, bad := range []string{"soon", "-1", "1e3", "100000"} { + _, err = delivery.ParseTargetTimeout(bad) + require.Error(t, err, bad) + } +} + +func TestFormatTargetTimeout(t *testing.T) { + t.Parallel() + + assert.Equal(t, "30", delivery.FormatTargetTimeout(30)) + assert.Empty(t, delivery.FormatTargetTimeout(0)) + assert.Empty(t, delivery.FormatTargetTimeout(-1)) +} + +func TestNewTargetConfigForm(t *testing.T) { + t.Parallel() + + form, err := delivery.NewTargetConfigForm(&database.Target{ + Type: database.TargetTypeHTTP, + Config: `{"url":"https://example.com/h",` + + `"headers":{"Authorization":"Bearer abc"},` + + `"timeout":9}`, + }) + require.NoError(t, err) + assert.Equal(t, "https://example.com/h", form.URL) + assert.Equal(t, "Authorization: Bearer abc\n", form.Headers) + assert.Equal(t, "9", form.Timeout) + + form, err = delivery.NewTargetConfigForm(&database.Target{ + Type: database.TargetTypeSlack, + Config: `{"webhookUrl":"https://hooks.example/s"}`, + }) + require.NoError(t, err) + assert.Equal(t, "https://hooks.example/s", form.URL) + + form, err = delivery.NewTargetConfigForm(&database.Target{ + Type: database.TargetTypeDatabase, + Config: `{"expiry":"720h"}`, + }) + require.NoError(t, err) + assert.Equal(t, "720h", form.Expiry) + + form, err = delivery.NewTargetConfigForm(&database.Target{ + Type: database.TargetTypeLog, + }) + require.NoError(t, err) + assert.Empty(t, form.URL) +} + +// A keep-forever archive target must pre-fill as an empty field, so +// saving the form back unchanged stores the same empty config. +func TestNewTargetConfigForm_DatabaseNeverIsBlank(t *testing.T) { + t.Parallel() + + for _, cfg := range []string{"", `{}`, archiveConfigNever} { + form, err := delivery.NewTargetConfigForm( + &database.Target{ + Type: database.TargetTypeDatabase, + Config: cfg, + }, + ) + require.NoError(t, err, cfg) + assert.Empty(t, form.Expiry, cfg) + } +} + +// An unreadable stored config is an error rather than a blank form +// that looks like a target with no settings, so the caller can tell +// the operator that saving replaces the stored value. +func TestNewTargetConfigForm_UnreadableConfigErrors(t *testing.T) { + t.Parallel() + + cases := []*database.Target{ + {Type: database.TargetTypeHTTP, Config: "not json"}, + {Type: database.TargetTypeHTTP, Config: `{}`}, + {Type: database.TargetTypeSlack, Config: ""}, + { + Type: database.TargetTypeDatabase, + Config: `{"expiry":"soon"}`, + }, + {Type: database.TargetType("nope")}, + } + + for _, target := range cases { + _, err := delivery.NewTargetConfigForm(target) + require.Error(t, err, target.Type) + } +} + +// The ceiling exists so one misconfigured target cannot hold a +// delivery worker indefinitely, and it is inclusive. +func TestParseTargetTimeout_CeilingIsInclusive(t *testing.T) { + t.Parallel() + + assert.Positive(t, delivery.MaxTargetTimeoutSeconds) + + got, err := delivery.ParseTargetTimeout( + strconv.Itoa(delivery.MaxTargetTimeoutSeconds), + ) + require.NoError(t, err) + assert.Equal(t, delivery.MaxTargetTimeoutSeconds, got) + + _, err = delivery.ParseTargetTimeout( + strconv.Itoa(delivery.MaxTargetTimeoutSeconds + 1), + ) + require.Error(t, err) +} + +// Control characters in a value are how a second header would be +// smuggled into the outbound request. +func TestParseTargetHeaders_RejectsControlCharactersInValues( + t *testing.T, +) { + t.Parallel() + + for _, bad := range []string{ + "X-A: one\x01two", + "X-A: one\ttwo", + "X-A: one\x7ftwo", + } { + _, err := delivery.ParseTargetHeaders(bad) + require.Error(t, err, bad) + } +} diff --git a/internal/delivery/target_http.go b/internal/delivery/target_http.go index 3ad6fb2..127c7c4 100644 --- a/internal/delivery/target_http.go +++ b/internal/delivery/target_http.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "net/http" + "sort" "sync" "time" @@ -74,23 +75,31 @@ func (c *httpCore) fireAndForget( d *database.Delivery, res attemptResult, ) { - c.eng.recordResult( + c.eng.observeAttempt(d.Target.Type, res.elapsed()) + + err := c.eng.recordResult( webhookDB, d, 1, res.success, res.statusCode, res.respBody, res.errMsg, res.duration, ) + if err != nil { + c.eng.bookkeepingFailed(d, err) + + return + } if res.success { - c.eng.updateDeliveryStatus( - webhookDB, d, + c.eng.settleStatus( + webhookDB, d, d.Target.Type, database.DeliveryStatusDelivered, ) return } - c.eng.updateDeliveryStatus( - webhookDB, d, database.DeliveryStatusFailed, + c.eng.settleStatus( + webhookDB, d, d.Target.Type, + database.DeliveryStatusFailed, ) } @@ -107,21 +116,37 @@ func (c *httpCore) withRetry( return } + // Allow may have moved the breaker to half-open, and the + // attempt below may open or close it, so the gauge is + // republished on every exit from here. + defer c.publishCircuitState(d.Target.Type) + attemptNum := task.AttemptNum res := attempt() - c.eng.recordResult( + c.eng.observeAttempt(d.Target.Type, res.elapsed()) + + err := c.eng.recordResult( webhookDB, d, attemptNum, res.success, res.statusCode, res.respBody, res.errMsg, res.duration, ) + if err != nil { + // The breaker still learns the outcome: it describes the + // target's health, which is unaffected by this database's. + c.recordCircuitOutcome(cb, res.success) + + c.eng.bookkeepingFailed(d, err) + + return + } if res.success { cb.RecordSuccess() - c.eng.updateDeliveryStatus( - webhookDB, d, + c.eng.settleStatus( + webhookDB, d, d.Target.Type, database.DeliveryStatusDelivered, ) @@ -135,6 +160,20 @@ func (c *httpCore) withRetry( ) } +// recordCircuitOutcome feeds one attempt's outcome to the target's +// circuit breaker. +func (c *httpCore) recordCircuitOutcome( + cb *CircuitBreaker, success bool, +) { + if success { + cb.RecordSuccess() + + return + } + + cb.RecordFailure() +} + func (c *httpCore) circuitBreakerBlock( webhookDB *gorm.DB, d *database.Delivery, @@ -146,6 +185,8 @@ func (c *httpCore) circuitBreakerBlock( return false } + defer c.publishCircuitState(d.Target.Type) + remaining := cb.CooldownRemaining() c.eng.log.Info( @@ -156,8 +197,8 @@ func (c *httpCore) circuitBreakerBlock( "cooldown_remaining", remaining, ) - c.eng.updateDeliveryStatus( - webhookDB, d, + c.eng.settleStatus( + webhookDB, d, d.Target.Type, database.DeliveryStatusRetrying, ) @@ -176,16 +217,17 @@ func (c *httpCore) handleRetry( attemptNum int, ) { if attemptNum >= maxRetries { - c.eng.updateDeliveryStatus( - webhookDB, d, + c.eng.settleStatus( + webhookDB, d, d.Target.Type, database.DeliveryStatusFailed, ) return } - c.eng.updateDeliveryStatus( - webhookDB, d, database.DeliveryStatusRetrying, + c.eng.settleStatus( + webhookDB, d, d.Target.Type, + database.DeliveryStatusRetrying, ) backoff := calcBackoff(attemptNum) @@ -215,6 +257,28 @@ func (c *httpCore) getCircuitBreaker( return cb } +// publishCircuitState recounts this core's open breakers and +// publishes the gauge. Each core holds the breakers of exactly one +// target type, so the recount is over that type's targets alone. +// Counting rather than adjusting a delta keeps the gauge honest +// however a breaker changed state. +func (c *httpCore) publishCircuitState( + targetType database.TargetType, +) { + open := 0 + + c.circuitBreakers.Range(func(_, val any) bool { + cb, ok := val.(*CircuitBreaker) + if ok && cb.State() == CircuitOpen { + open++ + } + + return true + }) + + c.eng.mtr.SetCircuitBreakersOpen(targetType, open) +} + // remainingBackoff returns how long remains of the backoff // window for the last attempt of a recovered retrying // delivery. It implements rescheduler. @@ -296,13 +360,19 @@ func (t *httpTarget) Deliver( "error", err, ) - t.eng.recordResult( + recErr := t.eng.recordResult( webhookDB, d, task.AttemptNum, false, 0, "", err.Error(), 0, ) + if recErr != nil { + t.eng.bookkeepingFailed(d, recErr) - t.eng.updateDeliveryStatus( - webhookDB, d, database.DeliveryStatusFailed, + return + } + + t.eng.settleStatus( + webhookDB, d, d.Target.Type, + database.DeliveryStatusFailed, ) return @@ -363,13 +433,14 @@ func (t *httpTarget) doHTTPRequest( ) if reqErr != nil { return 0, "", 0, fmt.Errorf( - "creating request: %w", reqErr, + "creating request: %w", + maskURLError(reqErr), ) } - applyRequestHeaders(req, event, cfg) + originScoped := applyRequestHeaders(req, event, cfg) - client := t.clientForConfig(cfg) + client := t.clientForRequest(cfg, originScoped) resp, doErr := executeHTTPRequest(client, req) @@ -395,23 +466,41 @@ func (t *httpTarget) doHTTPRequest( return resp.StatusCode, string(body), dur, nil } -func (t *httpTarget) clientForConfig( +// clientForRequest returns the client for one delivery attempt. +// originScoped is the header set applyRequestHeaders built for that +// attempt; a request with neither a per-target timeout nor an +// origin-scoped header gets the shared client, because there is +// then nothing for the redirect policy to strip and net/http's +// default policy already withholds Authorization and Cookie across +// hosts. +func (t *httpTarget) clientForRequest( cfg *HTTPTargetConfig, + originScoped []string, ) *http.Client { - if cfg.Timeout > 0 { - // Reuse the shared client's SSRF-safe transport so - // a per-target timeout does not drop the - // request-time private-IP guard. Only the timeout - // is overridden. - return &http.Client{ - Timeout: time.Duration( - cfg.Timeout, - ) * time.Second, - Transport: t.client.Transport, - } + if cfg.Timeout <= 0 && len(originScoped) == 0 { + return t.client } - return t.client + // Reuse the shared client's SSRF-safe transport so neither a + // per-target timeout nor the redirect policy drops the + // request-time private-IP guard — which, being a dial hook, + // also covers every redirect hop. + client := &http.Client{ + Timeout: t.client.Timeout, + Transport: t.client.Transport, + } + + if cfg.Timeout > 0 { + client.Timeout = time.Duration( + cfg.Timeout, + ) * time.Second + } + + if len(originScoped) > 0 { + client.CheckRedirect = offOriginHeaderPolicy(originScoped) + } + + return client } func parseHTTPConfig( @@ -453,47 +542,106 @@ func isForwardableHeader(name string) bool { } } +// applyRequestHeaders builds one outbound delivery's header set and +// returns the canonical names of every header in it that is scoped +// to the configured origin: the inbound event headers this delivery +// forwarded, plus the operator's configured headers. The redirect +// policy strips exactly that set on a hop that leaves the origin, +// so the forward set is decided here and only here — a header added +// to it is covered off-origin without a second edit elsewhere. func applyRequestHeaders( req *http.Request, event *database.Event, cfg *HTTPTargetConfig, -) { +) []string { if event.ContentType != "" { req.Header.Set( "Content-Type", event.ContentType, ) } - var originalHeaders map[string][]string - - if event.Headers != "" { - jsonErr := json.Unmarshal( - []byte(event.Headers), - &originalHeaders, - ) - if jsonErr == nil { - for k, vals := range originalHeaders { - if isForwardableHeader(k) { - for _, v := range vals { - req.Header.Add(k, v) - } - } - } - } - } + originScoped := forwardEventHeaders(req, event) for k, v := range cfg.Headers { req.Header.Set(k, v) + originScoped[http.CanonicalHeaderKey(k)] = struct{}{} } req.Header.Set("User-Agent", "webhooker/1.0") + + // Content-Type describes the body being sent rather than the + // sender, and the delivery path sets it from the event itself. + // A 307/308 preserves the body across hosts, so stripping it + // would send that body untyped. + delete(originScoped, "Content-Type") + + // User-Agent is overwritten just above, so an inbound one never + // reaches the wire and the value that does identifies this + // delivery path rather than the sender. Reporting it would strip + // it off-origin and leave net/http's own default in its place. + delete(originScoped, "User-Agent") + + names := make([]string, 0, len(originScoped)) + for name := range originScoped { + names = append(names, name) + } + + sort.Strings(names) + + return names +} + +// forwardEventHeaders copies the inbound event's forwardable +// headers onto the outbound request and returns the canonical names +// it forwarded. Headers the event never carried are absent from the +// result, so the redirect policy strips what was actually sent. +func forwardEventHeaders( + req *http.Request, + event *database.Event, +) map[string]struct{} { + forwarded := make(map[string]struct{}) + + if event.Headers == "" { + return forwarded + } + + var inbound map[string][]string + + if json.Unmarshal([]byte(event.Headers), &inbound) != nil { + return forwarded + } + + for k, vals := range inbound { + if !isForwardableHeader(k) || len(vals) == 0 { + continue + } + + for _, v := range vals { + req.Header.Add(k, v) + } + + forwarded[http.CanonicalHeaderKey(k)] = struct{}{} + } + + return forwarded } // executeHTTPRequest sends an HTTP request using the provided // client. URLs are validated by the config parsers and the // SSRF-safe transport before reaching here. +// +// Transport failures are masked here, at the single point +// where every target's request errors are born, because the +// caller stores them in DeliveryResult.Error: an unmasked +// *url.Error would write the target URL — the credential for +// a Slack incoming webhook — into the per-webhook database. func executeHTTPRequest( client *http.Client, req *http.Request, ) (*http.Response, error) { - return client.Do(req) //#nosec G704 -- validated URL, SSRF-safe transport + resp, err := client.Do(req) //#nosec G704 -- validated URL, SSRF-safe transport + if err != nil { + return nil, maskURLError(err) + } + + return resp, nil } diff --git a/internal/delivery/target_log.go b/internal/delivery/target_log.go index 2484bec..096a5c7 100644 --- a/internal/delivery/target_log.go +++ b/internal/delivery/target_log.go @@ -2,6 +2,7 @@ package delivery import ( "context" + "time" "gorm.io/gorm" "sneak.berlin/go/webhooker/internal/database" @@ -11,6 +12,17 @@ import ( // inbound webhook — the full request body and headers, plus // the method, content type, and the webhook and entrypoint // ids — then records a single successful attempt. +// +// This is the one log call in the service that deliberately writes +// unbounded client-chosen bytes, so it is the one exception to the +// per-field budgets in internal/logfield and to the ceiling stated on +// middleware.MaxAccessLogLineBytes. Capping here would defeat the +// target: emitting the payload IS the delivery. It costs nothing by +// default — an authenticated operator has to create a target of this +// type on a specific webhook before a single line is written — and the +// bytes it writes are bounded per event by maxWebhookBodySize (1 MB). +// An operator who adds one is choosing to spend log volume on the +// payloads that webhook receives. type logTarget struct { eng *Engine } @@ -23,6 +35,8 @@ func (t *logTarget) Deliver( _ *Task, _ Scheduler, ) { + start := time.Now() + t.eng.log.Info( "webhook event delivered to log target", "delivery_id", d.ID, @@ -37,11 +51,22 @@ func (t *logTarget) Deliver( "body", d.Event.Body, ) - t.eng.recordResult( - webhookDB, d, 1, true, 0, "", "", 0, - ) + elapsed := time.Since(start) - t.eng.updateDeliveryStatus( - webhookDB, d, database.DeliveryStatusDelivered, + t.eng.observeAttempt(d.Target.Type, elapsed) + + err := t.eng.recordResult( + webhookDB, d, 1, true, 0, "", "", + elapsed.Milliseconds(), + ) + if err != nil { + t.eng.bookkeepingFailed(d, err) + + return + } + + t.eng.settleStatus( + webhookDB, d, d.Target.Type, + database.DeliveryStatusDelivered, ) } diff --git a/internal/delivery/target_redact.go b/internal/delivery/target_redact.go new file mode 100644 index 0000000..a9050db --- /dev/null +++ b/internal/delivery/target_redact.go @@ -0,0 +1,272 @@ +package delivery + +import ( + "net/url" + "slices" + "strings" + + "sneak.berlin/go/webhooker/internal/database" +) + +// RedactionMarker stands in for a target credential found in +// text the target's remote peer chose. +const RedactionMarker = "(redacted)" + +// Redactor removes one target's own credential material from +// text that target's remote peer chose: a delivery response +// body, or a delivery error stored before the delivery path +// learned to mask the URLs it embeds. +// +// It removes byte-identical echoes of strings taken from the +// target's stored configuration, and nothing else. Anything +// the remote re-encodes survives: JSON "\/" escaping (what +// PHP's json_encode emits by default), percent-encoding, HTML +// entities, and an echo of only part of a path. It cannot +// remove a secret the remote invented. +// +// The zero Redactor removes nothing, which is what a caller +// holding no target for a delivery gets. +type Redactor struct { + secrets []string +} + +// NewRedactor builds the redactor for one target. +func NewRedactor(t *database.Target) Redactor { + // Drop empty strings here rather than at the site that + // produced one. strings.ReplaceAll with an empty old string + // inserts the marker at every byte boundary, so a single + // empty secret destroys every body and error the target + // renders; filtering at the collection point means no field + // added to targetSecrets later can reintroduce that. + // url.Parse("https://@example.com/in") is the known + // producer: a non-nil User whose String is "". + secrets := slices.DeleteFunc( + targetSecrets(t), + func(s string) bool { return s == "" }, + ) + + // Longest first, so replacing a secret that is contained + // in a longer one cannot leave a fragment of the longer + // one behind. Configured headers arrive in map order, so + // the sort is also what makes the result deterministic. + slices.SortFunc(secrets, func(a, b string) int { + if d := len(b) - len(a); d != 0 { + return d + } + + return strings.Compare(a, b) + }) + + return Redactor{secrets: secrets} +} + +// Redact replaces every occurrence of the target's credential +// material in s. +func (r Redactor) Redact(s string) string { + if s == "" { + return s + } + + for _, secret := range r.secrets { + s = strings.ReplaceAll(s, secret, RedactionMarker) + } + + return s +} + +// RedactCut redacts s, which its caller has already cut to a +// byte budget, and additionally drops any tail of s that is a +// proper prefix of a secret. +// +// The cut lands wherever the remote's padding puts it, so the +// remote chooses where inside the credential it falls. The +// severed prefix left behind equals no secret, so plain +// Redact would render it verbatim. +func (r Redactor) RedactCut(s string) string { + s = r.Redact(s) + + if n := r.secretPrefixSuffix(s); n > 0 { + return s[:len(s)-n] + RedactionMarker + } + + return s +} + +// secretPrefixSuffix returns the length of the longest suffix +// of s that is a proper prefix of one of the secrets, or 0 +// when there is none. +func (r Redactor) secretPrefixSuffix(s string) int { + longest := 0 + + for _, secret := range r.secrets { + // Proper prefixes only: a whole secret at the tail was + // already replaced by Redact. + n := min(len(secret)-1, len(s)) + for ; n > longest; n-- { + if strings.HasSuffix(s, secret[:n]) { + longest = n + + break + } + } + } + + return longest +} + +// targetSecrets returns the credential-bearing strings a +// target's configuration carries. +// +// The destination URL contributes. Its path, query and +// userinfo are the credential for both target types that have +// one — an incoming-webhook URL is a bearer token, which is +// why MaskURL elides exactly those parts — and they are the +// material this service actually sends, so a remote that +// echoes the request back echoes them. +// +// Configured request headers contribute their values, but +// only for the credential-shaped names isCredentialHeaderName +// picks out. That is the same class-based rule applied to +// URLs: an echoed Accept or User-Agent still renders, an +// echoed Authorization does not. +func targetSecrets(t *database.Target) []string { + if t == nil { + return nil + } + + switch t.Type { + case database.TargetTypeSlack: + cfg, err := parseSlackConfig(t.Config) + if err != nil { + return nil + } + + return urlSecrets(cfg.WebhookURL) + case database.TargetTypeHTTP: + cfg, err := parseHTTPConfig(t.Config) + if err != nil { + return nil + } + + return append( + urlSecrets(cfg.URL), + headerSecrets(cfg.Headers)..., + ) + case database.TargetTypeDatabase, database.TargetTypeLog: + // Neither has a destination URL, so neither has + // anything to redact. + return nil + default: + return nil + } +} + +// urlSecrets returns the substrings of a destination URL that +// must not survive into a rendered page: the whole URL, the +// parts of it MaskURL elides, and any userinfo. +// +// No length floor is applied to the path, and none to the +// userinfo. A short path or a four-byte username is treated as +// a credential exactly like a long one, because the field takes +// an arbitrary URL and no part of it can be assumed non-secret — +// the same rule MaskURL applies. headerSecrets does carry a +// floor, and the difference is deliberate: a header is picked +// out by a name-shaped guess and its value may be ordinary +// text, whereas a URL's path and userinfo are credential +// material by position. +func urlSecrets(raw string) []string { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil + } + + secrets := []string{raw} + + parsed, err := url.Parse(raw) + if err != nil { + return secrets + } + + if parsed.Path != "" && parsed.Path != "/" { + requestURI := parsed.RequestURI() + secrets = append(secrets, requestURI) + + if escaped := parsed.EscapedPath(); escaped != requestURI { + secrets = append(secrets, escaped) + } + } + + if parsed.User != nil { + secrets = append(secrets, parsed.User.String()) + + if pw, ok := parsed.User.Password(); ok && pw != "" { + secrets = append(secrets, pw) + } + } + + return secrets +} + +// minHeaderSecretBytes is the shortest header value treated as +// a credential. Unlike a URL path, a header value can be a +// couple of bytes long, and redacting those would scatter the +// marker through ordinary response text for no gain. +const minHeaderSecretBytes = 4 + +// headerSecrets returns the values of the configured headers +// whose names are credential-shaped. +func headerSecrets(headers map[string]string) []string { + var secrets []string + + for name, value := range headers { + value = strings.TrimSpace(value) + if len(value) < minHeaderSecretBytes { + continue + } + + if isCredentialHeaderName(name) { + secrets = append(secrets, value) + } + } + + return secrets +} + +// isCredentialHeaderName classifies a header by its name. The +// value is never inspected, so the rule is the same +// class-based one MaskURL applies to a destination URL. +// +// The fragments are short on purpose, and match anywhere in +// the name, so abbreviations an operator might use are covered +// too: X-Sig, X-Pass, X-HMAC. That over-matches — a header +// named X-Design contains "sig" — and over-matching is the +// safe direction here: the cost is a marker where an echoed +// header value would have rendered. +func isCredentialHeaderName(name string) bool { + name = strings.ToLower(strings.TrimSpace(name)) + + // Names that carry a credential by definition. + switch name { + case "authorization", "proxy-authorization", "cookie": + return true + } + + // What operators call their own credential headers: + // X-Api-Key, X-Hub-Signature, X-Auth-Token. + for _, fragment := range []string{ + "auth", + "credential", + "hmac", + "key", + "pass", + "secret", + "sig", + "token", + } { + if strings.Contains(name, fragment) { + return true + } + } + + return false +} diff --git a/internal/delivery/target_redact_test.go b/internal/delivery/target_redact_test.go new file mode 100644 index 0000000..e28e302 --- /dev/null +++ b/internal/delivery/target_redact_test.go @@ -0,0 +1,284 @@ +package delivery_test + +import ( + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" +) + +// The secret path segments of a Slack incoming webhook URL. +const ( + redactSecretPath = "/services/T11111111/B11111111/" + + "YYYYYYYYYYYYYYYYYYYYYYYY" + redactWebhookURL = "https://hooks.slack.com" + + redactSecretPath +) + +func TestRedactor_RemovesSlackWebhookURL(t *testing.T) { + t.Parallel() + + r := delivery.NewRedactor(&database.Target{ + Type: database.TargetTypeSlack, + Config: `{"webhookUrl":"` + redactWebhookURL + `"}`, + }) + + got := r.Redact("no_service for " + redactWebhookURL) + + assert.NotContains(t, got, redactSecretPath) + assert.NotContains(t, got, "T11111111") + + // One marker, not a marker with the host left in front of + // it: the whole URL is replaced before the path it + // contains, which is what sorting the secrets longest + // first buys. + assert.Equal( + t, + "no_service for "+delivery.RedactionMarker, + got, + ) +} + +// TestRedactor_RemovesSecretSeveredByACut covers the input the +// redactor exists for: text cut to a byte budget with the +// credential straddling the cut. The remote chooses the +// padding, so it chooses where the cut lands inside the +// credential, and the severed prefix that remains equals no +// secret. +func TestRedactor_RemovesSecretSeveredByACut(t *testing.T) { + t.Parallel() + + r := delivery.NewRedactor(&database.Target{ + Type: database.TargetTypeSlack, + Config: `{"webhookUrl":"` + redactWebhookURL + `"}`, + }) + + // Every cut position inside the credential, not just a + // convenient one. + for n := 1; n < len(redactWebhookURL); n++ { + severed := redactWebhookURL[:n] + cut := "padding " + severed + + got := r.RedactCut(cut) + + assert.Equal( + t, + "padding "+delivery.RedactionMarker, + got, + "cut after %d bytes of the credential", n, + ) + } +} + +// TestRedactor_RedactsCredentialShapedHeaderValues pins the +// class-based header rule: a header whose name says credential +// has its value redacted, and a routine header does not, so +// ordinary response content survives. +func TestRedactor_RedactsCredentialShapedHeaderValues( + t *testing.T, +) { + t.Parallel() + + r := delivery.NewRedactor(&database.Target{ + Type: database.TargetTypeHTTP, + Config: `{"url":"https://example.com/in",` + + `"headers":{` + + `"Authorization":"Bearer AAAAAAAAAAAA",` + + `"Cookie":"session=BBBBBBBBBBBB",` + + `"X-Api-Key":"CCCCCCCCCCCC",` + + `"X-Hub-Signature":"sha256=DDDDDDDDDDDD",` + + `"X-Sig":"EEEEEEEEEEEE",` + + `"X-Pass":"FFFFFFFFFFFF",` + + `"X-HMAC":"GGGGGGGGGGGG",` + + `"X-Credential":"HHHHHHHHHHHH",` + + `"Accept":"application/json",` + + `"User-Agent":"webhooker/1.0"}}`, + }) + + for _, secret := range []string{ + "Bearer AAAAAAAAAAAA", + "session=BBBBBBBBBBBB", + "CCCCCCCCCCCC", + "sha256=DDDDDDDDDDDD", + // Abbreviated names an operator might use. + "EEEEEEEEEEEE", + "FFFFFFFFFFFF", + "GGGGGGGGGGGG", + "HHHHHHHHHHHH", + } { + got := r.Redact("echo: " + secret) + + assert.Equal( + t, + "echo: "+delivery.RedactionMarker, + got, + secret, + ) + } + + const routine = "Accept: application/json, " + + "User-Agent: webhooker/1.0" + + assert.Equal(t, routine, r.Redact(routine)) +} + +// TestRedactor_IgnoresVeryShortHeaderValues pins the floor +// under a header value. Redacting a two-byte value would put +// the marker through every response that happens to contain +// those bytes. +func TestRedactor_IgnoresVeryShortHeaderValues(t *testing.T) { + t.Parallel() + + r := delivery.NewRedactor(&database.Target{ + Type: database.TargetTypeHTTP, + Config: `{"url":"https://example.com/in",` + + `"headers":{"X-Api-Key":"ab"}}`, + }) + + const response = "rabbit" + + assert.Equal(t, response, r.Redact(response)) +} + +// TestRedactor_RemovesBarePath covers a remote that echoes +// only the request path rather than the whole URL. The path +// segments are the credential on their own. +func TestRedactor_RemovesBarePath(t *testing.T) { + t.Parallel() + + r := delivery.NewRedactor(&database.Target{ + Type: database.TargetTypeSlack, + Config: `{"webhookUrl":"` + redactWebhookURL + `"}`, + }) + + got := r.Redact("POST " + redactSecretPath + " 404") + + assert.NotContains(t, got, redactSecretPath) + assert.Equal( + t, + "POST "+delivery.RedactionMarker+" 404", + got, + ) +} + +// TestRedactor_RemovesHTTPURLQueryAndUserinfo covers the HTTP +// target, whose destination is an arbitrary URL: the query +// string and the userinfo carry credentials as readily as the +// path does. +func TestRedactor_RemovesHTTPURLQueryAndUserinfo(t *testing.T) { + t.Parallel() + + // Assembled rather than written out, so the literal is + // not itself a credential-shaped string. + dest := url.URL{ + Scheme: "https", + User: url.UserPassword("user", "hunter2"), + Host: "example.com", + Path: "/in", + RawQuery: "token=s3cr3t", + } + raw := dest.String() + + r := delivery.NewRedactor(&database.Target{ + Type: database.TargetTypeHTTP, + Config: `{"url":"` + raw + `"}`, + }) + + for _, echoed := range []string{ + raw, + "/in?token=s3cr3t", + "hunter2", + } { + got := r.Redact("rejected: " + echoed) + + assert.NotContains(t, got, "s3cr3t", echoed) + assert.NotContains(t, got, "hunter2", echoed) + assert.Contains( + t, got, delivery.RedactionMarker, echoed, + ) + } +} + +// TestRedactor_LeavesUnrelatedTextAlone pins that the +// redactor matches literally: it does not guess at what a +// secret looks like, so ordinary response content survives. +func TestRedactor_LeavesUnrelatedTextAlone(t *testing.T) { + t.Parallel() + + const response = "ok=false error=channel_not_found" + + r := delivery.NewRedactor(&database.Target{ + Type: database.TargetTypeSlack, + Config: `{"webhookUrl":"` + redactWebhookURL + `"}`, + }) + + assert.Equal(t, response, r.Redact(response)) +} + +// TestRedactor_EmptyUserinfoDoesNotShredTheBody covers a +// destination URL written with a bare "@" and no userinfo: +// url.Parse returns a non-nil User whose String is empty. An +// empty secret in the list would make strings.ReplaceAll +// insert the marker at every byte boundary, destroying every +// body and error string the target renders. +func TestRedactor_EmptyUserinfoDoesNotShredTheBody(t *testing.T) { + t.Parallel() + + const dest = "https://@example.com/in" + + // The premise: this URL really does parse to a non-nil + // User contributing an empty string. + parsed, err := url.Parse(dest) + require.NoError(t, err) + require.NotNil(t, parsed.User) + require.Empty(t, parsed.User.String()) + + r := delivery.NewRedactor(&database.Target{ + Type: database.TargetTypeHTTP, + Config: `{"url":"` + dest + `"}`, + }) + + const body = "ok=false error=channel_not_found" + + assert.Equal(t, body, r.Redact(body)) + assert.Equal(t, body, r.RedactCut(body)) + + // The real credential material still goes, so filtering the + // empty string out did not disarm the redactor. + assert.Equal( + t, + "POST "+delivery.RedactionMarker, + r.Redact("POST "+dest), + ) +} + +// TestRedactor_ZeroValueAndConfiglessTargets pins that a +// caller with no target, an unparseable config, or a target +// type with no destination URL gets a redactor that changes +// nothing rather than one that panics. +func TestRedactor_ZeroValueAndConfiglessTargets(t *testing.T) { + t.Parallel() + + const text = "some response body" + + var zero delivery.Redactor + + assert.Equal(t, text, zero.Redact(text)) + assert.Equal(t, text, delivery.NewRedactor(nil).Redact(text)) + + for _, tgt := range []database.Target{ + {Type: database.TargetTypeLog}, + {Type: database.TargetTypeDatabase}, + {Type: database.TargetTypeSlack, Config: "not json"}, + {Type: database.TargetTypeHTTP, Config: ""}, + } { + assert.Equal( + t, text, + delivery.NewRedactor(&tgt).Redact(text), + tgt.Type, + ) + } +} diff --git a/internal/delivery/target_slack.go b/internal/delivery/target_slack.go index fdb95f6..c2f7ff7 100644 --- a/internal/delivery/target_slack.go +++ b/internal/delivery/target_slack.go @@ -95,13 +95,19 @@ func (t *slackTarget) failConfig( d *database.Delivery, err error, ) { - t.eng.recordResult( + recErr := t.eng.recordResult( webhookDB, d, 1, false, 0, "", err.Error(), 0, ) + if recErr != nil { + t.eng.bookkeepingFailed(d, recErr) - t.eng.updateDeliveryStatus( - webhookDB, d, database.DeliveryStatusFailed, + return + } + + t.eng.settleStatus( + webhookDB, d, d.Target.Type, + database.DeliveryStatusFailed, ) } @@ -125,7 +131,7 @@ func (t *slackTarget) attempt( if err != nil { return attemptResult{ success: false, - errMsg: err.Error(), + errMsg: maskURLError(err).Error(), } } @@ -225,10 +231,15 @@ func FormatSlackMessage( event.ContentType, ) + timestamp := "unknown" + if !event.CreatedAt.IsZero() { + timestamp = event.CreatedAt.UTC().Format(time.RFC3339) + } + fmt.Fprintf( &b, "*Timestamp:* `%s`\n", - event.CreatedAt.UTC().Format(time.RFC3339), + timestamp, ) fmt.Fprintf( diff --git a/internal/delivery/terminal_state_test.go b/internal/delivery/terminal_state_test.go new file mode 100644 index 0000000..382a6ff --- /dev/null +++ b/internal/delivery/terminal_state_test.go @@ -0,0 +1,531 @@ +package delivery_test + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" +) + +// The two terminal-state gaps of +// https://git.eeqj.de/sneak/webhooker/issues/107: a delivery failed +// with nothing in its event log to say why, and a retrying delivery +// whose target was deleted, which used to keep sending and then never +// terminalise. + +// tUnknownType is a target type no build implements. It stands in for +// a target whose type was written by a build that knew a type this one +// does not. +const tUnknownType = database.TargetType("pubsub") + +// tSeedDeletedTarget creates a target, a retrying delivery against it +// with one recorded failed attempt, and then deletes the target the +// way the source page does. +// +// It asserts the delete is soft, because that is the whole reason the +// engine could not tell a deleted target from a target id that never +// named a row: the surviving row is invisible to a scoped read. +func tSeedDeletedTarget( + t *testing.T, + s iSetup, + name, url string, +) string { + t.Helper() + + targetID := uuid.New().String() + + iCreateTarget( + t, s.MainDB, targetID, s.WebhookID, name, + database.TargetTypeHTTP, iHTTPConfig(url), 5, + ) + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"target":"deleted"}`, + ) + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusRetrying, + ) + + iSeedFailedResult(t, s.WebhookDB, d.ID) + + require.NoError(t, s.MainDB.Delete( + &database.Target{}, "id = ?", targetID, + ).Error) + + var scoped, unscoped int64 + + require.NoError(t, s.MainDB. + Model(&database.Target{}). + Where("id = ?", targetID). + Count(&scoped).Error) + + require.NoError(t, s.MainDB.Unscoped(). + Model(&database.Target{}). + Where("id = ?", targetID). + Count(&unscoped).Error) + + require.Zero(t, scoped, + "the deleted target is still visible to a scoped read", + ) + require.Equal(t, int64(1), unscoped, + "the delete was hard, so this test proves nothing about "+ + "the soft-delete case it exists for", + ) + + return d.ID +} + +// tLastResult returns a delivery's final recorded attempt, asserting +// the expected number of them. +func tLastResult( + t *testing.T, + s iSetup, + deliveryID string, + want int, +) database.DeliveryResult { + t.Helper() + + results := iResults(t, s.WebhookDB, deliveryID) + require.Len(t, results, want) + + return results[want-1] +} + +// --- 1. A failure with nothing recorded --- + +func TestProcessDelivery_UnknownTargetType_RecordsWhy( + t *testing.T, +) { + t.Parallel() + + s := newISetup(t) + + targetID := uuid.New().String() + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"unknown":"type"}`, + ) + + seeded := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusPending, + ) + + target := database.Target{ + Name: "mystery", + Type: tUnknownType, + Config: iHTTPConfig("http://example.com/hook"), + } + target.ID = targetID + + d := database.Delivery{ + EventID: event.ID, + TargetID: targetID, + Status: database.DeliveryStatusPending, + Event: event, + Target: target, + } + d.ID = seeded.ID + + body := event.Body + task := iTask( + seeded, event, s.WebhookID, targetID, "mystery", + target.Config, 0, 1, &body, + ) + task.TargetType = tUnknownType + + s.Engine.ExportProcessDelivery( + context.Background(), s.WebhookDB, &d, &task, + ) + + iAssertStatus( + t, s.WebhookDB, d.ID, database.DeliveryStatusFailed, + ) + + last := tLastResult(t, s, d.ID, 1) + + assert.False(t, last.Success) + assert.Equal(t, 1, last.AttemptNum) + assert.Contains(t, last.Error, string(tUnknownType), + "the recorded reason does not name the offending type", + ) +} + +// --- 2. A retrying delivery whose target is gone --- + +func TestRecoverSingleRetry_TargetDeleted(t *testing.T) { + t.Parallel() + + s := newISetup(t) + + iCreateWebhook( + t, s.MainDB, s.WebhookID, "deleted-target-recovery", + ) + + deliveryID := tSeedDeletedTarget( + t, s, "gone-on-recovery", "http://example.com/hook", + ) + + s.Engine.ExportRecoverWebhookDeliveries( + context.Background(), s.WebhookID, + ) + + iAssertStatus( + t, s.WebhookDB, deliveryID, + database.DeliveryStatusFailed, + ) + + last := tLastResult(t, s, deliveryID, 2) + + assert.False(t, last.Success) + assert.Equal(t, 2, last.AttemptNum) + assert.Contains(t, last.Error, "gone-on-recovery") + assert.Contains(t, last.Error, "was deleted") + + assert.Empty(t, s.Engine.ExportRetryCh(), + "a delivery whose target is gone was rescheduled", + ) + assert.Zero(t, s.Engine.ExportInflightHeld(), + "the terminal path leaked its ownership reference", + ) +} + +func TestSweepSingleRetry_TargetDeleted(t *testing.T) { + t.Parallel() + + s := newISetup(t) + + iCreateWebhook( + t, s.MainDB, s.WebhookID, "deleted-target-sweep", + ) + + deliveryID := tSeedDeletedTarget( + t, s, "gone-on-sweep", "http://example.com/hook", + ) + + // Twice, because the bug was an error the sweep repeated every + // minute for the life of the database: the second sweep must + // find nothing left to do. + s.Engine.ExportSweepWebhookRetries( + context.Background(), s.WebhookID, + ) + s.Engine.ExportSweepWebhookRetries( + context.Background(), s.WebhookID, + ) + + iAssertStatus( + t, s.WebhookDB, deliveryID, + database.DeliveryStatusFailed, + ) + + last := tLastResult(t, s, deliveryID, 2) + + assert.Contains(t, last.Error, "gone-on-sweep") + assert.Contains(t, last.Error, "was deleted") + + assert.Empty(t, s.Engine.ExportRetryCh()) + assert.Zero(t, s.Engine.ExportInflightHeld()) +} + +// TestSweepSingleRetry_TargetNeverExisted covers the other half of the +// soft-delete distinction: an id with no row at all, deleted or +// otherwise, must not be reported as something the operator deleted. +func TestSweepSingleRetry_TargetNeverExisted(t *testing.T) { + t.Parallel() + + s := newISetup(t) + + iCreateWebhook( + t, s.MainDB, s.WebhookID, "target-never-existed", + ) + + targetID := uuid.New().String() + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"target":"absent"}`, + ) + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusRetrying, + ) + + iSeedFailedResult(t, s.WebhookDB, d.ID) + + s.Engine.ExportSweepWebhookRetries( + context.Background(), s.WebhookID, + ) + + iAssertStatus( + t, s.WebhookDB, d.ID, database.DeliveryStatusFailed, + ) + + last := tLastResult(t, s, d.ID, 2) + + assert.Contains(t, last.Error, targetID) + assert.Contains(t, last.Error, "no longer exists") + assert.NotContains(t, last.Error, "was deleted", + "an id that never named a row was reported as a deletion", + ) +} + +// TestFailMissingTargetRetry_WritesNoTargetRow holds the new terminal +// path to the same rule as the existing one: no target row, and so no +// plaintext target config, may be written into the per-webhook event +// database. See https://git.eeqj.de/sneak/webhooker/issues/206. +func TestFailMissingTargetRetry_WritesNoTargetRow( + t *testing.T, +) { + t.Parallel() + + s := newISetup(t) + + iCreateWebhook( + t, s.MainDB, s.WebhookID, "no-target-row-deleted", + ) + + hookURL := "https://hooks.slack.com/services/T00/B00/x" + + deliveryID := tSeedDeletedTarget( + t, s, "credential-bearing", hookURL, + ) + + s.Engine.ExportSweepWebhookRetries( + context.Background(), s.WebhookID, + ) + + iAssertStatus( + t, s.WebhookDB, deliveryID, + database.DeliveryStatusFailed, + ) + + var configs []string + + require.NoError(t, s.WebhookDB. + Table("targets"). + Pluck("config", &configs).Error) + + assert.Empty(t, configs, + "the deleted-target terminal path wrote a target row "+ + "into the per-webhook event database", + ) +} + +// --- 3. The scheduled retry chain --- + +// tRetryChainSetup wires a counting sink and a retrying delivery +// against a live target pointing at it, and returns the task a +// scheduled retry would carry — config and all, snapshotted as +// ScheduleRetry snapshots it. +func tRetryChainSetup( + t *testing.T, + s iSetup, + name string, + hits *atomic.Int64, +) (delivery.Task, string) { + t.Helper() + + ts := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusOK) + }, + )) + t.Cleanup(ts.Close) + + iCreateWebhook(t, s.MainDB, s.WebhookID, name) + + targetID := uuid.New().String() + cfg := iHTTPConfig(ts.URL) + + iCreateTarget( + t, s.MainDB, targetID, s.WebhookID, name, + database.TargetTypeHTTP, cfg, 5, + ) + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"chain":"retry"}`, + ) + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusRetrying, + ) + + iSeedFailedResult(t, s.WebhookDB, d.ID) + + body := event.Body + + return iTask( + d, event, s.WebhookID, targetID, name, cfg, 5, 2, &body, + ), targetID +} + +// TestProcessRetryTask_TargetDeleted_MakesNoAttempt is the half the +// deployability audit found worse than filed: terminalising on +// recovery and sweep alone leaves the already-scheduled timer chain +// running, and it holds the target's configuration from before the +// deletion, so it goes on sending to a destination that was removed. +func TestProcessRetryTask_TargetDeleted_MakesNoAttempt( + t *testing.T, +) { + t.Parallel() + + s := newISetup(t) + + var hits atomic.Int64 + + task, targetID := tRetryChainSetup( + t, s, "gone-mid-chain", &hits, + ) + + require.NoError(t, s.MainDB.Delete( + &database.Target{}, "id = ?", targetID, + ).Error) + + s.Engine.ExportProcessRetryTask( + context.Background(), &task, + ) + + assert.Zero(t, hits.Load(), + "a scheduled retry fired at a target the operator "+ + "had already deleted", + ) + + iAssertStatus( + t, s.WebhookDB, task.DeliveryID, + database.DeliveryStatusFailed, + ) + + last := tLastResult(t, s, task.DeliveryID, 2) + + assert.False(t, last.Success) + assert.Contains(t, last.Error, "was deleted") + + assert.Zero(t, s.Engine.ExportInflightHeld()) +} + +// TestProcessRetryTask_TargetPresent_StillDelivers is the guard's +// mutation check: a liveness check that refused every retry would pass +// the test above and break every retry there is. +func TestProcessRetryTask_TargetPresent_StillDelivers( + t *testing.T, +) { + t.Parallel() + + s := newISetup(t) + + var hits atomic.Int64 + + task, _ := tRetryChainSetup(t, s, "still-there", &hits) + + s.Engine.ExportProcessRetryTask( + context.Background(), &task, + ) + + assert.Equal(t, int64(1), hits.Load()) + + iAssertStatus( + t, s.WebhookDB, task.DeliveryID, + database.DeliveryStatusDelivered, + ) +} + +// TestProcessRetryTask_TargetUnreadable_StillDelivers pins the other +// half of the guard: only a target that is confirmed gone stops a +// retry. A main database that cannot be read is a transient fault, and +// a guard that abandoned deliveries on one would be a worse bug than +// the one it fixes. +func TestProcessRetryTask_TargetUnreadable_StillDelivers( + t *testing.T, +) { + t.Parallel() + + s := newISetup(t) + + var hits atomic.Int64 + + task, _ := tRetryChainSetup(t, s, "unreadable-main", &hits) + + sqlDB, err := s.MainDB.DB() + require.NoError(t, err) + require.NoError(t, sqlDB.Close()) + + s.Engine.ExportProcessRetryTask( + context.Background(), &task, + ) + + assert.Equal(t, int64(1), hits.Load(), + "a retry was abandoned because the main database "+ + "could not be read, not because its target was gone", + ) + + iAssertStatus( + t, s.WebhookDB, task.DeliveryID, + database.DeliveryStatusDelivered, + ) +} + +// TestRecoverSingleRetry_TargetUnreadable_LeavesDeliveryAlone is the +// same rule on the recovery path. A read failure that is not +// "record not found" must leave every retrying delivery of every +// webhook exactly as it was. +func TestRecoverSingleRetry_TargetUnreadable_LeavesDeliveryAlone( + t *testing.T, +) { + t.Parallel() + + s := newISetup(t) + + iCreateWebhook( + t, s.MainDB, s.WebhookID, "unreadable-on-recovery", + ) + + targetID := uuid.New().String() + + iCreateTarget( + t, s.MainDB, targetID, s.WebhookID, "healthy", + database.TargetTypeHTTP, + iHTTPConfig("http://example.com/hook"), 5, + ) + + event := iSeedEvent( + t, s.WebhookDB, s.WebhookID, `{"still":"retrying"}`, + ) + + d := iSeedDelivery( + t, s.WebhookDB, event.ID, targetID, + database.DeliveryStatusRetrying, + ) + + iSeedFailedResult(t, s.WebhookDB, d.ID) + + sqlDB, err := s.MainDB.DB() + require.NoError(t, err) + require.NoError(t, sqlDB.Close()) + + s.Engine.ExportRecoverRetryingDeliveries( + s.WebhookDB, s.WebhookID, + ) + + iAssertStatus( + t, s.WebhookDB, d.ID, + database.DeliveryStatusRetrying, + ) + + assert.Len(t, iResults(t, s.WebhookDB, d.ID), 1, + "an unreadable main database produced a terminal "+ + "failure row", + ) + + assert.Zero(t, s.Engine.ExportInflightHeld()) +} diff --git a/internal/delivery/url_mask.go b/internal/delivery/url_mask.go new file mode 100644 index 0000000..95821d2 --- /dev/null +++ b/internal/delivery/url_mask.go @@ -0,0 +1,61 @@ +package delivery + +import ( + "errors" + "net/url" +) + +// urlPathElision stands in for a URL's elided path. +const urlPathElision = "/..." + +// MaskURL renders a URL as scheme plus host with everything +// that can carry a secret removed. A delivery target URL is +// itself a credential — a Slack incoming webhook URL is a +// bearer token — so the path, query and userinfo are never +// reproduced, in a page, a log line or a stored error. A URL +// that does not parse into a scheme and host yields the +// neutral placeholder, never the raw string. +func MaskURL(raw string) string { + parsed, err := url.Parse(raw) + if err != nil || parsed.Scheme == "" || + parsed.Host == "" { + return configUnavailable + } + + masked := parsed.Scheme + "://" + parsed.Host + + if parsed.Path != "" && parsed.Path != "/" { + masked += urlPathElision + } + + return masked +} + +// maskURLError strips the credential from an error raised +// against a request URL. The net/http and net/url packages +// embed the full request URL in every *url.Error they return, +// so an unmodified transport error persisted into +// DeliveryResult.Error writes the credential to disk. +// +// The masked error keeps the operation and the wrapped cause, +// so a DNS failure still reads differently from a refused +// connection, a TLS handshake failure or a timeout, and Is, +// As, Timeout and Temporary keep working on it. Only the +// path, query and userinfo of the URL are dropped. Errors +// that carry no URL are returned unchanged. +// +// Call it where the error is raised, before any wrapping: it +// replaces the *url.Error itself, so any context wrapped +// around it first would be discarded. +func maskURLError(err error) error { + var urlErr *url.Error + if !errors.As(err, &urlErr) { + return err + } + + return &url.Error{ + Op: urlErr.Op, + URL: MaskURL(urlErr.URL), + Err: urlErr.Err, + } +} diff --git a/internal/delivery/url_mask_test.go b/internal/delivery/url_mask_test.go new file mode 100644 index 0000000..f699b36 --- /dev/null +++ b/internal/delivery/url_mask_test.go @@ -0,0 +1,196 @@ +package delivery_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" +) + +// The path of a Slack incoming webhook URL is the credential: +// whoever holds these segments can post to the channel +// forever. None of them may reach a stored delivery error, +// which lives on disk in the per-webhook database and is +// serialized by the JSON tag on DeliveryResult.Error. +const ( + maskSecretPath = "/services/T00000000/B00000000/" + + "XXXXXXXXXXXXXXXXXXXXXXXX" +) + +// assertNoCredential fails if the whole path or any single +// segment of it survived into the message, so a partial leak +// fails the test too. +func assertNoCredential(t *testing.T, msg string) { + t.Helper() + + segments := []string{ + maskSecretPath, + "services", + "T00000000", + "B00000000", + "XXXXXXXXXXXXXXXXXXXXXXXX", + } + + for _, segment := range segments { + assert.NotContains(t, msg, segment) + } +} + +// storedDeliveryError returns the error string persisted for a +// delivery, which is what an operator and any future API read. +func storedDeliveryError( + t *testing.T, db *gorm.DB, deliveryID string, +) string { + t.Helper() + + var result database.DeliveryResult + + require.NoError(t, db.Where( + "delivery_id = ?", deliveryID, + ).First(&result).Error) + + return result.Error +} + +// deliverSlackTo runs a Slack delivery against webhookURL and +// returns the error string it persisted. +func deliverSlackTo( + t *testing.T, webhookURL string, +) string { + t.Helper() + + db := testWebhookDB(t) + e := testEngine(t, 1) + targetID := uuid.New().String() + + slackCfg, err := json.Marshal( + delivery.SlackTargetConfig{ + WebhookURL: webhookURL, + }, + ) + require.NoError(t, err) + + event := seedEvent(t, db, `{"test":true}`) + + dlv := seedDelivery( + t, db, event.ID, targetID, + database.DeliveryStatusPending, + ) + + d := buildSlackDelivery( + dlv, event, targetID, + "test-slack-mask", string(slackCfg), + ) + + e.ExportDeliverSlack(context.TODO(), db, d) + + assertDeliveryStatus(t, db, dlv.ID, + database.DeliveryStatusFailed, + ) + + return storedDeliveryError(t, db, dlv.ID) +} + +// TestDeliverSlack_TransportErrorMasksWebhookURL is the +// load-bearing regression test: a transport failure must not +// persist the webhook URL's credential into the database, and +// must still say what went wrong and where. +func TestDeliverSlack_TransportErrorMasksWebhookURL( + t *testing.T, +) { + t.Parallel() + + // A server closed before use gives a deterministic + // transport failure against a known host. + ts := httptest.NewServer(http.NewServeMux()) + host := ts.URL + + ts.Close() + + errMsg := deliverSlackTo(t, host+maskSecretPath) + + require.NotEmpty(t, errMsg) + assertNoCredential(t, errMsg) + + // The diagnostic value survives: the operation, the host + // and the transport failure are all still reported, and + // only the path is elided. + assert.Contains(t, errMsg, "sending request") + assert.Contains(t, errMsg, "Post") + assert.Contains(t, errMsg, host+"/...") + assert.Contains(t, errMsg, "connection refused") +} + +// TestDeliverSlack_UnparsableURLMasksWebhookURL covers the +// other error path out of a Slack attempt: url.Parse also +// embeds the whole URL in the error it returns. +func TestDeliverSlack_UnparsableURLMasksWebhookURL( + t *testing.T, +) { + t.Parallel() + + errMsg := deliverSlackTo( + t, + "https://hooks.slack.com"+maskSecretPath+"\n", + ) + + require.NotEmpty(t, errMsg) + assertNoCredential(t, errMsg) + assert.Contains(t, errMsg, "invalid control character") +} + +// TestDoHTTPRequest_TransportErrorMasksURL proves the HTTP +// target's transport errors are masked too; its destination +// URL can carry a token in a query string. +func TestDoHTTPRequest_TransportErrorMasksURL(t *testing.T) { + t.Parallel() + + ts := httptest.NewServer(http.NewServeMux()) + host := ts.URL + + ts.Close() + + e := testEngine(t, 1) + + cfg, err := e.ExportParseHTTPConfig( + newHTTPTargetConfig(host + maskSecretPath), + ) + require.NoError(t, err) + + statusCode, _, _, reqErr := e.ExportDoHTTPRequest( + context.TODO(), cfg, + &database.Event{Body: `{"test":true}`}, + ) + require.Error(t, reqErr) + assert.Zero(t, statusCode) + + assertNoCredential(t, reqErr.Error()) + assert.Contains(t, reqErr.Error(), host+"/...") + assert.Contains( + t, reqErr.Error(), "connection refused", + ) +} + +// TestValidateTargetURL_UnparsableURLIsMasked proves the SSRF +// validator's error does not carry the submitted URL, which +// the handler both logs and shows. +func TestValidateTargetURL_UnparsableURLIsMasked(t *testing.T) { + t.Parallel() + + err := delivery.NewTestGuard().ValidateTargetURL( + context.TODO(), + "https://hooks.slack.com"+maskSecretPath+"\n", + ) + require.Error(t, err) + + assertNoCredential(t, err.Error()) + assert.Contains(t, err.Error(), "invalid URL") +} diff --git a/internal/gormlog/export_test.go b/internal/gormlog/export_test.go new file mode 100644 index 0000000..3d249b7 --- /dev/null +++ b/internal/gormlog/export_test.go @@ -0,0 +1,17 @@ +package gormlog + +import ( + "log/slog" + "time" +) + +// ExportNewWithSlowThreshold builds a Logger whose slow-statement +// threshold is d rather than DefaultSlowThreshold, so a test can pin +// which arm of Trace it is exercising instead of racing the clock on a +// loaded machine. The threshold is set at construction, like every +// other field, so the type's concurrency guarantee still holds. +func ExportNewWithSlowThreshold( + log *slog.Logger, d time.Duration, +) *Logger { + return &Logger{log: log, slowThreshold: d} +} diff --git a/internal/gormlog/firstboot_test.go b/internal/gormlog/firstboot_test.go new file mode 100644 index 0000000..6fab1a5 --- /dev/null +++ b/internal/gormlog/firstboot_test.go @@ -0,0 +1,229 @@ +package gormlog_test + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/fx" + "go.uber.org/fx/fxtest" + _ "modernc.org/sqlite" // Pure Go SQLite driver. + "sneak.berlin/go/webhooker/internal/config" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/globals" + "sneak.berlin/go/webhooker/internal/logger" + "sneak.berlin/go/webhooker/internal/session" +) + +// argon2Prefix opens every encoded Argon2id hash this service +// produces. It is asserted on separately from the hash itself, so that +// a change to the password encoding cannot quietly turn the hash +// assertion into a comparison against a string the log never held. +const argon2Prefix = "$argon2id$" + +// settingsInsert and usersInsert are the two statements a first boot +// runs that carry a secret. The sqlite dialector quotes identifiers +// with backticks. +const ( + settingsInsert = "INSERT INTO `settings`" + usersInsert = "INSERT INTO `users`" +) + +// captureStdoutToFile redirects os.Stdout into a file for the rest of +// the test and returns a function that reads back everything written +// to it. +// +// A file rather than a pipe: internal/logger writes synchronously to +// whatever os.Stdout is when it builds its handler, so once fx's start +// returns, every byte the boot produced is already in the file and no +// draining goroutine is needed to prove it. Redirecting the variable +// before the application is built is what puts the service logger — +// and therefore the GORM adapter, which writes through it — into the +// capture. +// +// The redirect also decides the handler: a regular file is not a +// character device, so internal/logger installs its JSON handler, the +// one it installs in production under a log collector. +func captureStdoutToFile(t *testing.T) func() string { + t.Helper() + + path := filepath.Join(t.TempDir(), "stdout.log") + + //nolint:gosec // The path is this test's own t.TempDir(). + f, err := os.Create(path) + require.NoError(t, err) + + orig := os.Stdout + os.Stdout = f + + t.Cleanup(func() { + os.Stdout = orig + _ = f.Close() + }) + + return func() string { + require.NoError(t, f.Sync()) + + //nolint:gosec // As above. + b, readErr := os.ReadFile(path) + require.NoError(t, readErr) + + return string(b) + } +} + +// firstBootSecrets are the two values a first boot generates and +// stores, read back out of the database. +type firstBootSecrets struct { + sessionKey string + passwordHash string +} + +// readFirstBootSecrets reads those two secrets straight out of the +// SQLite file with database/sql rather than through GORM, so that +// reading them cannot itself add a line to the log under test. +func readFirstBootSecrets( + t *testing.T, dataDir string, +) firstBootSecrets { + t.Helper() + + db, err := sql.Open("sqlite", filepath.Join( + dataDir, "webhooker.db", + )) + require.NoError(t, err) + + defer func() { require.NoError(t, db.Close()) }() + + ctx := context.Background() + + var got firstBootSecrets + + require.NoError(t, db.QueryRowContext( + ctx, `SELECT value FROM settings WHERE key = 'session_key'`, + ).Scan(&got.sessionKey)) + + require.NoError(t, db.QueryRowContext( + ctx, `SELECT password FROM users WHERE username = 'admin'`, + ).Scan(&got.passwordHash)) + + require.NotEmpty(t, got.sessionKey) + require.Contains(t, got.passwordHash, argon2Prefix) + + return got +} + +// bootAtDebug starts and stops the real application graph against +// dataDir with DEBUG=true, and returns everything it wrote to standard +// output. +// +// config.New reads DEBUG from the environment exactly as the binary +// does, internal/logger builds the handler it builds in production, +// database.New runs the migrations and creates the admin user, and +// session.New takes the session key. Those four are the whole of the +// path that writes either secret. +func bootAtDebug(t *testing.T, dataDir string) string { + t.Helper() + + t.Setenv("DEBUG", "true") + t.Setenv("DATA_DIR", dataDir) + + read := captureStdoutToFile(t) + + var sess *session.Session + + app := fxtest.New( + t, + fx.Provide( + globals.New, + logger.New, + config.New, + database.New, + session.New, + ), + fx.Populate(&sess), + ) + + app.RequireStart() + app.RequireStop() + + return read() +} + +// requireFirstBootWasLogged is the non-vacuity half of the test below. +// Without it a build that logged no SQL at all, or that never reached +// DEBUG, would satisfy every absence assertion. +func requireFirstBootWasLogged(t *testing.T, out string) { + t.Helper() + + require.Contains( + t, out, `"level":"DEBUG"`, + "DEBUG=true did not reach the logger", + ) + require.Contains( + t, out, settingsInsert, + "the session key INSERT was not logged, so its absence "+ + "proves nothing", + ) + require.Contains( + t, out, usersInsert, + "the admin user INSERT was not logged, so its absence "+ + "proves nothing", + ) +} + +// TestFirstBootAtDebug_LogsNeitherSecret is the definition of done. +// +// A first boot is the only boot that writes either secret. The +// settings INSERT carries the base64 session encryption key, which is +// the whole of the session security model: anyone holding it can forge +// an authenticated session cookie. The users INSERT carries the admin +// account's Argon2id hash. Under interpolated statement logging both +// landed in the log an operator diagnosing a startup problem pastes +// into an issue. +// +// The secrets are read back out of the database file afterwards, so +// the assertions are made against the values this boot actually +// generated rather than against a pattern that might not match them. +// +// What this test does not cover: the initial admin password itself, +// which internal/database logs once in the clear, on purpose, because +// that line is the only place an operator ever sees it. That is a +// separate decision from the SQL log, and it is documented in the +// README rather than asserted here. +// +// Not parallel: it redirects os.Stdout and sets environment variables, +// both process-global. +// +//nolint:paralleltest // Deliberately sequential; see above. +func TestFirstBootAtDebug_LogsNeitherSecret(t *testing.T) { + dataDir := t.TempDir() + + // An empty DATA_DIR is what makes this a first boot: with a + // database already in place neither INSERT runs. + entries, err := os.ReadDir(dataDir) + require.NoError(t, err) + require.Empty(t, entries, "DATA_DIR was not empty") + + out := bootAtDebug(t, dataDir) + + requireFirstBootWasLogged(t, out) + + secrets := readFirstBootSecrets(t, dataDir) + + assert.NotContains( + t, out, secrets.sessionKey, + "the session encryption key reached the debug log", + ) + assert.NotContains( + t, out, secrets.passwordHash, + "the admin password hash reached the debug log", + ) + assert.NotContains( + t, out, argon2Prefix, + "an encoded Argon2id hash reached the debug log", + ) +} diff --git a/internal/gormlog/gormlog.go b/internal/gormlog/gormlog.go new file mode 100644 index 0000000..a44dd03 --- /dev/null +++ b/internal/gormlog/gormlog.go @@ -0,0 +1,219 @@ +// Package gormlog adapts GORM's logger onto the service's slog +// logger. +// +// GORM's own default logger is not usable here. It is built at package +// init with log.New(os.Stdout, ...) at LogLevel Warn with +// IgnoreRecordNotFoundError false, so it writes the fully interpolated +// SQL — parameters and all — for every statement that returns an +// error, including gorm.ErrRecordNotFound. Two of this service's +// lookups miss by design on unauthenticated routes: the entrypoint +// lookup on /webhook/{uuid}, whose path segment the client picks +// outright, and the user lookup behind the login form, whose username +// the client picks outright. Under the default logger each of those +// misses printed an unbounded, attacker-chosen string, at no level the +// operator can turn down, past every handler internal/logger installs. +// +// This adapter fixes all three properties at once: the lines get a +// level the operator controls, they are shaped by whichever handler +// internal/logger selected, and every value a client can influence is +// spent through logfield.Truncate. +// +// It also logs no bound value at all. See ParamsFilter: the statement +// is written with its placeholders intact, at every level, so the +// values a statement carries never reach the log in the first place. +package gormlog + +import ( + "context" + "errors" + "fmt" + "log/slog" + "time" + + "gorm.io/gorm" + gormlogger "gorm.io/gorm/logger" + "sneak.berlin/go/webhooker/internal/logfield" +) + +// DefaultSlowThreshold is the duration at or above which a statement +// is logged as slow. It is GORM's own default, kept deliberately: slow +// SQL is the one thing GORM's logger reports that nothing else in this +// service does, so silencing the logger outright would have cost real +// observability to fix a log-volume defect. +const DefaultSlowThreshold = 200 * time.Millisecond + +// Logger implements gormlogger.Interface on top of an *slog.Logger. +// +// It is safe for concurrent use: every field is set at construction +// and never written again. +type Logger struct { + log *slog.Logger + slowThreshold time.Duration +} + +// Interface compliance is asserted here rather than discovered at the +// gorm.Open call sites. gorm.ParamsFilter is the optional half: GORM +// type-asserts for it and silently keeps interpolating if it is +// missing, so losing it would cost no build error and no test that +// does not look at the emitted SQL. +var ( + _ gormlogger.Interface = (*Logger)(nil) + _ gorm.ParamsFilter = (*Logger)(nil) +) + +// New returns a GORM logger that writes through log. +func New(log *slog.Logger) *Logger { + return &Logger{ + log: log, + slowThreshold: DefaultSlowThreshold, + } +} + +// LogMode returns the logger unchanged. +// +// GORM's LogLevel is deliberately not honoured. Level is the operator's +// decision and it is expressed once, through LOG_LEVEL and the +// slog.LevelVar internal/logger holds; a second level knob inside the +// database layer could only disagree with it. The mapping from GORM's +// four categories onto slog levels is fixed in Trace below. +// +//nolint:ireturn // The interface return is GORM's signature, not a choice. +func (l *Logger) LogMode(gormlogger.LogLevel) gormlogger.Interface { + return l +} + +// ParamsFilter drops every bound value before GORM renders a statement +// for the log, so what is logged is the statement's shape — its +// placeholders — and never the values in it. +// +// GORM builds the string it hands to Trace by calling +// Dialector.Explain(sql, vars...), which substitutes each value into +// the statement. Discarding vars here leaves the '?' placeholders in +// place, because ExplainSQL only substitutes while it still has a +// value for the next one. That happens before Trace is reached, so it +// holds on all three of its arms: the failed statement, the slow one, +// and the routine one an operator sees at DEBUG. +// +// This is the whole of the fix, and it is deliberately unconditional +// rather than a list of tables to redact. At first boot the two +// statements that carry a secret are the INSERT into settings holding +// the base64 session key — which is the entire session security model, +// since anyone with it can mint a valid cookie — and the INSERT into +// users holding the Argon2id hash. A denylist would have had to be +// extended by hand for every table added afterwards, and the cost of +// missing one is a credential in a log that gets pasted into issues. +// +// What is given up is the ability to read a value out of the log. The +// statement, the table, the error and the row count are all still +// there, which is what identifies a failing statement; reproducing it +// needs the values, and those an operator now gets from the database +// rather than from the log. +// +// One GORM path does not consult this: (*gorm.DB).Scan records the +// statement through gorm's own traceRecorder, which does not implement +// this interface. No production code path calls it; its one caller is +// internal/database/database_test.go:91, whose SELECT 1 binds nothing. +// scan_guard_test.go fails if a non-test file calls it. +// (*gorm.DB).Pluck, Row and Raw all run through the normal callback +// processor and are filtered. +func (l *Logger) ParamsFilter( + _ context.Context, sql string, _ ...any, +) (string, []any) { + return sql, nil +} + +// Info logs one of GORM's own informational messages. +func (l *Logger) Info( + ctx context.Context, msg string, data ...any, +) { + l.log.InfoContext(ctx, "gorm", "message", format(msg, data...)) +} + +// Warn logs one of GORM's own warnings. +func (l *Logger) Warn( + ctx context.Context, msg string, data ...any, +) { + l.log.WarnContext(ctx, "gorm", "message", format(msg, data...)) +} + +// Error logs one of GORM's own errors. +func (l *Logger) Error( + ctx context.Context, msg string, data ...any, +) { + l.log.ErrorContext(ctx, "gorm", "message", format(msg, data...)) +} + +// Trace reports the outcome of a single statement. GORM calls it for +// every statement it runs, so the cheap paths stay cheap: fc() renders +// the statement — with placeholders, per ParamsFilter — and is called +// only on a branch that will actually emit. +// +// The arms are ordered exactly as GORM's own Trace orders them — +// non-record-not-found error, then slow, then the routine case — so +// that a statement which both misses and runs slow is still reported +// as slow. A miss is the likeliest statement to be slow, since it is +// the one that scans without finding a row, and ordering the drop +// ahead of the slow arm would have made this adapter less observant +// than the IgnoreRecordNotFoundError option it was chosen over. +func (l *Logger) Trace( + ctx context.Context, + begin time.Time, + fc func() (string, int64), + err error, +) { + elapsed := time.Since(begin) + + switch { + case err != nil && !errors.Is(err, gormlogger.ErrRecordNotFound): + sql, rows := fc() + l.log.ErrorContext(ctx, "sql statement failed", + "error", logfield.Truncate(err.Error(), logfield.MaxBytes), + "sql", logfield.Truncate(sql, logfield.MaxBytes), + "rows", rows, + "elapsed_ms", elapsed.Milliseconds(), + ) + + case l.slowThreshold > 0 && elapsed >= l.slowThreshold: + sql, rows := fc() + l.log.WarnContext(ctx, "slow sql statement", + "sql", logfield.Truncate(sql, logfield.MaxBytes), + "rows", rows, + "elapsed_ms", elapsed.Milliseconds(), + "threshold_ms", l.slowThreshold.Milliseconds(), + ) + + case err != nil: + // gorm.ErrRecordNotFound is not an error on the paths that + // produce it here: an invented entrypoint UUID and an unknown + // username are the expected outcome of an unauthenticated + // request, not a fault. This is the IgnoreRecordNotFoundError + // behaviour, and it is unconditional rather than configurable + // because no caller in this service wants the other one — the + // two handlers that care already record the miss themselves, + // at DEBUG, without the SQL. A miss that ran slow has already + // been reported by the arm above. + return + + case l.log.Enabled(ctx, slog.LevelDebug): + sql, rows := fc() + l.log.DebugContext(ctx, "sql statement", + "sql", logfield.Truncate(sql, logfield.MaxBytes), + "rows", rows, + "elapsed_ms", elapsed.Milliseconds(), + ) + } +} + +// format renders one of GORM's printf-style internal messages and +// bounds it. GORM builds these itself, but they can quote a value the +// statement carried, so they are spent through the same budget as +// everything else rather than trusted. +func format(msg string, data ...any) string { + if len(data) == 0 { + return logfield.Truncate(msg, logfield.MaxBytes) + } + + return logfield.Truncate( + fmt.Sprintf(msg, data...), logfield.MaxBytes, + ) +} diff --git a/internal/gormlog/gormlog_test.go b/internal/gormlog/gormlog_test.go new file mode 100644 index 0000000..084f98f --- /dev/null +++ b/internal/gormlog/gormlog_test.go @@ -0,0 +1,438 @@ +package gormlog_test + +import ( + "bytes" + "context" + "database/sql" + "fmt" + "log/slog" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + _ "modernc.org/sqlite" // Pure Go SQLite driver. + "sneak.berlin/go/webhooker/internal/gormlog" + "sneak.berlin/go/webhooker/internal/middleware" +) + +// fillBytes is how much client-chosen text each case drives into the +// statement. It is well past every budget in play, so a value that +// arrives short arrived short because something cut it. +const fillBytes = 8 << 10 + +// tailMarker sits at the far end of every generated value. A line that +// contains it carried the whole value, which means nothing cut it — so +// a value that merely happened to be short cannot pass for a truncated +// one. +const tailMarker = "ENDOFCLIENTVALUE" + +// fills are the characters a client can drive into a SQL parameter, +// chosen for what the log handlers charge for them rather than for +// looking dangerous. +// +// The C0 control is the one that matters. Both handlers spell U+0001 +// as a six-byte escape for the single byte it costs a client to send, +// which is the widest multiplier available in the basic multilingual +// plane and the case a raw-byte budget breaks on first. The astral +// non-printable costs ten under the text handler, four more than the +// JSON handler ever spends. +func fills() []struct { + name string + fill string +} { + return []struct { + name string + fill string + }{ + {"plain", "x"}, + {"quote", `"`}, + {"backslash", `\`}, + {"tab", "\t"}, + {"newline", "\n"}, + {"c0_control", "\x01"}, + {"astral_nonprintable", "\U0001000C"}, + } +} + +// clientValue builds a value of at least fillBytes raw bytes out of +// fill, ending in tailMarker. +func clientValue(fill string) string { + var b strings.Builder + + for b.Len() < fillBytes { + b.WriteString(fill) + } + + b.WriteString(tailMarker) + + return b.String() +} + +// handlers are the two slog handlers internal/logger can install. The +// ceiling is quoted to operators unqualified, so every case is +// asserted under both. +func handlers() []struct { + name string + make func(*bytes.Buffer) slog.Handler +} { + opts := &slog.HandlerOptions{Level: slog.LevelDebug} + + return []struct { + name string + make func(*bytes.Buffer) slog.Handler + }{ + {"json", func(b *bytes.Buffer) slog.Handler { + return slog.NewJSONHandler(b, opts) + }}, + {"text", func(b *bytes.Buffer) slog.Handler { + return slog.NewTextHandler(b, opts) + }}, + } +} + +type thing struct { + ID string `gorm:"primaryKey"` + Name string +} + +// neverSlow is a slow-statement threshold no statement in this file +// can reach. Cases that are about a non-slow arm of Trace set it, so +// that a machine under load cannot turn a miss into a slow report and +// decide the outcome for them. +const neverSlow = time.Hour + +// alwaysSlow makes every statement count as slow, so the slow arm is +// reached without the test waiting for it. +const alwaysSlow = time.Nanosecond + +// openDB opens a real SQLite database behind the adapter under test, +// so every assertion below is made against SQL that GORM actually +// rendered rather than against a string a test wrote by hand. slow is +// the adapter's slow-statement threshold. +func openDB( + t *testing.T, buf *bytes.Buffer, h slog.Handler, slow time.Duration, +) *gorm.DB { + t.Helper() + + sqlDB, err := sql.Open("sqlite", fmt.Sprintf( + "file:%s?mode=rwc", + filepath.Join(t.TempDir(), "gormlog.db"), + )) + require.NoError(t, err) + + t.Cleanup(func() { _ = sqlDB.Close() }) + + gl := gormlog.ExportNewWithSlowThreshold(slog.New(h), slow) + + gdb, err := gorm.Open( + sqlite.Dialector{Conn: sqlDB}, + &gorm.Config{Logger: gl}, + ) + require.NoError(t, err) + + require.NoError(t, gdb.AutoMigrate(&thing{})) + + // Migration chatter is not what any of these cases is about. + buf.Reset() + + return gdb +} + +// assertBounded holds every line the adapter wrote to the stated +// ceiling and proves each was cut rather than merely short. +func assertBounded(t *testing.T, out string) { + t.Helper() + + assert.NotContains( + t, out, tailMarker, + "the far end of the client value reached the log, so "+ + "nothing truncated it", + ) + + for line := range strings.SplitSeq( + strings.TrimRight(out, "\n"), "\n", + ) { + if line == "" { + continue + } + + assert.LessOrEqual( + t, len(line), middleware.MaxAccessLogLineBytes, + "log line exceeded its bound: %s", + line[:min(len(line), 300)], + ) + } +} + +// TestRecordNotFound_WritesNothing is the defect itself. GORM's own +// default logger prints the fully interpolated SELECT on every +// ErrRecordNotFound, and on this service's two unauthenticated +// lookups the interpolated parameter is whatever the client sent. +func TestRecordNotFound_WritesNothing(t *testing.T) { + t.Parallel() + + for _, h := range handlers() { + for _, f := range fills() { + t.Run(h.name+"/"+f.name, func(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + + gdb := openDB(t, &buf, h.make(&buf), neverSlow) + + var got thing + + err := gdb.Where( + "id = ?", clientValue(f.fill), + ).First(&got).Error + require.ErrorIs(t, err, gorm.ErrRecordNotFound) + + assert.Empty( + t, buf.String(), + "a miss on a client-chosen key must not "+ + "write a log line", + ) + }) + } + } +} + +// TestSlowRecordNotFound_IsStillReportedSlow pins the arm ordering in +// Trace against the drop above. +// +// GORM's own Trace orders its cases error-that-is-not-a-miss, then +// slow, then routine, so IgnoreRecordNotFoundError: true — the cheap +// option this adapter was chosen over — still reports a miss that ran +// slow. An adapter that dropped the miss first would be strictly less +// observant than the option it replaced, on exactly the two lookups +// this package exists for. A miss is also the statement most likely to +// be slow, since it is the one that scans without finding a row. +func TestSlowRecordNotFound_IsStillReportedSlow(t *testing.T) { + t.Parallel() + + for _, h := range handlers() { + for _, f := range fills() { + t.Run(h.name+"/"+f.name, func(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + + gdb := openDB(t, &buf, h.make(&buf), alwaysSlow) + + var got thing + + err := gdb.Where( + "id = ?", clientValue(f.fill), + ).First(&got).Error + require.ErrorIs(t, err, gorm.ErrRecordNotFound) + + assert.Contains( + t, buf.String(), slowLine, + "a slow statement that missed was not "+ + "reported as slow", + ) + assertBounded(t, buf.String()) + }) + } + } +} + +// TestRecordNotFoundFlood_DoesNotGrowWithInput states the definition +// of done directly: a flood of misses at two input sizes 64 times +// apart must cost the same number of bytes of log. +func TestRecordNotFoundFlood_DoesNotGrowWithInput(t *testing.T) { + t.Parallel() + + const requests = 50 + + flood := func(t *testing.T, size int) int { + t.Helper() + + var buf bytes.Buffer + + gdb := openDB( + t, &buf, + slog.NewJSONHandler(&buf, &slog.HandlerOptions{ + Level: slog.LevelDebug, + }), + neverSlow, + ) + + value := strings.Repeat("\x01", size) + + for range requests { + var got thing + + _ = gdb.Where("id = ?", value).First(&got).Error + } + + return buf.Len() + } + + small := flood(t, 128) + big := flood(t, 128*64) + + assert.Equal( + t, small, big, + "log volume tracked the size of the client's input", + ) +} + +// TestStatementError_LineIsBounded covers the branch that does log. +// A driver error is not ErrRecordNotFound, so the statement is +// written, and the driver's own error text can quote what the client +// supplied. The statement's parameters are no longer part of that — +// see TestBoundValues_NeverReachTheLog — but the budget is what holds +// the line when the statement itself, or the error, is the long part. +func TestStatementError_LineIsBounded(t *testing.T) { + t.Parallel() + + for _, h := range handlers() { + for _, f := range fills() { + t.Run(h.name+"/"+f.name, func(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + + gdb := openDB(t, &buf, h.make(&buf), neverSlow) + + row := thing{ID: clientValue(f.fill), Name: "a"} + + require.NoError(t, gdb.Create(&row).Error) + + buf.Reset() + + // The same primary key a second time: a UNIQUE + // constraint failure, which is an error GORM logs. + err := gdb.Create(&thing{ + ID: row.ID, Name: "b", + }).Error + require.Error(t, err) + + assert.Contains( + t, buf.String(), errorLine, + ) + assertBounded(t, buf.String()) + }) + } + } +} + +// TestSucceedingStatement_LineIsBoundedOnEitherArm covers the two +// arms a statement that returns no error can take, over the same +// query, so neither can be bounded by accident of the other. +// +// - slow. Silencing GORM outright would have been the cheaper fix +// and would have cost this report, which is the one thing GORM's +// logger gave an operator that nothing else in this service does. +// - routine. The branch an operator reaches by turning the level +// down to DEBUG: every statement is reported, so every statement +// has to be bounded too. +func TestSucceedingStatement_LineIsBoundedOnEitherArm(t *testing.T) { + t.Parallel() + + // routineLine is a substring of slowLine, so the routine arm + // carries notWant as well: Contains alone cannot tell the two arms + // apart in that direction. + arms := []struct { + name string + slow time.Duration + want string + notWant string + }{ + {"slow", alwaysSlow, slowLine, ""}, + {"routine", neverSlow, routineLine, slowLine}, + } + + for _, a := range arms { + for _, h := range handlers() { + for _, f := range fills() { + name := a.name + "/" + h.name + "/" + f.name + + t.Run(name, func(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + + gdb := openDB(t, &buf, h.make(&buf), a.slow) + + var got []thing + + require.NoError(t, gdb.Where( + "name = ?", clientValue(f.fill), + ).Find(&got).Error) + + assert.Contains(t, buf.String(), a.want) + + if a.notWant != "" { + assert.NotContains( + t, buf.String(), a.notWant, + ) + } + + assertBounded(t, buf.String()) + }) + } + } + } +} + +// TestGORMOwnMessages_AreBounded covers the three printf-style +// entry points. GORM builds these itself, but nothing stops one of +// them quoting a value the statement carried. +func TestGORMOwnMessages_AreBounded(t *testing.T) { + t.Parallel() + + for _, h := range handlers() { + for _, f := range fills() { + t.Run(h.name+"/"+f.name, func(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + + gl := gormlog.New(slog.New(h.make(&buf))) + ctx := context.Background() + value := clientValue(f.fill) + + gl.Info(ctx, "%s", value) + gl.Warn(ctx, "%s", value) + gl.Error(ctx, "%s", value) + + // The no-argument form, which is how GORM reports + // most of its own conditions. Reached through a + // function value so the vet printf check does not + // read the message as a format string — which is + // also why the adapter does not. + noArgs := func( + f func(context.Context, string, ...any), + msg string, + ) { + f(ctx, msg) + } + noArgs(gl.Info, value) + + assertBounded(t, buf.String()) + }) + } + } +} + +// TestLogMode_KeepsTheOperatorsLevel records that GORM's own level +// knob is deliberately inert: level belongs to LOG_LEVEL, and a +// second one inside the database layer could only disagree with it. +func TestLogMode_KeepsTheOperatorsLevel(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + + gl := gormlog.New(slog.New(slog.NewJSONHandler( + &buf, &slog.HandlerOptions{Level: slog.LevelDebug}, + ))) + + assert.Same(t, gl, gl.LogMode(0)) +} diff --git a/internal/gormlog/scan_guard_test.go b/internal/gormlog/scan_guard_test.go new file mode 100644 index 0000000..13a1de5 --- /dev/null +++ b/internal/gormlog/scan_guard_test.go @@ -0,0 +1,260 @@ +package gormlog_test + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// minNonTestFiles guards the walk below against passing because it +// found nothing to look at. The tree held 60 non-test .go files when +// this was written. +const minNonTestFiles = 40 + +// isRowProducer reports whether name is a method that returns a +// database/sql row handle. GORM's Row and Rows return *sql.Row and +// *sql.Rows, so Scan on the result of one of them is database/sql's +// Scan and never (*gorm.DB).Scan. +func isRowProducer(name string) bool { + switch name { + case "Row", "Rows", "QueryRow", "QueryRowContext": + return true + default: + return false + } +} + +// receiverIsRowHandle reports whether x is syntactically a call to a +// row producer, which is the only receiver form this check accepts for +// a Scan. +func receiverIsRowHandle(x ast.Expr) bool { + call, ok := x.(*ast.CallExpr) + if !ok { + return false + } + + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return false + } + + return isRowProducer(sel.Sel.Name) +} + +// unguardedScans returns the position of every Scan call in file whose +// receiver is not a row handle. It fails closed: a receiver it cannot +// resolve syntactically — a local variable, a struct field — is +// reported rather than assumed safe. +func unguardedScans( + fset *token.FileSet, file *ast.File, +) []token.Position { + var found []token.Position + + ast.Inspect(file, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok || sel.Sel.Name != "Scan" { + return true + } + + if !receiverIsRowHandle(sel.X) { + found = append(found, fset.Position(sel.Sel.Pos())) + } + + return true + }) + + return found +} + +// moduleRoot walks up from the working directory to the directory +// holding go.mod. +func moduleRoot(t *testing.T) string { + t.Helper() + + dir, err := os.Getwd() + require.NoError(t, err) + + for { + _, statErr := os.Stat(filepath.Join(dir, "go.mod")) + if statErr == nil { + return dir + } + + parent := filepath.Dir(dir) + require.NotEqual(t, parent, dir, "no go.mod above %s", dir) + + dir = parent + } +} + +// skipDir reports whether a directory holds no source this check +// governs. +func skipDir(name string) bool { + switch name { + case ".git", "bin", "node_modules", "testdata": + return true + default: + return false + } +} + +// walkNonTestGo parses every non-test .go file under root and returns +// how many it parsed along with every unguarded Scan it found. +func walkNonTestGo(t *testing.T, root string) (int, []string) { + t.Helper() + + var ( + parsed int + hits []string + ) + + fset := token.NewFileSet() + + require.NoError(t, filepath.WalkDir( + root, + func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if d.IsDir() { + if skipDir(d.Name()) { + return fs.SkipDir + } + + return nil + } + + if !isNonTestGo(d.Name()) { + return nil + } + + file, err := parser.ParseFile(fset, path, nil, 0) + if err != nil { + return err + } + + parsed++ + + for _, pos := range unguardedScans(fset, file) { + hits = append(hits, relPosition(root, pos)) + } + + return nil + }, + )) + + return parsed, hits +} + +// isNonTestGo reports whether a file name is Go source this check +// governs. +func isNonTestGo(name string) bool { + return strings.HasSuffix(name, ".go") && + !strings.HasSuffix(name, "_test.go") +} + +// relPosition renders pos with its path relative to root, so a failure +// names the file the way the repository does. +func relPosition(root string, pos token.Position) string { + name := pos.Filename + + rel, err := filepath.Rel(root, name) + if err == nil { + name = rel + } + + return fmt.Sprintf("%s:%d:%d", name, pos.Line, pos.Column) +} + +// TestGormScanIsNeverCalledOutsideTests keeps (*gorm.DB).Scan out of +// non-test code. +// +// It is the one statement path (*Logger).ParamsFilter does not reach: +// Scan swaps GORM's own trace recorder in for the adapter, and that +// recorder does not implement gorm.ParamsFilter, so the statement is +// logged with its values interpolated. The package comment states the +// limit; this fails when someone adds a call site anyway. +// +// The current tree has one caller, internal/database/database_test.go, +// which this check does not govern: it is test-only and its SELECT 1 +// binds nothing. +func TestGormScanIsNeverCalledOutsideTests(t *testing.T) { + t.Parallel() + + parsed, offenders := walkNonTestGo(t, moduleRoot(t)) + + require.GreaterOrEqual( + t, parsed, minNonTestFiles, + "parsed %d non-test .go files, so this check found "+ + "nothing to look at", parsed, + ) + require.Empty( + t, offenders, + "Scan called on a receiver this check cannot show is a "+ + "database/sql row handle. (*gorm.DB).Scan logs the "+ + "statement with its bound values interpolated — use "+ + "Find, Pluck, or Raw(...).Row().Scan instead. A "+ + "database/sql Scan reached through a variable is "+ + "reported too; write it as ().Scan rather "+ + "than widening this check.", + ) +} + +// scanGuardCase is one planted snippet and whether the check above +// should report it. +type scanGuardCase struct { + name string + body string + want int +} + +func scanGuardCases() []scanGuardCase { + return []scanGuardCase{ + {"gorm chain", `db.DB().Raw("SELECT 1").Scan(&v)`, 1}, + {"gorm receiver", `gdb.Scan(&v)`, 1}, + {"gorm via variable", "q := gdb.Raw(\"x\")\nq.Scan(&v)", 1}, + {"gorm model chain", `gdb.Model(&x).Scan(&v)`, 1}, + {"sql row", `gdb.Raw("SELECT 1").Row().Scan(&v)`, 0}, + {"sql rows", `gdb.Raw("SELECT 1").Rows().Scan(&v)`, 0}, + {"unrelated call", `gdb.Find(&v)`, 0}, + } +} + +// TestScanGuard_ReportsPlantedCalls proves the check fires. Without it +// a detector that matched nothing would satisfy the walk above no +// matter what the tree contained. +func TestScanGuard_ReportsPlantedCalls(t *testing.T) { + t.Parallel() + + for _, tc := range scanGuardCases() { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + fset := token.NewFileSet() + src := fmt.Sprintf( + "package p\n\nfunc f() {\n\t%s\n}\n", tc.body, + ) + + file, err := parser.ParseFile( + fset, tc.name+".go", src, 0, + ) + require.NoError(t, err) + + require.Len(t, unguardedScans(fset, file), tc.want) + }) + } +} diff --git a/internal/gormlog/values_test.go b/internal/gormlog/values_test.go new file mode 100644 index 0000000..8d05a50 --- /dev/null +++ b/internal/gormlog/values_test.go @@ -0,0 +1,185 @@ +package gormlog_test + +import ( + "bytes" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +// secretValue is bound as a parameter by every case below. Nothing +// else in this package writes it, so finding it in captured output +// means a bound value was rendered into the log. +const secretValue = "QQBOUNDVALUEMARKERQQ" + +// The three messages Trace emits under, one per arm. +// +// routineLine is a substring of slowLine, so a case that wants the +// routine arm has to rule the slow one out as well rather than rely on +// Contains alone. +const ( + routineLine = "sql statement" + slowLine = "slow sql statement" + errorLine = "sql statement failed" +) + +// boundValueCase is one arm of Trace, driven by a statement that binds +// secretValue. +type boundValueCase struct { + name string + slow time.Duration + want string + drive func(t *testing.T, gdb *gorm.DB) +} + +// insertSecret returns a driver that inserts one row whose Name is the +// secret. +func insertSecret(id string) func(*testing.T, *gorm.DB) { + return func(t *testing.T, gdb *gorm.DB) { + t.Helper() + + require.NoError(t, gdb.Create(&thing{ + ID: id, Name: secretValue, + }).Error) + } +} + +// insertSecretTwice drives the error arm: the same primary key a +// second time is a UNIQUE constraint failure, which is an error GORM +// logs with the statement. +func insertSecretTwice(t *testing.T, gdb *gorm.DB) { + t.Helper() + + require.NoError(t, gdb.Create(&thing{ + ID: secretValue, Name: secretValue, + }).Error) + require.Error(t, gdb.Create(&thing{ + ID: secretValue, Name: "other", + }).Error) +} + +// selectSecret drives a query whose WHERE clause binds the secret, +// covering the read side as well as the write side. +func selectSecret(t *testing.T, gdb *gorm.DB) { + t.Helper() + + var got []thing + + require.NoError( + t, gdb.Where("name = ?", secretValue).Find(&got).Error, + ) +} + +func boundValueCases() []boundValueCase { + return []boundValueCase{ + { + name: "routine", slow: neverSlow, + want: routineLine, drive: insertSecret("routine"), + }, + { + name: "slow", slow: alwaysSlow, + want: slowLine, drive: insertSecret("slow"), + }, + { + name: "error", slow: neverSlow, + want: errorLine, drive: insertSecretTwice, + }, + { + name: "select", slow: neverSlow, + want: routineLine, drive: selectSecret, + }, + } +} + +// TestBoundValues_NeverReachTheLog states the values-off property +// directly, on each arm of Trace that emits. +// +// Truncation is not what is being asserted. A bounded secret is still +// a secret: the session key is 44 base64 characters and an Argon2id +// hash under 100, so both fit inside every budget this package +// applies. What keeps them out is that the adapter logs the +// statement's shape and discards its parameters — see +// (*Logger).ParamsFilter — and that has to hold at DEBUG as much as on +// an error, because DEBUG is the level at which a successful INSERT is +// written at all. +// +// Each case also requires a placeholder in the logged statement. +// Without that, the absence of the value would be satisfied by a +// logger that wrote nothing useful. +func TestBoundValues_NeverReachTheLog(t *testing.T) { + t.Parallel() + + for _, tc := range boundValueCases() { + for _, h := range handlers() { + t.Run(tc.name+"/"+h.name, func(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + + gdb := openDB(t, &buf, h.make(&buf), tc.slow) + + tc.drive(t, gdb) + + assertNoBoundValue(t, buf.String(), tc.want) + }) + } + } +} + +// assertNoBoundValue holds one captured arm to the property: it wrote +// the line it was supposed to write, that line kept its placeholders, +// and it carried no bound value. +func assertNoBoundValue(t *testing.T, out, want string) { + t.Helper() + + require.Contains( + t, out, want, + "the arm under test wrote nothing, so the assertions "+ + "below are vacuous", + ) + assert.NotContains( + t, out, secretValue, + "a bound parameter was rendered into the log", + ) + assert.Contains( + t, out, "?", + "the statement was logged without its placeholders", + ) +} + +// TestInsert_KeepsOnePlaceholderPerBoundValue pins the shape of the +// INSERT specifically, since that is the statement that carries both +// first-boot secrets. A statement that dropped one value and kept the +// other would satisfy the assertions above. +func TestInsert_KeepsOnePlaceholderPerBoundValue(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + + gdb := openDB(t, &buf, handlers()[0].make(&buf), neverSlow) + + require.NoError(t, gdb.Create(&thing{ + ID: "m", Name: secretValue, + }).Error) + + out := buf.String() + + require.Contains(t, out, "INSERT INTO") + assert.NotContains(t, out, secretValue) + + for line := range strings.SplitSeq(out, "\n") { + if !strings.Contains(line, "INSERT INTO") { + continue + } + + assert.GreaterOrEqual( + t, strings.Count(line, "?"), 2, + "insert logged fewer placeholders than it bound "+ + "values: %s", line, + ) + } +} diff --git a/internal/handlers/auth.go b/internal/handlers/auth.go index b934280..39fa5dc 100644 --- a/internal/handlers/auth.go +++ b/internal/handlers/auth.go @@ -2,8 +2,10 @@ package handlers import ( "net/http" + "strconv" "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/logfield" ) // HandleLoginPage returns a handler for the login page (GET) @@ -29,10 +31,8 @@ func (h *Handlers) HandleLoginPage() http.HandlerFunc { // HandleLoginSubmit handles the login form submission (POST) func (h *Handlers) HandleLoginSubmit() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - // Limit request body to prevent memory exhaustion - r.Body = http.MaxBytesReader(w, r.Body, 1<= 0 { + next.AddCookie(c) + } + } + + s, err := sess.Get(next) + require.NoError(t, err) + assert.True( + t, sess.IsAuthenticated(s), + "the issued cookie must carry an authenticated session", + ) +} diff --git a/internal/handlers/delivery_replay.go b/internal/handlers/delivery_replay.go new file mode 100644 index 0000000..ee9425b --- /dev/null +++ b/internal/handlers/delivery_replay.go @@ -0,0 +1,378 @@ +package handlers + +import ( + "net/http" + "strconv" + + "github.com/go-chi/chi" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" +) + +// replayOutcomeParam is the query parameter the replay POST redirects +// with and the event log page reads its banner from. +const replayOutcomeParam = "replay" + +// replayOutcomeCode is the outcome of a replay POST. The redirect +// carries one of these fixed codes rather than a message, so nothing a +// client submits can reach the rendered page through it. +type replayOutcomeCode string + +const ( + // replayQueued reports that a new delivery was created and handed + // to the delivery engine. + replayQueued replayOutcomeCode = "queued" + + // replayTargetDeleted reports a target that once existed and has + // since been deleted. Deletes are soft and deliveries carry no + // foreign key to the target row, so the history survives its + // target and this is the ordinary case for an old event. + replayTargetDeleted replayOutcomeCode = "target-deleted" + + // replayTargetMissing reports a target id that names no row at + // all, deleted or otherwise. + replayTargetMissing replayOutcomeCode = "target-missing" + + // replayTargetInactive reports a target the operator has + // deactivated. A deactivated target receives no new deliveries, so + // a replay to it would be a delivery they switched off. + replayTargetInactive replayOutcomeCode = "target-inactive" + + // replayNotTerminal reports a delivery the engine has not finished + // with. + replayNotTerminal replayOutcomeCode = "not-terminal" + + // replayInFlight reports that an earlier replay of this event to + // this target is still running. + replayInFlight replayOutcomeCode = "in-flight" +) + +// replayOutcome returns the banner the event log page shows for an +// outcome code, and whether the replay was queued. An unrecognised +// code yields no banner. +func replayOutcome(code string) (string, bool) { + switch replayOutcomeCode(code) { + case replayQueued: + return "Replay queued: a new delivery was created against " + + "the target's current configuration.", true + case replayTargetDeleted: + return "Not replayed: the target this delivery was for has " + + "been deleted. Recreate the target, then replay.", false + case replayTargetMissing: + return "Not replayed: the target this delivery was for no " + + "longer exists.", false + case replayTargetInactive: + return "Not replayed: the target this delivery was for is " + + "deactivated. Activate it, then replay.", false + case replayNotTerminal: + return "Not replayed: this delivery has not finished yet.", + false + case replayInFlight: + return "Not replayed: a delivery of this event to this " + + "target is already in flight.", false + default: + return "", false + } +} + +// HandleDeliveryReplay re-sends a finished delivery's event to its +// target. +// +// A replay never touches the delivery it repeats. It creates a NEW +// pending delivery row for the same event and target and hands it to +// the delivery engine through the same Notifier the receiver uses, so +// the original's status, attempts and timestamps stand as the record +// of what actually happened, and the replay is retried, SSRF-guarded +// and circuit-broken exactly as a first attempt is. +// +// What is re-sent is the stored EVENT body, never the response the +// original delivery received. +// +// The target's configuration is read now rather than as it stood when +// the original ran: a replay exists to deliver where the operator +// currently wants the event to go. That is also why a deleted target +// is refused rather than delivered to from stale configuration. +func (h *Handlers) HandleDeliveryReplay() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + webhook, ok := h.ownedWebhook(w, r) + if !ok { + return + } + + // The body size cap is enforced by the MaxBodySize + // middleware, which runs before CSRF parses the form. + err := r.ParseForm() + if err != nil { + http.Error( + w, "Bad request", http.StatusBadRequest, + ) + + return + } + + h.replayDelivery(w, r, webhook) + } +} + +// replayDelivery performs the replay for a webhook the caller has +// already established the session's user owns. +func (h *Handlers) replayDelivery( + w http.ResponseWriter, + r *http.Request, + webhook database.Webhook, +) { + if !h.dbMgr.DBExists(webhook.ID) { + http.NotFound(w, r) + + return + } + + webhookDB, err := h.dbMgr.GetDB(webhook.ID) + if err != nil { + h.serverError(w, "failed to get webhook database", err) + + return + } + + original, ok := h.loadReplaySource(w, r, webhookDB) + if !ok { + return + } + + if !original.Status.Terminal() { + h.finishReplay(w, r, webhook, replayNotTerminal) + + return + } + + target, code := h.replayTarget(webhook.ID, original.TargetID) + if target == nil { + h.finishReplay(w, r, webhook, code) + + return + } + + h.queueReplay(w, r, webhookDB, webhook, original, target) +} + +// loadReplaySource loads the delivery to be replayed, selecting only +// the columns the replay needs so no association is populated. A +// delivery id that names no row in this webhook's database is a 404. +func (h *Handlers) loadReplaySource( + w http.ResponseWriter, + r *http.Request, + webhookDB *gorm.DB, +) (*database.Delivery, bool) { + var original database.Delivery + + err := webhookDB. + Select("id", "event_id", "target_id", "status"). + First( + &original, "id = ?", chi.URLParam(r, "deliveryID"), + ).Error + if err != nil { + http.NotFound(w, r) + + return nil, false + } + + return &original, true +} + +// queueReplay writes the new delivery and hands it to the engine. +func (h *Handlers) queueReplay( + w http.ResponseWriter, + r *http.Request, + webhookDB *gorm.DB, + webhook database.Webhook, + original *database.Delivery, + target *database.Target, +) { + inFlight, err := countInFlightDeliveries( + webhookDB, original.EventID, target.ID, + ) + if err != nil { + h.serverError( + w, "failed to count in-flight deliveries", err, + ) + + return + } + + if inFlight > 0 { + h.finishReplay(w, r, webhook, replayInFlight) + + return + } + + var event database.Event + + err = webhookDB. + First(&event, "id = ?", original.EventID).Error + if err != nil { + h.serverError(w, "failed to load event for replay", err) + + return + } + + task, err := createReplayDelivery( + webhookDB, webhook.ID, &event, target, + ) + if err != nil { + h.serverError( + w, "failed to create replay delivery", err, + ) + + return + } + + h.mtr.DeliveryReplayed(target.Type) + h.notifier.Notify([]delivery.Task{task}) + + h.log.Info( + "delivery replay queued", + "webhook_id", webhook.ID, + "event_id", event.ID, + "target_id", target.ID, + "replayed_delivery_id", original.ID, + "delivery_id", task.DeliveryID, + ) + + h.finishReplay(w, r, webhook, replayQueued) +} + +// replayTarget loads the delivery's target as it stands now. +// +// The load is Unscoped so that a soft-deleted row is still found: +// deletes are soft and a delivery carries no foreign key to its +// target, so a target's history outlives it, and without the deleted +// row there is no way to tell "you deleted this target" from "this id +// never named anything". A nil target means the replay is refused, +// with the returned code saying why. +func (h *Handlers) replayTarget( + webhookID, targetID string, +) (*database.Target, replayOutcomeCode) { + var target database.Target + + err := h.db.DB().Unscoped().Where( + "id = ? AND webhook_id = ?", targetID, webhookID, + ).First(&target).Error + if err != nil { + return nil, replayTargetMissing + } + + if target.DeletedAt.Valid { + return nil, replayTargetDeleted + } + + if !target.Active { + return nil, replayTargetInactive + } + + return &target, replayQueued +} + +// countInFlightDeliveries reports how many deliveries of this event to +// this target the engine has not finished. +// +// It is the replay-storm guard: a replay is refused while an earlier +// one is still pending or retrying, so a held-down button or a scripted +// loop cannot stack copies of work already queued. It is a check and +// not a lock, so two simultaneous POSTs can still both pass it; the +// per-client rate limit on the route is what bounds that. +func countInFlightDeliveries( + webhookDB *gorm.DB, eventID, targetID string, +) (int64, error) { + var count int64 + + err := webhookDB.Model(&database.Delivery{}).Where( + "event_id = ? AND target_id = ? AND status IN ?", + eventID, targetID, + []database.DeliveryStatus{ + database.DeliveryStatusPending, + database.DeliveryStatusRetrying, + }, + ).Count(&count).Error + + return count, err +} + +// createReplayDelivery writes the new pending delivery row and returns +// the task that carries it to the delivery engine. +// +// The row is written with associations omitted, and neither Event nor +// Target is populated on it: GORM's SaveBeforeAssociations would +// otherwise upsert the whole target row — plaintext config, which for a +// Slack target is the credential — into the per-webhook event database. +// See https://git.eeqj.de/sneak/webhooker/issues/206. +func createReplayDelivery( + webhookDB *gorm.DB, + webhookID string, + event *database.Event, + target *database.Target, +) (delivery.Task, error) { + dlv := &database.Delivery{ + EventID: event.ID, + TargetID: target.ID, + Status: database.DeliveryStatusPending, + } + + err := webhookDB.Omit(clause.Associations).Create(dlv).Error + if err != nil { + return delivery.Task{}, err + } + + return delivery.Task{ + DeliveryID: dlv.ID, + EventID: event.ID, + WebhookID: webhookID, + EntrypointID: event.EntrypointID, + TargetID: target.ID, + TargetName: target.Name, + TargetType: target.Type, + TargetConfig: target.Config, + MaxRetries: target.MaxRetries, + Method: event.Method, + Headers: event.Headers, + ContentType: event.ContentType, + Body: replayBody(event.Body), + AttemptNum: 1, + }, nil +} + +// replayBody returns the stored event body for a replay task to carry +// inline, or nil when it is large enough that the engine should fetch +// it from the per-webhook database instead. +func replayBody(body string) *string { + if len(body) >= delivery.MaxInlineBodySize { + return nil + } + + return &body +} + +// finishReplay redirects back to the event log the replay was +// triggered from, carrying the outcome code the page turns into a +// banner and the page number the form submitted. +func (h *Handlers) finishReplay( + w http.ResponseWriter, + r *http.Request, + webhook database.Webhook, + code replayOutcomeCode, +) { + dest := "/source/" + webhook.ID + "/logs?" + + replayOutcomeParam + "=" + string(code) + + // The page is read from the form rather than the query string: + // this is a POST, and its query string is what logs and Referer + // headers record. + if page := pageOrFirst( + r.PostFormValue("page"), + ); page > 1 { + dest += "&page=" + strconv.Itoa(page) + } + + http.Redirect(w, r, dest, http.StatusSeeOther) +} diff --git a/internal/handlers/delivery_replay_test.go b/internal/handlers/delivery_replay_test.go new file mode 100644 index 0000000..4d8a3be --- /dev/null +++ b/internal/handlers/delivery_replay_test.go @@ -0,0 +1,526 @@ +package handlers_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" + "sneak.berlin/go/webhooker/internal/handlers" + "sneak.berlin/go/webhooker/internal/session" +) + +// paramDeliveryID is the chi URL parameter name the replay handler +// reads. +const paramDeliveryID = "deliveryID" + +// replayTargetURL is a public destination, so a target configured with +// it is one the SSRF guard would accept. Nothing in these tests +// dispatches to it: the notifier is recorded, not run. +const replayTargetURL = "http://93.184.216.34/hook" + +// seedFailedDelivery records an event, a terminally failed delivery of +// it to the given target, and the attempt that failed. +func seedFailedDelivery( + t *testing.T, + dbMgr *database.WebhookDBManager, + webhookID, targetID string, +) (*database.Event, *database.Delivery) { + t.Helper() + + webhookDB, err := dbMgr.GetDB(webhookID) + require.NoError(t, err) + + event := &database.Event{ + WebhookID: webhookID, + EntrypointID: "entrypoint-" + webhookID, + Method: http.MethodPost, + Headers: `{"X-Test":["yes"]}`, + Body: `{"replay":"me"}`, + ContentType: contentTypeJSON, + } + + require.NoError(t, webhookDB.Omit( + clause.Associations, + ).Create(event).Error) + + dlv := &database.Delivery{ + EventID: event.ID, + TargetID: targetID, + Status: database.DeliveryStatusFailed, + } + + require.NoError(t, webhookDB.Omit( + clause.Associations, + ).Create(dlv).Error) + + result := &database.DeliveryResult{ + DeliveryID: dlv.ID, + AttemptNum: 1, + Success: false, + StatusCode: http.StatusBadGateway, + Error: "connection refused", + } + + require.NoError(t, webhookDB.Omit( + clause.Associations, + ).Create(result).Error) + + return event, dlv +} + +// loadDelivery reads a delivery back out of a webhook's database. +func loadDelivery( + t *testing.T, webhookDB *gorm.DB, deliveryID string, +) database.Delivery { + t.Helper() + + var dlv database.Delivery + + require.NoError( + t, + webhookDB.First(&dlv, "id = ?", deliveryID).Error, + ) + + return dlv +} + +// listDeliveries reads every delivery of an event. +func listDeliveries( + t *testing.T, webhookDB *gorm.DB, eventID string, +) []database.Delivery { + t.Helper() + + var deliveries []database.Delivery + + require.NoError(t, webhookDB.Where( + "event_id = ?", eventID, + ).Find(&deliveries).Error) + + return deliveries +} + +// theOtherDelivery returns the one delivery in the slice that is not +// excludeID. Identity is used rather than an ordering because the rows +// are minted milliseconds apart and their ids are random. +func theOtherDelivery( + t *testing.T, + deliveries []database.Delivery, + excludeID string, +) database.Delivery { + t.Helper() + + var found []database.Delivery + + for _, d := range deliveries { + if d.ID != excludeID { + found = append(found, d) + } + } + + require.Len(t, found, 1) + + return found[0] +} + +// postReplay runs the real replay handler for one delivery. +func postReplay( + t *testing.T, + h *handlers.Handlers, + sess *session.Session, + webhookID, deliveryID string, +) *httptest.ResponseRecorder { + t.Helper() + + req := postRequest( + "/source/"+webhookID+"/deliveries/"+ + deliveryID+"/replay", + authenticatedCookies( + t, sess, deleteTestUserID, deleteTestUsername, + ), + map[string]string{ + paramSourceID: webhookID, + paramDeliveryID: deliveryID, + }, + ) + + w := httptest.NewRecorder() + h.HandleDeliveryReplay().ServeHTTP(w, req) + + return w +} + +// TestHandleDeliveryReplay_AppendsDeliveryAndLeavesOriginal is the +// core requirement: replaying a failed delivery succeeds, appends a +// new delivery, and leaves the original row and its recorded attempt +// exactly as they were. +// +// It also pins the two things a replay would be wrong to get from the +// original: the task carries the target's CURRENT configuration, which +// this test changes between the failure and the replay, and it carries +// the stored EVENT body rather than anything the failed attempt +// received back. +func TestHandleDeliveryReplay_AppendsDeliveryAndLeavesOriginal( + t *testing.T, +) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + notif *recordingNotifier + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr, ¬if) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + tgt := seedConfiguredTarget( + t, db, wh.ID, database.TargetTypeHTTP, + `{"url":"`+replayTargetURL+`"}`, + ) + + event, original := seedFailedDelivery( + t, dbMgr, wh.ID, tgt.ID, + ) + + webhookDB, err := dbMgr.GetDB(wh.ID) + require.NoError(t, err) + + before := loadDelivery(t, webhookDB, original.ID) + + // The operator fixes the destination, which is the whole reason + // to replay. The replay must use this, not the config the + // original delivery ran against. + const fixedConfig = `{"url":"http://93.184.216.34/fixed"}` + + require.NoError(t, db.DB().Model(&database.Target{}). + Where("id = ?", tgt.ID). + Update("config", fixedConfig).Error) + + w := postReplay(t, h, sess, wh.ID, original.ID) + + require.Equal(t, http.StatusSeeOther, w.Code) + assert.Equal( + t, + "/source/"+wh.ID+"/logs?replay=queued", + w.Header().Get("Location"), + ) + + deliveries := listDeliveries(t, webhookDB, event.ID) + require.Len( + t, deliveries, 2, + "replay must append a delivery, not reuse one", + ) + + replayed := theOtherDelivery(t, deliveries, original.ID) + assert.Equal(t, tgt.ID, replayed.TargetID) + assert.Equal(t, event.ID, replayed.EventID) + assert.Equal( + t, database.DeliveryStatusPending, replayed.Status, + ) + + assertDeliveryUntouched(t, webhookDB, before) + + tasks := notif.Tasks() + require.Len(t, tasks, 1) + assertReplayTask( + t, tasks[0], wh.ID, event, tgt, replayed.ID, fixedConfig, + ) + assertNoLeakedTarget(t, webhookDB) +} + +// assertDeliveryUntouched proves a delivery row is exactly as it was +// read before: same terminal status, same timestamps, and the same +// recorded attempts. +func assertDeliveryUntouched( + t *testing.T, + webhookDB *gorm.DB, + before database.Delivery, +) { + t.Helper() + + after := loadDelivery(t, webhookDB, before.ID) + assert.Equal( + t, before.Status, after.Status, + "replay must not resurrect the original delivery", + ) + assert.Equal(t, before.UpdatedAt, after.UpdatedAt) + assert.Equal(t, before.CreatedAt, after.CreatedAt) + + var attempts int64 + + require.NoError(t, webhookDB. + Model(&database.DeliveryResult{}). + Where("delivery_id = ?", before.ID). + Count(&attempts).Error) + assert.Equal( + t, int64(1), attempts, + "the original delivery's attempt history must stand", + ) +} + +// assertReplayTask proves the task handed to the delivery engine is +// the one the receiver would build for this event and this target, and +// that it carries wantConfig — the target's configuration as it stands +// now rather than as the original delivery ran against it. +func assertReplayTask( + t *testing.T, + task delivery.Task, + webhookID string, + event *database.Event, + target *database.Target, + wantDeliveryID, wantConfig string, +) { + t.Helper() + + assert.Equal(t, wantDeliveryID, task.DeliveryID) + assert.Equal(t, event.ID, task.EventID) + assert.Equal(t, webhookID, task.WebhookID) + assert.Equal(t, event.EntrypointID, task.EntrypointID) + assert.Equal(t, target.ID, task.TargetID) + assert.Equal(t, target.Type, task.TargetType) + assert.JSONEq( + t, wantConfig, task.TargetConfig, + "replay must use the target's current configuration", + ) + assert.Equal(t, event.Method, task.Method) + assert.Equal(t, event.Headers, task.Headers) + assert.Equal(t, event.ContentType, task.ContentType) + assert.Equal(t, 1, task.AttemptNum) + require.NotNil(t, task.Body) + assert.Equal( + t, event.Body, *task.Body, + "replay re-sends the stored event body", + ) +} + +// assertNoLeakedTarget proves the per-webhook database holds no target +// rows. AutoMigrate creates the table there because Delivery declares +// the relation, so it is a ROW that signals a leak: an association +// write would have upserted the whole target, plaintext config and +// all, into the event database. See +// https://git.eeqj.de/sneak/webhooker/issues/206. +func assertNoLeakedTarget(t *testing.T, webhookDB *gorm.DB) { + t.Helper() + + var leaked int64 + + require.NoError(t, webhookDB.Unscoped(). + Model(&database.Target{}).Count(&leaked).Error) + assert.Zero( + t, leaked, + "replay must not write the target into the event database", + ) +} + +// TestHandleDeliveryReplay_RefusesDeletedTarget proves the required +// refusal: a target deleted since the delivery ran is reported as +// deleted rather than erroring, and nothing is created or queued. +func TestHandleDeliveryReplay_RefusesDeletedTarget(t *testing.T) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + notif *recordingNotifier + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr, ¬if) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + tgt := seedConfiguredTarget( + t, db, wh.ID, database.TargetTypeHTTP, + `{"url":"`+replayTargetURL+`"}`, + ) + + event, original := seedFailedDelivery( + t, dbMgr, wh.ID, tgt.ID, + ) + + // Deletes are soft, so the delivery history outlives the target. + require.NoError(t, db.DB().Where( + "id = ?", tgt.ID, + ).Delete(&database.Target{}).Error) + + w := postReplay(t, h, sess, wh.ID, original.ID) + + require.Equal(t, http.StatusSeeOther, w.Code) + assert.Equal( + t, + "/source/"+wh.ID+"/logs?replay=target-deleted", + w.Header().Get("Location"), + ) + + webhookDB, err := dbMgr.GetDB(wh.ID) + require.NoError(t, err) + + assert.Len( + t, listDeliveries(t, webhookDB, event.ID), 1, + "a refused replay must create no delivery", + ) + assert.Empty( + t, notif.Tasks(), + "a refused replay must queue nothing", + ) + + // The refusal is specific, which is why the target is looked up + // including soft-deleted rows: an id that never named a target + // is a different outcome, and a different message, from one the + // operator deleted. + _, orphan := seedFailedDelivery( + t, dbMgr, wh.ID, "target-that-never-existed", + ) + + missing := postReplay(t, h, sess, wh.ID, orphan.ID) + require.Equal(t, http.StatusSeeOther, missing.Code) + assert.Equal( + t, + "/source/"+wh.ID+"/logs?replay=target-missing", + missing.Header().Get("Location"), + ) +} + +// TestHandleDeliveryReplay_RefusesWhileEarlierReplayInFlight proves +// the replay-storm guard: a second replay of the same event to the +// same target is refused while the first is still queued, so repeated +// submissions cannot stack copies of work the engine has not done. +func TestHandleDeliveryReplay_RefusesWhileEarlierReplayInFlight( + t *testing.T, +) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + notif *recordingNotifier + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr, ¬if) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + tgt := seedConfiguredTarget( + t, db, wh.ID, database.TargetTypeHTTP, + `{"url":"`+replayTargetURL+`"}`, + ) + + event, original := seedFailedDelivery( + t, dbMgr, wh.ID, tgt.ID, + ) + + first := postReplay(t, h, sess, wh.ID, original.ID) + require.Equal(t, http.StatusSeeOther, first.Code) + require.Equal( + t, + "/source/"+wh.ID+"/logs?replay=queued", + first.Header().Get("Location"), + ) + + second := postReplay(t, h, sess, wh.ID, original.ID) + require.Equal(t, http.StatusSeeOther, second.Code) + assert.Equal( + t, + "/source/"+wh.ID+"/logs?replay=in-flight", + second.Header().Get("Location"), + ) + + webhookDB, err := dbMgr.GetDB(wh.ID) + require.NoError(t, err) + + assert.Len( + t, listDeliveries(t, webhookDB, event.ID), 2, + "the refused second replay must add nothing", + ) + assert.Len( + t, notif.Tasks(), 1, + "only the first replay reaches the delivery engine", + ) + + // A delivery the engine has not finished is not replayable + // either, which is the same rule seen from the other side. + queued := theOtherDelivery( + t, listDeliveries(t, webhookDB, event.ID), original.ID, + ) + + pending := postReplay(t, h, sess, wh.ID, queued.ID) + require.Equal(t, http.StatusSeeOther, pending.Code) + assert.Equal( + t, + "/source/"+wh.ID+"/logs?replay=not-terminal", + pending.Header().Get("Location"), + ) +} + +// TestHandleSourceLogs_RendersReplayControlAndBanner proves the action +// reaches the page it belongs on: a finished delivery renders a POST +// form carrying a CSRF token, and the outcome code a refusal redirects +// with becomes a readable message. +func TestHandleSourceLogs_RendersReplayControlAndBanner(t *testing.T) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + tgt := seedConfiguredTarget( + t, db, wh.ID, database.TargetTypeHTTP, + `{"url":"`+replayTargetURL+`"}`, + ) + + _, original := seedFailedDelivery(t, dbMgr, wh.ID, tgt.ID) + + body := renderSourceLogsPage(t, h, sess, wh.ID) + + assert.Contains( + t, body, + `action="/source/`+wh.ID+`/deliveries/`+ + original.ID+`/replay"`, + ) + assert.Contains(t, body, `method="POST"`) + assert.Contains(t, body, `name="csrf_token"`) + assert.Contains(t, body, ">Replay<") + + refused := renderSourceLogsPageWithQuery( + t, h, sess, wh.ID, "?replay=target-deleted", + ) + + assert.Contains(t, refused, "alert-error") + assert.Contains(t, refused, "has been deleted") + + // An outcome code nobody issued renders no banner at all. + unknown := renderSourceLogsPageWithQuery( + t, h, sess, wh.ID, "?replay=made-up", + ) + + assert.NotContains(t, unknown, "alert-error") + assert.NotContains(t, unknown, "alert-success") + assert.NotContains(t, unknown, "made-up") +} diff --git a/internal/handlers/delivery_result_view.go b/internal/handlers/delivery_result_view.go new file mode 100644 index 0000000..e3bb27b --- /dev/null +++ b/internal/handlers/delivery_result_view.go @@ -0,0 +1,169 @@ +package handlers + +import ( + "sneak.berlin/go/webhooker/internal/delivery" +) + +// maxRenderedResponseBytes caps how many bytes of one stored +// delivery response body reach the event log page. +// +// The bound is enforced here and in SQL, because this page's +// memory profile must not depend on a constant in another +// package staying where it is, and because rows predating the +// delivery engine's own cap or restored from an archive are +// not covered by it at all. +// +// It happens to equal that engine cap, so a row written by the +// current engine reaches this one exactly and is never cut +// twice. Nothing here may assume the two differ: see view. +const maxRenderedResponseBytes = 4096 + +// deliveryResultColumns is the delivery attempt projection. +// The casts to blob are load-bearing for the same reason they +// are in eventLogColumns: they make substr and length count +// bytes rather than characters, and they make SQLite do the +// cut, so an oversized stored response never becomes a Go +// string at all. +const deliveryResultColumns = "delivery_id, attempt_num, success, " + + "status_code, error, duration, " + + "substr(cast(response_body as blob), 1, ?) AS response_body, " + + "length(cast(response_body as blob)) AS response_bytes" + +// DeliveryResultView is the display-safe projection of one +// delivery attempt for the event log page. It carries a +// capped response body plus the true stored size, so the page +// can mark a response as truncated without holding the whole +// thing. +// +// Both Error and ResponseBody have been through the target's +// Redactor. The engine already masks the URL out of the +// errors it stores, so for errors this is a second line +// covering rows written before it did; for response bodies it +// is the only line, and its reach is what +// delivery.Redactor documents. +type DeliveryResultView struct { + AttemptNum int + Success bool + + // StatusCode is 0 when the attempt never got a response, + // which is why the page asks HasStatusCode rather than + // printing the number. + StatusCode int + + // Error is the stored failure message, redacted. + Error string + + // DurationMS is how long the attempt took. + DurationMS int64 + + // ResponseBody holds at most maxRenderedResponseBytes + // bytes of the stored response, redacted. It is remote + // content and must only ever be rendered escaped. + ResponseBody string + + // ResponseBytes is the size of the stored response body, + // before the cut and before redaction. It is what the + // remote sent only when ResponseSizeKnown is set. + ResponseBytes int64 + + // ResponseShownBytes is how much of that the page is + // showing. It is the size of the cut, taken before + // redaction, so the truncation marker reports what SQLite + // returned rather than how much the marker substitution + // then changed the length. + ResponseShownBytes int + + // ResponseTruncated reports that the body shown may be + // incomplete, so the page owes the reader a marker. Every + // body that reaches the cap counts, because one the + // delivery engine cut at its own equal cap is + // indistinguishable from a complete one. + ResponseTruncated bool + + // ResponseSizeKnown reports that ResponseBytes is the whole + // response the remote sent, so the marker may quote it. It + // is false for a body that only reaches the cap, where how + // much came after it was never recorded. + ResponseSizeKnown bool +} + +// HasStatusCode reports whether the attempt got as far as an +// HTTP response. A transport failure stores no status code, +// and rendering that as "0" would read as a real status. +func (v DeliveryResultView) HasStatusCode() bool { + return v.StatusCode != 0 +} + +// deliveryResultRow is one row of the delivery attempt +// projection. Its response body arrives already cut to the +// cap by SQLite, with the true size beside it. +type deliveryResultRow struct { + DeliveryID string + AttemptNum int + Success bool + StatusCode int + Error string + Duration int64 + ResponseBody []byte + ResponseBytes int64 +} + +// view projects a loaded row for rendering, stripping the +// target's own credential out of the two fields a remote peer +// gets to influence. +func (r *deliveryResultRow) view( + redactor delivery.Redactor, +) DeliveryResultView { + body := r.ResponseBody + + // Two different cuts can have shortened this body, and the + // row records only one of them. SQLite cuts here, whenever + // the stored value is larger than the cap. The delivery + // engine cut earlier, whenever the remote sent more than + // its own maxBodyLog — which is this same number, so such a + // row stores the cut length as its whole length and nothing + // in it separates a response that ended at the cap from one + // severed there. + // + // So a body that reaches the cap is treated as cut either + // way. Gating on ResponseBytes alone would assume the two + // caps differ, and they do not: under the current engine + // that gate never opens. + cut := r.ResponseBytes > int64(len(body)) || + len(body) >= maxRenderedResponseBytes + + // The row holds more than the page shows only in the first + // of those cases. In the second the stored row is all there + // is, and its size is a floor rather than the true one. + sizeKnown := r.ResponseBytes > int64(len(body)) + + // Only a cut response can have been left mid-sequence, + // exactly as with an event body. + if cut { + body = trimPartialRune(body) + } + + // A cut body goes through RedactCut: the remote controls + // the padding ahead of a credential it echoes, so it + // controls where the cut falls inside that credential, and + // the severed prefix left behind matches no secret whole. + rendered := string(body) + if cut { + rendered = redactor.RedactCut(rendered) + } else { + rendered = redactor.Redact(rendered) + } + + return DeliveryResultView{ + AttemptNum: r.AttemptNum, + Success: r.Success, + StatusCode: r.StatusCode, + Error: redactor.Redact(r.Error), + DurationMS: r.Duration, + ResponseBody: rendered, + ResponseBytes: r.ResponseBytes, + ResponseShownBytes: len(body), + ResponseTruncated: cut, + ResponseSizeKnown: sizeKnown, + } +} diff --git a/internal/handlers/delivery_result_view_test.go b/internal/handlers/delivery_result_view_test.go new file mode 100644 index 0000000..8b4a388 --- /dev/null +++ b/internal/handlers/delivery_result_view_test.go @@ -0,0 +1,515 @@ +package handlers_test + +import ( + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm/clause" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" + "sneak.berlin/go/webhooker/internal/handlers" + "sneak.berlin/go/webhooker/internal/session" +) + +// responseCap is the number of response bytes the event log +// page is allowed to render for one delivery attempt. +const responseCap = handlers.MaxRenderedResponseBytesForTest + +// failedAttempt describes the failed delivery every test in +// this file seeds. The values are distinctive so that finding +// them in the rendered page cannot be a coincidence. +const ( + attemptStatusCode = 502 + attemptDurationMS = 1234 + attemptNumber = 3 + attemptError = "upstream returned 502 Bad Gateway" +) + +// seedFailedDeliveryWithResponse records an event, a failed +// delivery against targetID, and one delivery result carrying +// the given response body. It returns the delivery. +// +// Distinct from seedFailedDelivery in delivery_replay_test.go, +// which seeds an attempt with no response body and returns the +// event as well; these tests need the recorded response. +func seedFailedDeliveryWithResponse( + t *testing.T, + dbMgr *database.WebhookDBManager, + webhookID, targetID, responseBody string, +) *database.Delivery { + t.Helper() + + webhookDB, err := dbMgr.GetDB(webhookID) + require.NoError(t, err) + + event := &database.Event{ + WebhookID: webhookID, + Method: http.MethodPost, + Body: `{"test":true}`, + ContentType: contentTypeJSON, + } + + require.NoError(t, webhookDB.Omit( + clause.Associations, + ).Create(event).Error) + + dlv := &database.Delivery{ + EventID: event.ID, + TargetID: targetID, + Status: database.DeliveryStatusFailed, + } + + require.NoError(t, webhookDB.Omit( + clause.Associations, + ).Create(dlv).Error) + + result := &database.DeliveryResult{ + DeliveryID: dlv.ID, + AttemptNum: attemptNumber, + Success: false, + StatusCode: attemptStatusCode, + ResponseBody: responseBody, + Error: attemptError, + Duration: attemptDurationMS, + } + + require.NoError(t, webhookDB.Omit( + clause.Associations, + ).Create(result).Error) + + return dlv +} + +// seedFailureAndRender seeds a failed delivery against a +// target of the given type and config, and returns the +// rendered event log page. +func seedFailureAndRender( + t *testing.T, + targetType database.TargetType, + config, responseBody string, +) string { + t.Helper() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + tgt := seedConfiguredTarget( + t, db, wh.ID, targetType, config, + ) + + seedFailedDeliveryWithResponse( + t, dbMgr, wh.ID, tgt.ID, responseBody, + ) + + return renderSourceLogsPage(t, h, sess, wh.ID) +} + +// TestHandleSourceLogs_RendersFailedAttempt is the regression +// test for the reported gap: a failed delivery used to render +// as the status word alone, so diagnosing it meant opening the +// per-webhook SQLite file by hand. +func TestHandleSourceLogs_RendersFailedAttempt(t *testing.T) { + t.Parallel() + + body := seedFailureAndRender( + t, + database.TargetTypeHTTP, + `{"url":"https://example.com/hook/abc"}`, + "upstream exploded", + ) + + assert.Contains( + t, body, strconv.Itoa(attemptStatusCode), + "the attempt's status code must reach the page", + ) + assert.Contains( + t, body, attemptError, + "the attempt's error must reach the page", + ) + assert.Contains( + t, body, strconv.Itoa(attemptDurationMS), + "the attempt's duration must reach the page", + ) + assert.Contains( + t, body, "Attempt "+strconv.Itoa(attemptNumber), + "the attempt number must reach the page", + ) + assert.Contains( + t, body, "upstream exploded", + "the attempt's response body must reach the page", + ) +} + +// TestHandleSourceLogs_EscapesResponseBody proves the +// response body is treated as the untrusted remote content it +// is. The remote chooses these bytes and the page is rendered +// inside the operator's authenticated origin, where the +// application's own CSP allows inline script from 'self'. +func TestHandleSourceLogs_EscapesResponseBody(t *testing.T) { + t.Parallel() + + const payload = `` + + body := seedFailureAndRender( + t, + database.TargetTypeHTTP, + `{"url":"https://example.com/hook/abc"}`, + payload, + ) + + assert.NotContains(t, body, payload) + assert.NotContains(t, body, "` + + wh := seedWebhook(t, db) + evt := seedEventWithBody(t, dbMgr, wh.ID, payload) + + w := fetchEventBody(t, h, sess, wh.ID, evt.ID) + + require.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, payload, w.Body.String()) + + contentType := w.Header().Get("Content-Type") + assert.Equal(t, "application/octet-stream", contentType) + assert.NotContains(t, contentType, "html") + assert.NotContains(t, contentType, "xml") + assert.NotContains(t, contentType, "javascript") + assert.Contains( + t, w.Header().Get("Content-Disposition"), "attachment", + ) + assert.Equal( + t, "nosniff", + w.Header().Get("X-Content-Type-Options"), + ) +} + +// TestHandleEventBodyDownload_OtherUsersEvent404s is the +// authorization test the definition of done asks for: an event +// stored under a webhook the session user does not own is not +// readable, and the miss does not distinguish itself from a +// nonexistent one. +func TestHandleEventBodyDownload_OtherUsersEvent404s( + t *testing.T, +) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + const theirPayload = "OTHER-USERS-PAYLOAD-8b1d" + + theirs := seedWebhookFor(t, db, otherTestUserID) + evt := seedEventWithBody(t, dbMgr, theirs.ID, theirPayload) + + w := fetchEventBody(t, h, sess, theirs.ID, evt.ID) + + assert.Equal(t, http.StatusNotFound, w.Code) + assert.NotContains(t, w.Body.String(), theirPayload) +} + +// TestHandleEventBodyDownload_EventOfAnotherWebhook404s pins +// that holding a valid event id is not enough: the event has to +// belong to the webhook in the path. Both webhooks here are the +// session user's and both have event databases, so the +// ownership check cannot be what produces the 404. +// +// What does produce it is the per-webhook database file rather +// than the webhook_id predicate on the query — removing that +// predicate leaves this test green, because the sibling's event +// is in a different file. The test is kept as the behavioural +// guard the route owes; see serveEventBody for which mechanism +// is load-bearing. +func TestHandleEventBodyDownload_EventOfAnotherWebhook404s( + t *testing.T, +) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + const other = "BELONGS-TO-THE-OTHER-WEBHOOK-3c7e" + + mine := seedWebhook(t, db) + seedEventWithBody(t, dbMgr, mine.ID, `{"mine":true}`) + + sibling := seedWebhook(t, db) + evt := seedEventWithBody(t, dbMgr, sibling.ID, other) + + w := fetchEventBody(t, h, sess, mine.ID, evt.ID) + + assert.Equal(t, http.StatusNotFound, w.Code) + assert.NotContains(t, w.Body.String(), other) +} + +// TestHandleEventBodyDownload_UnknownEvent404s covers the plain +// miss, including an id that is not a uuid at all and so never +// reaches the query or the response header. +func TestHandleEventBodyDownload_UnknownEvent404s(t *testing.T) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + seedEventWithBody(t, dbMgr, wh.ID, `{"mine":true}`) + + for _, id := range []string{ + uuid.New().String(), + `../../etc/passwd`, + "not-a-uuid", + `x"; rm -rf /`, + } { + w := fetchEventBody(t, h, sess, wh.ID, id) + + assert.Equal( + t, http.StatusNotFound, w.Code, + "event id %q", id, + ) + assert.Empty( + t, w.Header().Get("Content-Disposition"), + "event id %q must not reach a header", id, + ) + } +} + +// TestHandleEventBodyDownload_ReapedEvent404s pins what happens +// when the retention reaper takes an event out from under this +// route. The body is read in one query before any header is +// written, so a reaped event cannot produce a partial download: +// it is a clean 404 with no Content-Length and no +// Content-Disposition. Both removals the codebase performs are +// covered — the reaper hard-deletes, and a soft-deleted row is +// excluded by the query's own deleted_at predicate rather than +// by GORM's default scope, which Raw bypasses. +func TestHandleEventBodyDownload_ReapedEvent404s(t *testing.T) { + t.Parallel() + + for name, hard := range map[string]bool{ + "soft deleted": false, + "hard deleted": true, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + const payload = "REAPED-PAYLOAD-4d2a" + + wh := seedWebhook(t, db) + evt := seedEventWithBody(t, dbMgr, wh.ID, payload) + + webhookDB, err := dbMgr.GetDB(wh.ID) + require.NoError(t, err) + + del := webhookDB + if hard { + del = del.Unscoped() + } + + require.NoError( + t, + del.Delete(&database.Event{}, "id = ?", evt.ID). + Error, + ) + + w := fetchEventBody(t, h, sess, wh.ID, evt.ID) + + assert.Equal(t, http.StatusNotFound, w.Code) + assert.NotContains(t, w.Body.String(), payload) + assert.Empty(t, w.Header().Get("Content-Length")) + assert.Empty( + t, w.Header().Get("Content-Disposition"), + ) + }) + } +} + +// TestHandleSourceLogs_TruncationMarkerLinksToDownload proves +// the page tells the reader where the rest of the body is, and +// only when there is a rest to fetch. +func TestHandleSourceLogs_TruncationMarkerLinksToDownload( + t *testing.T, +) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + big := seedWebhook(t, db) + bigEvt := seedEventWithBody( + t, dbMgr, big.ID, strings.Repeat("A", 4*bodyCap), + ) + + page := renderSourceLogsPage(t, h, sess, big.ID) + assert.Contains( + t, page, + "/source/"+big.ID+"/logs/"+bigEvt.ID+"/body", + ) + + small := seedWebhook(t, db) + smallEvt := seedEventWithBody( + t, dbMgr, small.ID, `{"kept":"whole"}`, + ) + + page = renderSourceLogsPage(t, h, sess, small.ID) + assert.NotContains( + t, page, + "/source/"+small.ID+"/logs/"+smallEvt.ID+"/body", + ) +} diff --git a/internal/handlers/event_log_view.go b/internal/handlers/event_log_view.go new file mode 100644 index 0000000..5cfc0d1 --- /dev/null +++ b/internal/handlers/event_log_view.go @@ -0,0 +1,144 @@ +package handlers + +import ( + "time" + "unicode/utf8" +) + +// maxRenderedBodyBytes caps how many bytes of a stored event +// body reach the event log page. Bodies come from the +// unauthenticated receiver under the 1 MB ingest cap and +// renderTemplate buffers a whole page before writing it, so +// an uncapped page of paginationPerPage events is tens of +// megabytes of resident memory per concurrent viewer. +const maxRenderedBodyBytes = 8192 + +// eventLogColumns is the event log's projection. The casts to +// blob are load-bearing: they make substr and length count +// bytes rather than characters, so the cap bounds the page in +// bytes whatever the payload's encoding. Cutting in SQLite +// rather than in Go is the point of the projection — an +// oversized body never becomes a Go string at all. +const eventLogColumns = "id, created_at, method, content_type, " + + "resubmitted_from_id, " + + "substr(cast(body as blob), 1, ?) AS body, " + + "length(cast(body as blob)) AS body_bytes" + +// EventLogView is the display-safe projection of an event for +// the event log page, alongside DeliveryView and TargetView. +// It carries a capped body plus the true stored size, so the +// page can mark a body as truncated without ever holding the +// whole thing. +type EventLogView struct { + ID string + CreatedAt time.Time + Method string + ContentType string + + // Body holds at most maxRenderedBodyBytes bytes of the + // stored body. + Body string + + // BodyBytes is the true size of the stored body. + BodyBytes int64 + + // BodyTruncated reports that the stored body was larger + // than the cap, so the page owes the reader a marker. + BodyTruncated bool + + // ResubmittedFromID names the event this one was copied + // from, empty for an event that arrived on the receiver. + ResubmittedFromID string + + // ResubmitCount is how many events have been resubmitted + // from this one. Both directions are shown, because after + // a few resubmits of one captured event the log is + // otherwise a row of identical bodies with nothing saying + // which came from which. + ResubmitCount int + + Deliveries []DeliveryView +} + +// ResubmittedFrom reports that this event is a copy of another. +func (v EventLogView) ResubmittedFrom() bool { + return v.ResubmittedFromID != "" +} + +// BodyShownBytes is how many body bytes the page is actually +// rendering, which the truncation marker reports beside the +// true size. +func (v EventLogView) BodyShownBytes() int { + return len(v.Body) +} + +// eventLogRow is one row of the event log projection. Its +// body column arrives already cut to the cap by SQLite, with +// the true size beside it. +type eventLogRow struct { + ID string + CreatedAt time.Time + Method string + ContentType string + ResubmittedFromID *string + Body []byte + BodyBytes int64 +} + +// view projects a loaded row for rendering. +func (r *eventLogRow) view() EventLogView { + body := r.Body + truncated := r.BodyBytes > int64(len(body)) + + // Only a cut body can have been left mid-sequence by + // this query. A whole body is passed through exactly as + // stored, however malformed. + if truncated { + body = trimPartialRune(body) + } + + var from string + if r.ResubmittedFromID != nil { + from = *r.ResubmittedFromID + } + + return EventLogView{ + ID: r.ID, + CreatedAt: r.CreatedAt, + Method: r.Method, + ContentType: r.ContentType, + Body: string(body), + BodyBytes: r.BodyBytes, + BodyTruncated: truncated, + ResubmittedFromID: from, + } +} + +// trimPartialRune drops a trailing UTF-8 sequence that the +// byte-wise cut left incomplete, so a multi-byte rune severed +// at the cap does not surface as a mojibake tail. +// +// Bytes that are merely invalid UTF-8 are left exactly as +// stored: this service receives binary payloads, and rewriting +// them would misreport what was delivered. The distinction is +// utf8.FullRune's — it reports a complete sequence for an +// invalid encoding too, since that decodes to a width-1 error +// rune, so only a valid prefix still waiting for its +// continuation bytes is removed. A tail with no rune start in +// its last utf8.UTFMax bytes cannot be an incomplete sequence +// either, and is likewise left alone. +func trimPartialRune(b []byte) []byte { + for i := len(b) - 1; i >= 0 && len(b)-i <= utf8.UTFMax; i-- { + if !utf8.RuneStart(b[i]) { + continue + } + + if utf8.FullRune(b[i:]) { + return b + } + + return b[:i] + } + + return b +} diff --git a/internal/handlers/event_log_view_test.go b/internal/handlers/event_log_view_test.go new file mode 100644 index 0000000..e1d5a41 --- /dev/null +++ b/internal/handlers/event_log_view_test.go @@ -0,0 +1,261 @@ +package handlers_test + +import ( + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "unicode/utf8" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm/clause" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/handlers" + "sneak.berlin/go/webhooker/internal/session" +) + +// bodyCap is the number of body bytes the event log page is +// allowed to render for one event. +const bodyCap = handlers.MaxRenderedBodyBytesForTest + +// snowman is a three-byte rune, so a body of them straddles the +// byte-wise cut: bodyCap is not a multiple of three. +const snowman = "☃" + +// seedEventWithBody records one event with the given body in the +// webhook's own database and returns it, so a caller that needs +// the generated event id can have it. +func seedEventWithBody( + t *testing.T, + dbMgr *database.WebhookDBManager, + webhookID string, + body string, +) *database.Event { + t.Helper() + + webhookDB, err := dbMgr.GetDB(webhookID) + require.NoError(t, err) + + event := &database.Event{ + WebhookID: webhookID, + Method: http.MethodPost, + Body: body, + ContentType: "application/octet-stream", + } + + require.NoError(t, webhookDB.Omit( + clause.Associations, + ).Create(event).Error) + + return event +} + +// seedAndProject stores one body and returns the projection the +// event log page would be handed for it. +func seedAndProject( + t *testing.T, + body string, +) handlers.EventLogView { + t.Helper() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + seedEventWithBody(t, dbMgr, wh.ID, body) + + views := h.LoadEventLogViewsForTest( + httptest.NewRecorder(), *wh, 1, + ) + require.Len(t, views, 1) + + return views[0] +} + +// TestHandleSourceLogs_BoundsOversizeBody proves the rendered +// page is bounded by the cap rather than by the stored payload: +// the body here is 64 times the cap, and the ingest path would +// accept twice as much again. +func TestHandleSourceLogs_BoundsOversizeBody(t *testing.T) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + const ( + sentinel = "TAIL-SENTINEL-1f4a9c" + storedBytes = 512 * 1024 + ) + + wh := seedWebhook(t, db) + seedEventWithBody( + t, dbMgr, wh.ID, + strings.Repeat("A", storedBytes-len(sentinel))+sentinel, + ) + + page := renderSourceLogsPage(t, h, sess, wh.ID) + + // Nothing past the cap reaches the page, and the whole page + // stays far below the stored body it is reporting on. + assert.NotContains(t, page, sentinel) + assert.Less(t, len(page), 4*bodyCap) + + // The marker states the true stored size, not the cut one. + assert.Contains( + t, page, + "showing "+strconv.Itoa(bodyCap)+ + " of "+strconv.Itoa(storedBytes)+" bytes", + ) +} + +// TestHandleSourceLogs_SmallBodyRendersWhole guards the other +// side of the cap: a body under it is shown in full and carries +// no truncation marker. +func TestHandleSourceLogs_SmallBodyRendersWhole(t *testing.T) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + seedEventWithBody(t, dbMgr, wh.ID, `{"kept":"whole"}`) + + page := renderSourceLogsPage(t, h, sess, wh.ID) + + assert.Contains(t, page, ""kept"") + assert.NotContains(t, page, "Body truncated for display") +} + +// TestEventLogView_CutMidRune proves a multi-byte rune severed +// by the byte-wise cut is dropped rather than surfaced as a +// mojibake tail. +func TestEventLogView_CutMidRune(t *testing.T) { + t.Parallel() + + body := strings.Repeat(snowman, 4096) + view := seedAndProject(t, body) + + // bodyCap bytes hold bodyCap/3 whole snowmen and two bytes + // of the next one; those two are dropped. + whole := bodyCap / len(snowman) + + assert.True(t, view.BodyTruncated) + assert.Equal(t, int64(len(body)), view.BodyBytes) + assert.Equal(t, strings.Repeat(snowman, whole), view.Body) + assert.True(t, utf8.ValidString(view.Body)) + assert.LessOrEqual(t, len(view.Body), bodyCap) +} + +// TestEventLogView_BinaryBodyLeftAsStored proves a binary +// payload is passed through byte for byte. Its tail is invalid +// UTF-8 however the cut falls, so repairing it would misreport +// what the sender delivered. +func TestEventLogView_BinaryBodyLeftAsStored(t *testing.T) { + t.Parallel() + + raw := make([]byte, bodyCap+808) + for i := range raw { + // 0x80..0xBF: continuation bytes, never a rune start. + raw[i] = 0x80 | byte(i%0x40) + } + + view := seedAndProject(t, string(raw)) + + assert.True(t, view.BodyTruncated) + assert.Equal(t, int64(len(raw)), view.BodyBytes) + assert.Equal(t, string(raw[:bodyCap]), view.Body) + assert.False(t, utf8.ValidString(view.Body)) +} + +// TestTrimPartialRune covers the distinction the cut repair +// turns on: an incomplete but valid sequence is dropped, while +// bytes that are merely invalid UTF-8 are left alone. +func TestTrimPartialRune(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + in []byte + want []byte + }{{ + name: "complete ascii", + in: []byte("abc"), + want: []byte("abc"), + }, { + name: "complete multibyte", + in: []byte("ab" + snowman), + want: []byte("ab" + snowman), + }, { + name: "two byte rune cut", + in: []byte{'a', 0xC3}, + want: []byte{'a'}, + }, { + name: "three byte rune cut after one", + in: []byte{'a', 0xE2}, + want: []byte{'a'}, + }, { + name: "three byte rune cut after two", + in: []byte{'a', 0xE2, 0x98}, + want: []byte{'a'}, + }, { + name: "four byte rune cut", + in: []byte{'a', 0xF0, 0x9F, 0x92}, // U+1F4A9 cut + want: []byte{'a'}, + }, { + name: "invalid start byte kept", + in: []byte{'a', 0xFF}, + want: []byte{'a', 0xFF}, + }, { + name: "orphan continuation bytes kept", + in: []byte{0x80, 0x81, 0x82, 0x83, 0x84}, + want: []byte{0x80, 0x81, 0x82, 0x83, 0x84}, + }, { + name: "truncated sequence followed by junk kept", + in: []byte{0xE2, 0x98, 0xFF}, + want: []byte{0xE2, 0x98, 0xFF}, + }, { + name: "empty", + in: []byte{}, + want: []byte{}, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + assert.Equal( + t, tc.want, + handlers.TrimPartialRuneForTest(tc.in), + ) + }) + } +} diff --git a/internal/handlers/event_resubmit.go b/internal/handlers/event_resubmit.go new file mode 100644 index 0000000..21e54b8 --- /dev/null +++ b/internal/handlers/event_resubmit.go @@ -0,0 +1,273 @@ +package handlers + +import ( + "errors" + "net/http" + "strconv" + + "github.com/go-chi/chi" + "github.com/google/uuid" + "gorm.io/gorm" + "sneak.berlin/go/webhooker/internal/database" +) + +// resubmitOutcomeParam is the query parameter the resubmit POST +// redirects with and the event log page reads its banner from. +const resubmitOutcomeParam = "resubmit" + +// resubmitOutcomeCode is the outcome of a resubmit POST. The redirect +// carries one of these fixed codes rather than a message, so nothing a +// client submits can reach the rendered page through it. +type resubmitOutcomeCode string + +const ( + // resubmitQueued reports that a new event was stored and its + // deliveries handed to the delivery engine. + resubmitQueued resubmitOutcomeCode = "queued" + + // resubmitNoTargets reports a source with no active targets. The + // new event is stored either way, exactly as a received event + // with no targets is. + resubmitNoTargets resubmitOutcomeCode = "no-targets" +) + +// resubmitOutcome returns the banner the event log page shows for an +// outcome code, and whether the resubmit was queued. An unrecognised +// code yields no banner. +func resubmitOutcome(code string) (string, bool) { + switch resubmitOutcomeCode(code) { + case resubmitQueued: + return "Resubmitted: a new event was created from the stored " + + "one and queued to every active target.", true + case resubmitNoTargets: + return "Resubmitted: a new event was created, but this " + + "source has no active targets, so nothing was queued.", + true + default: + return "", false + } +} + +// resubmitSource is the stored event a resubmit copies. Its body is +// read as bytes rather than as a string so the copy is byte-identical +// to what was received, whatever the payload's encoding. +type resubmitSource struct { + ID string + EntrypointID string + Method string + Headers string + ContentType string + Body []byte +} + +// resubmitColumns is the projection resubmitSource is loaded through. +// The cast to blob is what makes the driver hand back the stored bytes +// rather than a string conversion, the same reason eventBodyQuery +// casts. +const resubmitColumns = "id, entrypoint_id, method, headers, " + + "content_type, cast(body as blob) AS body" + +// HandleEventResubmit re-injects a stored event as a new undelivered +// event. +// +// This is the testing counterpart to per-delivery replay, and the two +// select targets differently on purpose. A replay re-sends ONE +// finished delivery to ITS OWN target, which is recovery. A resubmit +// stores a NEW event copied from the stored one and fans it out to the +// webhook's currently ACTIVE targets, resolved fresh by the query the +// receiver uses — so a target created after the original event arrived +// receives it, which is what makes capturing real traffic and firing +// it at a backend under development possible. The original event's +// deliveries have no bearing on where the copy goes. +// +// Nothing about the original delivery is re-sent: what is re-injected +// is the stored EVENT. The response bodies and headers the original +// deliveries received stay where they are. +// +// Resubmitting the same event repeatedly is supported and is the point +// of the feature, so replay's in-flight refusal is deliberately not +// applied here. The route's rate limit is what bounds a held-down +// button. +func (h *Handlers) HandleEventResubmit() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + webhook, ok := h.ownedWebhook(w, r) + if !ok { + return + } + + // The body size cap is enforced by the MaxBodySize + // middleware, which runs before CSRF parses the form. + err := r.ParseForm() + if err != nil { + http.Error(w, "Bad request", http.StatusBadRequest) + + return + } + + h.resubmitEvent(w, r, webhook) + } +} + +// resubmitEvent performs the resubmit for a webhook the caller has +// already established the session's user owns. +func (h *Handlers) resubmitEvent( + w http.ResponseWriter, + r *http.Request, + webhook database.Webhook, +) { + // Parsing the id before use keeps a malformed id out of the SQL + // and makes the value the query sees come from uuid's own fixed + // alphabet rather than from the request. + eventID, err := uuid.Parse(chi.URLParam(r, "eventID")) + if err != nil { + http.NotFound(w, r) + + return + } + + if !h.dbMgr.DBExists(webhook.ID) { + http.NotFound(w, r) + + return + } + + webhookDB, err := h.dbMgr.GetDB(webhook.ID) + if err != nil { + h.serverError(w, "failed to get webhook database", err) + + return + } + + // Read before the write transaction is opened. The body can be up + // to the 1 MB ingest cap, and every transaction on these files + // takes the write lock at BEGIN (_txlock=immediate, see + // internal/database/sqlite_open.go), so reading inside it would + // hold that lock against the receiver for the length of the read. + src, found, err := loadResubmitSource( + webhookDB, webhook.ID, eventID.String(), + ) + if err != nil { + h.serverError(w, "failed to load event to resubmit", err) + + return + } + + // A miss is a 404 whether the event was reaped, belongs to + // another webhook, or never existed. + if !found { + http.NotFound(w, r) + + return + } + + h.queueResubmit(w, r, webhook, src) +} + +// loadResubmitSource reads the stored event a resubmit copies, and +// whether it exists within the webhook. +// +// The webhook_id predicate is currently redundant against the +// per-webhook database files — a sibling webhook's event is not in the +// database being queried at all — and is there so the scoping survives +// any future change that puts more than one webhook's events in one +// file. Going through Model applies GORM's soft-delete scope, which is +// what stops a reaped event being resubmitted. +func loadResubmitSource( + webhookDB *gorm.DB, + webhookID, eventID string, +) (resubmitSource, bool, error) { + var src resubmitSource + + err := webhookDB.Model(&database.Event{}). + Select(resubmitColumns). + Where("id = ? AND webhook_id = ?", eventID, webhookID). + First(&src).Error + + if errors.Is(err, gorm.ErrRecordNotFound) { + return src, false, nil + } + + if err != nil { + return src, false, err + } + + return src, true, nil +} + +// queueResubmit stores the copy and fans it out to the webhook's +// active targets. +func (h *Handlers) queueResubmit( + w http.ResponseWriter, + r *http.Request, + webhook database.Webhook, + src resubmitSource, +) { + // The receiver's own query, run now: an active target created + // after the original event arrived is included, and an + // inactive one is skipped rather than refused. + targets, err := h.loadActiveTargets(webhook.ID) + if err != nil { + h.serverError(w, "failed to query targets", err) + + return + } + + event, tasks, err := h.createAndFanOut( + eventSource{ + WebhookID: webhook.ID, + EntrypointID: src.EntrypointID, + Method: src.Method, + HeadersJSON: src.Headers, + ContentType: src.ContentType, + Body: src.Body, + ResubmittedFromID: &src.ID, + }, + targets, + ) + if err != nil { + h.serverError(w, "failed to store resubmitted event", err) + + return + } + + h.mtr.EventResubmitted() + + h.log.Info( + "event resubmitted", + "webhook_id", webhook.ID, + "event_id", event.ID, + "resubmitted_from_id", src.ID, + "target_count", len(tasks), + ) + + code := resubmitQueued + if len(tasks) == 0 { + code = resubmitNoTargets + } + + h.finishResubmit(w, r, webhook, code) +} + +// finishResubmit redirects back to the event log the resubmit was +// triggered from, carrying the outcome code the page turns into a +// banner and the page number the form submitted. +func (h *Handlers) finishResubmit( + w http.ResponseWriter, + r *http.Request, + webhook database.Webhook, + code resubmitOutcomeCode, +) { + dest := "/source/" + webhook.ID + "/logs?" + + resubmitOutcomeParam + "=" + string(code) + + // The page is read from the form rather than the query string: + // this is a POST, and its query string is what logs and Referer + // headers record. + if page := pageOrFirst( + r.PostFormValue("page"), + ); page > 1 { + dest += "&page=" + strconv.Itoa(page) + } + + http.Redirect(w, r, dest, http.StatusSeeOther) +} diff --git a/internal/handlers/event_resubmit_test.go b/internal/handlers/event_resubmit_test.go new file mode 100644 index 0000000..b2ea384 --- /dev/null +++ b/internal/handlers/event_resubmit_test.go @@ -0,0 +1,603 @@ +package handlers_test + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" + "sneak.berlin/go/webhooker/internal/handlers" + "sneak.berlin/go/webhooker/internal/session" +) + +// resubmitTargetURL is a public destination, so a target configured +// with it is one the SSRF guard would accept. Nothing in these tests +// dispatches to it: the notifier is recorded, not run. +const resubmitTargetURL = "http://93.184.216.34/hook" + +// resubmitEventHeaders is the stored header JSON a seeded event +// carries, so a test can prove the copy takes it verbatim. +const resubmitEventHeaders = `{"X-Test":["yes"],"X-Trace":["abc"]}` + +// seedStoredEvent records one event in a webhook's own database with +// no deliveries at all, which is the state a captured event is in when +// the operator has yet to create the target to test. +func seedStoredEvent( + t *testing.T, + dbMgr *database.WebhookDBManager, + webhookID, body string, +) *database.Event { + t.Helper() + + webhookDB, err := dbMgr.GetDB(webhookID) + require.NoError(t, err) + + event := &database.Event{ + WebhookID: webhookID, + EntrypointID: "entrypoint-" + webhookID, + Method: http.MethodPost, + Headers: resubmitEventHeaders, + Body: body, + ContentType: contentTypeJSON, + } + + require.NoError(t, webhookDB.Omit( + clause.Associations, + ).Create(event).Error) + + return event +} + +// postResubmit runs the real resubmit handler for one event. +func postResubmit( + t *testing.T, + h *handlers.Handlers, + sess *session.Session, + webhookID, eventID string, +) *httptest.ResponseRecorder { + t.Helper() + + req := postRequest( + "/source/"+webhookID+"/events/"+eventID+"/resubmit", + authenticatedCookies( + t, sess, deleteTestUserID, deleteTestUsername, + ), + map[string]string{ + paramSourceID: webhookID, + paramEventID: eventID, + }, + ) + + w := httptest.NewRecorder() + h.HandleEventResubmit().ServeHTTP(w, req) + + return w +} + +// listEvents reads every event in a webhook's database, oldest first. +func listEvents( + t *testing.T, webhookDB *gorm.DB, +) []database.Event { + t.Helper() + + var events []database.Event + + require.NoError(t, webhookDB. + Order("created_at ASC, id ASC"). + Find(&events).Error) + + return events +} + +// theOtherEvent returns the one event that is not excludeID. +func theOtherEvent( + t *testing.T, events []database.Event, excludeID string, +) database.Event { + t.Helper() + + var found []database.Event + + for _, e := range events { + if e.ID != excludeID { + found = append(found, e) + } + } + + require.Len(t, found, 1) + + return found[0] +} + +// TestHandleEventResubmit_DeliversToTargetCreatedAfterTheEvent is the +// core of the feature and the thing per-delivery replay cannot do: the +// event was captured before the target existed, so it has no delivery +// to replay, and the resubmit must still reach the new target. +func TestHandleEventResubmit_DeliversToTargetCreatedAfterTheEvent( + t *testing.T, +) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + notif *recordingNotifier + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr, ¬if) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + original := seedStoredEvent( + t, dbMgr, wh.ID, `{"captured":"traffic"}`, + ) + + // The dev backend is registered only now, after the traffic was + // captured. It has no prior delivery of anything. + tgt := seedConfiguredTarget( + t, db, wh.ID, database.TargetTypeHTTP, + `{"url":"`+resubmitTargetURL+`"}`, + ) + + w := postResubmit(t, h, sess, wh.ID, original.ID) + + require.Equal(t, http.StatusSeeOther, w.Code) + assert.Equal( + t, + "/source/"+wh.ID+"/logs?resubmit=queued", + w.Header().Get("Location"), + ) + + webhookDB, err := dbMgr.GetDB(wh.ID) + require.NoError(t, err) + + events := listEvents(t, webhookDB) + require.Len( + t, events, 2, + "resubmit must create a new event, not mark the old one", + ) + + fresh := theOtherEvent(t, events, original.ID) + assertEventCopy(t, original, fresh) + + // The delivery hangs off the NEW event, and the original event + // still has none. + assert.Empty( + t, listDeliveries(t, webhookDB, original.ID), + "the original event must be left untouched", + ) + + deliveries := listDeliveries(t, webhookDB, fresh.ID) + require.Len(t, deliveries, 1) + assert.Equal(t, tgt.ID, deliveries[0].TargetID) + assert.Equal( + t, database.DeliveryStatusPending, deliveries[0].Status, + ) + + tasks := notif.Tasks() + require.Len(t, tasks, 1) + assert.Equal(t, deliveries[0].ID, tasks[0].DeliveryID) + assertResubmitTask(t, tasks[0], wh.ID, &fresh, tgt) + + assertNoLeakedTarget(t, webhookDB) +} + +// assertEventCopy proves the resubmitted event copies every stored +// field of the event it came from, and records the provenance that +// keeps the log readable. It also pins that a received event carries +// no source event of its own. +func assertEventCopy( + t *testing.T, original *database.Event, fresh database.Event, +) { + t.Helper() + + assert.Equal(t, original.Method, fresh.Method) + assert.Equal(t, original.Headers, fresh.Headers) + assert.Equal(t, original.Body, fresh.Body) + assert.Equal(t, original.ContentType, fresh.ContentType) + assert.Equal(t, original.EntrypointID, fresh.EntrypointID) + assert.Equal(t, original.WebhookID, fresh.WebhookID) + assert.NotEqual(t, original.ID, fresh.ID) + + require.NotNil(t, fresh.ResubmittedFromID) + assert.Equal(t, original.ID, *fresh.ResubmittedFromID) + + assert.Nil( + t, original.ResubmittedFromID, + "a received event records no source event", + ) +} + +// assertResubmitTask proves the task handed to the delivery engine is +// the one the receiver would build for the NEW event and this target. +func assertResubmitTask( + t *testing.T, + task delivery.Task, + webhookID string, + fresh *database.Event, + target *database.Target, +) { + t.Helper() + + assert.Equal(t, fresh.ID, task.EventID) + assert.Equal(t, webhookID, task.WebhookID) + assert.Equal(t, fresh.EntrypointID, task.EntrypointID) + assert.Equal(t, target.ID, task.TargetID) + assert.Equal(t, target.Type, task.TargetType) + assert.Equal(t, fresh.Method, task.Method) + assert.Equal(t, fresh.Headers, task.Headers) + assert.Equal(t, fresh.ContentType, task.ContentType) + assert.Equal(t, 1, task.AttemptNum) + + require.NotNil(t, task.Body) + assert.Equal(t, fresh.Body, *task.Body) +} + +// TestHandleEventResubmit_IsRepeatable proves the requirement replay +// deliberately does not meet: firing the same captured event at a +// backend over and over must work, with no in-flight refusal, and each +// press must produce its own event and its own delivery. +func TestHandleEventResubmit_IsRepeatable(t *testing.T) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + notif *recordingNotifier + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr, ¬if) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + tgt := seedConfiguredTarget( + t, db, wh.ID, database.TargetTypeHTTP, + `{"url":"`+resubmitTargetURL+`"}`, + ) + original := seedStoredEvent(t, dbMgr, wh.ID, `{"fire":"again"}`) + + // Nothing between the presses marks the earlier deliveries + // finished, so every one of these is submitted while the last is + // still pending. + const presses = 5 + + for range presses { + w := postResubmit(t, h, sess, wh.ID, original.ID) + + require.Equal(t, http.StatusSeeOther, w.Code) + assert.Equal( + t, + "/source/"+wh.ID+"/logs?resubmit=queued", + w.Header().Get("Location"), + "a resubmit must not be refused while an earlier "+ + "one is in flight", + ) + } + + webhookDB, err := dbMgr.GetDB(wh.ID) + require.NoError(t, err) + + events := listEvents(t, webhookDB) + require.Len(t, events, presses+1) + + tasks := notif.Tasks() + require.Len(t, tasks, presses) + + seen := make(map[string]struct{}, presses) + + for _, task := range tasks { + assert.Equal(t, tgt.ID, task.TargetID) + assert.NotEqual( + t, original.ID, task.EventID, + "each resubmit delivers its own new event", + ) + + _, dup := seen[task.EventID] + assert.False(t, dup, "each resubmit creates its own event") + + seen[task.EventID] = struct{}{} + + require.Len(t, listDeliveries(t, webhookDB, task.EventID), 1) + } + + // Every copy names the same source event, so twenty presses stay + // traceable to the one captured request. + for _, e := range events { + if e.ID == original.ID { + continue + } + + require.NotNil(t, e.ResubmittedFromID) + assert.Equal(t, original.ID, *e.ResubmittedFromID) + } +} + +// TestHandleEventResubmit_OversizeBodySurvivesIntact covers the +// non-inline case: a body above delivery.MaxInlineBodySize is not +// carried on the task at all, so it has to be copied into the new +// event row byte-identically for the engine to load it from there. +func TestHandleEventResubmit_OversizeBodySurvivesIntact(t *testing.T) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + notif *recordingNotifier + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr, ¬if) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + // Well over the inline limit, and not text: a multibyte rune, a + // NUL and a byte that is not valid UTF-8, so a copy that went + // through a re-encode or a truncation is visible in the compare. + const sentinel = "TAIL-SENTINEL-1f4a9c" + + stored := strings.Repeat("A", delivery.MaxInlineBodySize) + + "é\x00\xff" + + strings.Repeat("B", 4096) + sentinel + + require.Greater(t, len(stored), delivery.MaxInlineBodySize) + + wh := seedWebhook(t, db) + seedConfiguredTarget( + t, db, wh.ID, database.TargetTypeHTTP, + `{"url":"`+resubmitTargetURL+`"}`, + ) + original := seedStoredEvent(t, dbMgr, wh.ID, stored) + + w := postResubmit(t, h, sess, wh.ID, original.ID) + require.Equal(t, http.StatusSeeOther, w.Code) + + webhookDB, err := dbMgr.GetDB(wh.ID) + require.NoError(t, err) + + fresh := theOtherEvent(t, listEvents(t, webhookDB), original.ID) + + assert.Len(t, fresh.Body, len(stored)) + assert.Equal( + t, stored, fresh.Body, + "the stored body must be copied byte for byte", + ) + + tasks := notif.Tasks() + require.Len(t, tasks, 1) + assert.Nil( + t, tasks[0].Body, + "a body over the inline limit is fetched from the new "+ + "event row rather than carried on the task", + ) + + // The engine's own read of the body, against the new event id: + // what it would send is what was stored. + var loaded database.Event + + require.NoError(t, webhookDB.Select("body"). + First(&loaded, "id = ?", tasks[0].EventID).Error) + assert.Equal(t, stored, loaded.Body) +} + +// TestHandleEventResubmit_SkipsInactiveTarget proves a deactivated +// target is skipped exactly as the receiver skips it — not an error, +// and not a delivery the operator switched off. +func TestHandleEventResubmit_SkipsInactiveTarget(t *testing.T) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + notif *recordingNotifier + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr, ¬if) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + active := seedConfiguredTarget( + t, db, wh.ID, database.TargetTypeHTTP, + `{"url":"`+resubmitTargetURL+`"}`, + ) + off := seedConfiguredTarget( + t, db, wh.ID, database.TargetTypeHTTP, + `{"url":"`+resubmitTargetURL+`/off"}`, + ) + + require.NoError(t, db.DB().Model(&database.Target{}). + Where("id = ?", off.ID). + Update("active", false).Error) + + original := seedStoredEvent(t, dbMgr, wh.ID, `{"skip":"one"}`) + + w := postResubmit(t, h, sess, wh.ID, original.ID) + + require.Equal(t, http.StatusSeeOther, w.Code) + assert.Equal( + t, + "/source/"+wh.ID+"/logs?resubmit=queued", + w.Header().Get("Location"), + "an inactive target is skipped, not an error", + ) + + tasks := notif.Tasks() + require.Len(t, tasks, 1) + assert.Equal(t, active.ID, tasks[0].TargetID) + + webhookDB, err := dbMgr.GetDB(wh.ID) + require.NoError(t, err) + + fresh := theOtherEvent(t, listEvents(t, webhookDB), original.ID) + require.Len(t, listDeliveries(t, webhookDB, fresh.ID), 1) +} + +// TestHandleEventResubmit_NoActiveTargetsStillStoresEvent proves a +// source with nothing to deliver to behaves as the receiver does: the +// event is stored, nothing is queued, and the operator is told so +// rather than being shown an error. +func TestHandleEventResubmit_NoActiveTargetsStillStoresEvent( + t *testing.T, +) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + notif *recordingNotifier + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr, ¬if) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + original := seedStoredEvent(t, dbMgr, wh.ID, `{"no":"targets"}`) + + w := postResubmit(t, h, sess, wh.ID, original.ID) + + require.Equal(t, http.StatusSeeOther, w.Code) + assert.Equal( + t, + "/source/"+wh.ID+"/logs?resubmit=no-targets", + w.Header().Get("Location"), + ) + + assert.Empty(t, notif.Tasks()) + + webhookDB, err := dbMgr.GetDB(wh.ID) + require.NoError(t, err) + + assert.Len(t, listEvents(t, webhookDB), 2) +} + +// TestHandleEventResubmit_RefusesEventOfAnotherWebhook proves the +// route cannot re-inject an event out of a webhook the session's user +// does not own, and reports the same 404 for an id that names nothing. +func TestHandleEventResubmit_RefusesEventOfAnotherWebhook( + t *testing.T, +) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + notif *recordingNotifier + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr, ¬if) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + theirs := seedWebhookFor(t, db, otherTestUserID) + theirEvent := seedStoredEvent(t, dbMgr, theirs.ID, `{"not":"mine"}`) + + mine := seedWebhook(t, db) + seedConfiguredTarget( + t, db, mine.ID, database.TargetTypeHTTP, + `{"url":"`+resubmitTargetURL+`"}`, + ) + seedStoredEvent(t, dbMgr, mine.ID, `{"mine":true}`) + + // Their webhook, as its owner would address it. + w := postResubmit(t, h, sess, theirs.ID, theirEvent.ID) + assert.Equal(t, http.StatusNotFound, w.Code) + + // Their event id, addressed through a webhook the user does own. + w = postResubmit(t, h, sess, mine.ID, theirEvent.ID) + assert.Equal(t, http.StatusNotFound, w.Code) + + // An id that names no event at all. + w = postResubmit(t, h, sess, mine.ID, uuid.NewString()) + assert.Equal(t, http.StatusNotFound, w.Code) + + // A malformed id never reaches the query. + w = postResubmit(t, h, sess, mine.ID, "not-a-uuid") + assert.Equal(t, http.StatusNotFound, w.Code) + + assert.Empty( + t, notif.Tasks(), + "a refused resubmit must queue nothing", + ) +} + +// TestHandleSourceLogs_ShowsResubmitProvenance proves the event log +// reports the relationship in both directions, which is what keeps it +// readable once one captured event has been fired repeatedly. +func TestHandleSourceLogs_ShowsResubmitProvenance(t *testing.T) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + seedConfiguredTarget( + t, db, wh.ID, database.TargetTypeHTTP, + `{"url":"`+resubmitTargetURL+`"}`, + ) + original := seedStoredEvent(t, dbMgr, wh.ID, `{"trace":"me"}`) + + for range 2 { + require.Equal( + t, + http.StatusSeeOther, + postResubmit(t, h, sess, wh.ID, original.ID).Code, + ) + } + + webhookDB, err := dbMgr.GetDB(wh.ID) + require.NoError(t, err) + + events := listEvents(t, webhookDB) + require.Len(t, events, 3) + + body := renderSourceLogsPage(t, h, sess, wh.ID) + + assert.Contains( + t, body, "Resubmitted as 2 new events", + "the source event must show it has been resubmitted", + ) + assert.Contains( + t, body, "Resubmitted from event", + "a copy must show where it came from", + ) + assert.Contains( + t, body, + "/source/"+wh.ID+"/events/"+original.ID+"/resubmit", + "the log must offer the resubmit action per event", + ) +} diff --git a/internal/handlers/export_test.go b/internal/handlers/export_test.go index 8d65e54..4612e94 100644 --- a/internal/handlers/export_test.go +++ b/internal/handlers/export_test.go @@ -1,6 +1,83 @@ package handlers -import "net/http" +import ( + "html/template" + "log/slog" + "net/http" + + "sneak.berlin/go/webhooker/internal/database" +) + +// SetLogForTest replaces the handler's logger, so the handlers_test +// package can assert on what a log line actually contains rather than +// on what it is meant to contain. +func (s *Handlers) SetLogForTest(log *slog.Logger) { + s.log = log +} + +// MaxRenderedBodyBytesForTest exposes the event log's body cap +// to the handlers_test package. +const MaxRenderedBodyBytesForTest = maxRenderedBodyBytes + +// MaxRenderedResponseBytesForTest exposes the event log's +// delivery response cap to the handlers_test package. +const MaxRenderedResponseBytesForTest = maxRenderedResponseBytes + +// MaxRenderedAttemptsForTest exposes the event log's +// per-delivery attempt ceiling to the handlers_test package. +const MaxRenderedAttemptsForTest = maxRenderedAttempts + +// MaxTargetRetriesForTest exposes the target max_retries ceiling to +// the handlers_test package, so the tests assert against the constant +// the handlers enforce rather than a number copied beside it. +const MaxTargetRetriesForTest = maxTargetRetries + +// PageOrFirstForTest exposes pageOrFirst for use in the handlers_test +// package. +func PageOrFirstForTest(s string) int { + return pageOrFirst(s) +} + +// DummyVerificationsForTest reports how many equivalent-cost +// verifications were charged for usernames that do not exist. It +// lets a test prove the anti-enumeration path ran without timing +// anything. +func (s *Handlers) DummyVerificationsForTest() uint64 { + return s.dummyVerifications.Load() +} + +// TrimPartialRuneForTest exposes trimPartialRune for use in the +// handlers_test package. +func TrimPartialRuneForTest(b []byte) []byte { + return trimPartialRune(b) +} + +// LoadEventLogViewsForTest exposes loadEventsWithDeliveries for +// use in the handlers_test package. Assertions on the projected +// body need the bytes as loaded: html/template rewrites invalid +// UTF-8 on the way out, so the rendered page cannot show whether +// a binary body survived the projection intact. +func (s *Handlers) LoadEventLogViewsForTest( + w http.ResponseWriter, + webhook database.Webhook, + page int, +) []EventLogView { + views, _, _ := s.loadEventsWithDeliveries( + w, webhook, nil, page, + ) + + return views +} + +// AddTemplateForTest registers a template under a page name so that +// the handlers_test package can drive the render path with a +// template of its own. +func (s *Handlers) AddTemplateForTest( + pageTemplate string, + tmpl *template.Template, +) { + s.templates[pageTemplate] = tmpl +} // RenderTemplateForTest exposes renderTemplate for use in the // handlers_test package. @@ -13,18 +90,29 @@ func (s *Handlers) RenderTemplateForTest( s.renderTemplate(w, r, pageTemplate, data) } -// BuildSlackTargetConfigForTest exposes buildURLTargetConfig -// with the Slack target parameters for use in the -// handlers_test package. +// BuildSlackTargetConfigForTest exposes +// buildSlackTargetConfig for use in the handlers_test package. func (s *Handlers) BuildSlackTargetConfigForTest( w http.ResponseWriter, r *http.Request, targetURL string, ) (string, error) { - return s.buildURLTargetConfig( - w, r, targetURL, "webhookUrl", - "Webhook URL is required for Slack targets", - ) + return s.buildSlackTargetConfig(w, r, targetURL) +} + +// BuildHTTPTargetConfigForTest exposes buildHTTPTargetConfig +// for use in the handlers_test package, taking the form fields +// an HTTP target's configuration is built from. +func (s *Handlers) BuildHTTPTargetConfigForTest( + w http.ResponseWriter, + r *http.Request, + targetURL, headers, timeout string, +) (string, error) { + return s.buildHTTPTargetConfig(w, r, targetFormInput{ + URL: targetURL, + Headers: headers, + Timeout: timeout, + }) } // BuildDatabaseTargetConfigForTest exposes diff --git a/internal/handlers/footer_version_test.go b/internal/handlers/footer_version_test.go new file mode 100644 index 0000000..58f2078 --- /dev/null +++ b/internal/handlers/footer_version_test.go @@ -0,0 +1,38 @@ +package handlers_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "sneak.berlin/go/webhooker/internal/globals" + "sneak.berlin/go/webhooker/internal/handlers" + "sneak.berlin/go/webhooker/internal/session" +) + +// The footer in base.html falls back to the literal "dev" when the +// template data carries no version, which is what every page rendered +// while nothing supplied one. The operator uses the footer to tell +// which build is live, so it has to carry the stamped value. +func TestFooterReportsStampedVersion(t *testing.T) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + g *globals.Globals + ) + + app := newTestApp(t, &h, &sess, &g) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + g.Version = "v9.9.9-test" + + html := renderPage(t, h, sess, "login.html", map[string]any{ + dataKeyError: "", + }) + + assert.Contains(t, html, "v9.9.9-test") + assert.NotContains(t, html, "dev") +} diff --git a/internal/handlers/gormlogbound_test.go b/internal/handlers/gormlogbound_test.go new file mode 100644 index 0000000..123eb4d --- /dev/null +++ b/internal/handlers/gormlogbound_test.go @@ -0,0 +1,462 @@ +package handlers_test + +import ( + "bytes" + "context" + "io" + "log" + "net/http" + "net/http/httptest" + "os" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/go-chi/chi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + gormlogger "gorm.io/gorm/logger" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/handlers" + "sneak.berlin/go/webhooker/internal/middleware" +) + +// gormBoundTailMarker sits at the far end of every client-chosen value +// this file sends. Its presence in the log means the whole value +// reached the log, so a value that merely happened to be short cannot +// pass for a truncated one. +const gormBoundTailMarker = "ENDOFCLIENTVALUE" + +// gormBoundFills are the characters a client can drive through the +// receiver path segment and the login username, chosen for what a log +// handler charges for them. +// +// The bare C0 control is the one that matters: both handlers spell +// U+0001 as a six-byte escape for the one byte it costs to send, the +// widest multiplier available below U+10000 and the case a raw-byte +// budget breaks on first. GORM's default logger applies no budget at +// all, so under the mutation every one of these arrives whole. +func gormBoundFills() []struct { + name string + fill string +} { + return []struct { + name string + fill string + }{ + {"plain", "x"}, + {"quote", `"`}, + {"backslash", `\`}, + {"tab", "\t"}, + {"newline", "\n"}, + {"c0_control", "\x01"}, + {"astral_nonprintable", "\U0001000C"}, + } +} + +// syncBuf collects captured output from the goroutine draining the +// pipe. +type syncBuf struct { + mu sync.Mutex + b bytes.Buffer +} + +func (s *syncBuf) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + + return s.b.Write(p) +} + +func (s *syncBuf) String() string { + s.mu.Lock() + defer s.mu.Unlock() + + return s.b.String() +} + +func (s *syncBuf) reset() { + s.mu.Lock() + defer s.mu.Unlock() + + s.b.Reset() +} + +// stdoutCapture redirects os.Stdout for the duration of a test. +// +// internal/logger builds its handler over os.Stdout at construction +// time, so redirecting the variable before the application is built +// captures everything the service logger — and therefore the GORM +// adapter, which writes through it — emits. +type stdoutCapture struct { + buf *syncBuf + r *os.File + w *os.File + orig *os.File + done chan struct{} + seq int +} + +func captureStdout(t *testing.T) *stdoutCapture { + t.Helper() + + r, w, err := os.Pipe() + require.NoError(t, err) + + c := &stdoutCapture{ + buf: &syncBuf{}, + r: r, + w: w, + orig: os.Stdout, + done: make(chan struct{}), + } + + os.Stdout = w + + go func() { + defer close(c.done) + + _, _ = io.Copy(c.buf, r) + }() + + t.Cleanup(func() { + os.Stdout = c.orig + _ = w.Close() + + <-c.done + + _ = r.Close() + }) + + return c +} + +// drain returns everything written since the previous drain and +// clears the buffer. +// +// A sentinel is pushed through the same pipe and waited for, so the +// draining goroutine is known to have caught up before the buffer is +// read. Without it the comparison below would race the reader rather +// than measure the writers. +func (c *stdoutCapture) drain(t *testing.T) string { + t.Helper() + + c.seq++ + + sentinel := "\n<>\n" + + _, err := c.w.WriteString(sentinel) + require.NoError(t, err) + + deadline := time.Now().Add(10 * time.Second) + for !strings.Contains(c.buf.String(), sentinel) { + require.False( + t, time.Now().After(deadline), + "timed out waiting for captured output", + ) + time.Sleep(time.Millisecond) + } + + out := strings.Replace(c.buf.String(), sentinel, "", 1) + + c.buf.reset() + + return out +} + +// teeStdout writes to a buffer and to whatever os.Stdout is at the +// moment of the write. +// +// The second half is the point. GORM's package-level default logger +// resolves os.Stdout once, at package init, so a logger built over the +// variable would keep writing to the real terminal no matter what a +// test redirects. Resolving it per write puts the bytes a defaulted +// gorm.Config would cost in production into the same capture as +// everything else internal/logger emits, which is what lets the volume +// assertions below measure the whole writer set rather than one member +// of it. +type teeStdout struct { + buf *syncBuf +} + +func (w teeStdout) Write(p []byte) (int, error) { + _, _ = os.Stdout.Write(p) + + return w.buf.Write(p) +} + +// captureGORMDefault replaces GORM's package-level default logger with +// one configured exactly as GORM configures its own, writing to a +// buffer and to os.Stdout. +// +// This is the mutation detector. gormlogger.Default is what a bare +// &gorm.Config{} installs, and its config here is GORM's verbatim — +// Warn, IgnoreRecordNotFoundError false — so a reverted call site +// behaves as it would in production rather than as a test dialed it. +// With every gorm.Open in this service naming its own logger, nothing +// consults this value and the buffer stays empty; revert any one of +// the three and the interpolated SQL lands here. +func captureGORMDefault(t *testing.T) *syncBuf { + t.Helper() + + buf := &syncBuf{} + orig := gormlogger.Default + + gormlogger.Default = gormlogger.New( + log.New(teeStdout{buf: buf}, "", log.LstdFlags), + gormlogger.Config{ + SlowThreshold: 200 * time.Millisecond, + LogLevel: gormlogger.Warn, + IgnoreRecordNotFoundError: false, + Colorful: false, + }, + ) + + t.Cleanup(func() { gormlogger.Default = orig }) + + return buf +} + +// floodUnauthenticated drives reps requests at each of the two +// unauthenticated lookups that miss by design, for every fill, with a +// client-chosen value of size raw bytes. +func floodUnauthenticated( + t *testing.T, h *handlers.Handlers, size, reps int, +) int { + t.Helper() + + requests := 0 + + for _, f := range gormBoundFills() { + var b strings.Builder + + for b.Len() < size { + b.WriteString(f.fill) + } + + b.WriteString(gormBoundTailMarker) + + value := b.String() + + for range reps { + postWebhook(t, h, value) + postUnknownLogin(t, h, value) + + requests += 2 + } + } + + return requests +} + +// floodPerWebhook drives the same client-chosen values at the second +// gorm.Open site, the per-webhook database internal/database's +// WebhookDBManager opens. +// +// That site is behind authentication in production, so this is not +// part of the unauthenticated flood above and is counted separately. +// It is here because the ceiling the README states covers every +// writer, and the manager is one of them: with nothing driving it, a +// bare &gorm.Config{} could be restored at +// internal/database/webhook_db_manager.go and the whole suite would +// stay green. +func floodPerWebhook( + t *testing.T, mgr *database.WebhookDBManager, size, reps int, +) int { + t.Helper() + + requests := 0 + + for _, f := range gormBoundFills() { + var b strings.Builder + + for b.Len() < size { + b.WriteString(f.fill) + } + + b.WriteString(gormBoundTailMarker) + + value := b.String() + + db, err := mgr.GetDB("pin-" + f.name) + require.NoError(t, err) + + for range reps { + var got database.Event + + err = db.Where("id = ?", value).First(&got).Error + require.ErrorIs(t, err, gorm.ErrRecordNotFound) + + requests++ + } + } + + return requests +} + +// postWebhook drives the receiver with an invented entrypoint path. +// The route pattern matches any single segment, so every byte of the +// value is the client's, and the lookup behind it misses by design. +func postWebhook( + t *testing.T, h *handlers.Handlers, entrypoint string, +) { + t.Helper() + + req := httptest.NewRequestWithContext( + context.Background(), http.MethodPost, "/webhook/x", + strings.NewReader("{}"), + ) + + rctx := chi.NewRouteContext() + rctx.URLParams.Add("uuid", entrypoint) + + req = req.WithContext(context.WithValue( + req.Context(), chi.RouteCtxKey, rctx, + )) + + w := httptest.NewRecorder() + h.HandleWebhook().ServeHTTP(w, req) + + require.Equal(t, http.StatusNotFound, w.Code) +} + +// postUnknownLogin submits the login form with an unknown username, +// through the postLogin helper in logbound_test.go. The field is +// bounded only by the 1 MB body cap, and the lookup behind it misses +// by design. +func postUnknownLogin( + t *testing.T, h *handlers.Handlers, username string, +) { + t.Helper() + + // 401 while the client still has failure budget against this + // username, 429 once the login guard has taken it away. Both + // outcomes sit behind the user lookup, which is the query this + // test is here to drive. + require.Contains( + t, + []int{http.StatusUnauthorized, http.StatusTooManyRequests}, + postLogin(t, h, username), + ) +} + +// assertFloodBounded holds every captured line to the stated ceiling +// and proves nothing carried a whole client value. +func assertFloodBounded(t *testing.T, label, out string) { + t.Helper() + + assert.NotContains( + t, out, gormBoundTailMarker, + "%s: the far end of a client-chosen value reached the "+ + "log, so nothing truncated it", label, + ) + + for line := range strings.SplitSeq( + strings.TrimRight(out, "\n"), "\n", + ) { + if line == "" { + continue + } + + assert.LessOrEqual( + t, len(line), middleware.MaxAccessLogLineBytes, + "%s: log line exceeded its bound: %s", + label, line[:min(len(line), 300)], + ) + } +} + +// TestFlood_NoWriterGrowsWithTheInput is the definition of done for +// the GORM logger defect, stated over every writer at once, for two of +// this service's three gorm.Open sites: the main database behind the +// two unauthenticated lookups, and the per-webhook database the +// WebhookDBManager opens. The third, the archive writer, is pinned in +// internal/delivery, where its type lives. +// +// What each assertion is worth, since two of the three would pass +// against a service that had never been fixed if the capture were set +// up differently: +// +// - The gormDefault check is the sharp one. It fires the moment any +// gorm.Open in this service goes back to a bare &gorm.Config{}. +// - The volume and per-line checks bite only because the replaced +// default logger tees into os.Stdout, so a reverted call site +// shows up in the same capture as everything internal/logger +// writes — the way it would in production. Without that tee both +// were vacuous: at INFO the two handler misses log at DEBUG and +// the adapter drops the record-not-found, so the capture holds +// nothing but fixed-string warnings. +// +// The level is left where newTestApp leaves it, at INFO: the level an +// operator runs at by default, and the one the defect was visible at. +// The handlers' own miss lines sit at DEBUG and spend the same +// logfield budget as everything else, so they are not what makes +// either assertion above bite at any level. +// +// It is deliberately not parallel: it redirects os.Stdout and replaces +// gormlogger.Default, both of which are process-global. Go runs every +// non-parallel top-level test to completion before it resumes the +// parallel ones, so nothing else in this package is running while the +// capture is installed. +// +//nolint:paralleltest // Deliberately sequential; see above. +func TestFlood_NoWriterGrowsWithTheInput(t *testing.T) { + const ( + smallBytes = 128 + bigBytes = 8 << 10 + reps = 5 + ) + + gormDefault := captureGORMDefault(t) + capture := captureStdout(t) + + var ( + h *handlers.Handlers + mgr *database.WebhookDBManager + ) + + app := newTestApp(t, &h, &mgr) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + // Startup chatter is not what this test measures. + capture.drain(t) + + floodUnauthenticated(t, h, smallBytes, reps) + floodPerWebhook(t, mgr, smallBytes, reps) + + small := capture.drain(t) + + requests := floodUnauthenticated(t, h, bigBytes, reps) + requests += floodPerWebhook(t, mgr, bigBytes, reps) + big := capture.drain(t) + + assertFloodBounded(t, "small flood", small) + assertFloodBounded(t, "big flood", big) + + // GORM's default logger is what the defect was. Nothing in this + // service may reach it. + got := gormDefault.String() + assert.Empty( + t, got, + "GORM's default logger wrote %d bytes; the first of them: %s", + len(got), got[:min(len(got), 300)], + ) + + // The same flood, with 64 times the client-chosen input, must not + // buy 64 times the log. A few bytes of slack covers a latency + // field changing width; the input grew by roughly half a megabyte. + const slackPerRequest = 64 + + assert.LessOrEqual( + t, len(big), len(small)+slackPerRequest*requests, + "log volume tracked the size of the client's input: "+ + "%d bytes at %d bytes of input per request, %d bytes "+ + "at %d", + len(small), smallBytes, len(big), bigBytes, + ) +} diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 193856c..349771e 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -3,12 +3,14 @@ package handlers import ( + "bytes" "context" "encoding/json" "errors" "html/template" "log/slog" "net/http" + "sync/atomic" "go.uber.org/fx" "sneak.berlin/go/webhooker/internal/database" @@ -16,6 +18,7 @@ import ( "sneak.berlin/go/webhooker/internal/globals" "sneak.berlin/go/webhooker/internal/healthcheck" "sneak.berlin/go/webhooker/internal/logger" + "sneak.berlin/go/webhooker/internal/metrics" "sneak.berlin/go/webhooker/internal/middleware" "sneak.berlin/go/webhooker/internal/session" "sneak.berlin/go/webhooker/templates" @@ -26,8 +29,6 @@ const ( maxBodyShift = 20 // recentEventLimit is the number of recent events to show. recentEventLimit = 20 - // defaultRetentionDays is the default event retention period. - defaultRetentionDays = 30 // paginationPerPage is the number of items per page. paginationPerPage = 25 @@ -40,6 +41,12 @@ const ( // errInvalidPassword is returned when a password does not match. var errInvalidPassword = errors.New("invalid password") +// errVerificationBusy is returned when no password-verification slot +// became free before the wait elapsed, so no password was verified. +var errVerificationBusy = errors.New( + "password verification capacity exhausted", +) + //nolint:revive // HandlersParams is a standard fx naming convention. type HandlersParams struct { fx.In @@ -50,7 +57,10 @@ type HandlersParams struct { WebhookDBMgr *database.WebhookDBManager Healthcheck *healthcheck.Healthcheck Session *session.Session + Middleware *middleware.Middleware Notifier delivery.Notifier + Evictor delivery.WebhookEvictor + SSRFGuard *delivery.Guard } // Handlers provides HTTP handler methods for all application @@ -62,8 +72,21 @@ type Handlers struct { db *database.Database dbMgr *database.WebhookDBManager session *session.Session + mw *middleware.Middleware notifier delivery.Notifier + evictor delivery.WebhookEvictor + mtr *metrics.Set templates map[string]*template.Template + + // ssrf validates submitted target URLs. It is the same guard + // the delivery engine dials through, so a URL accepted here + // is one delivery will actually attempt. + ssrf *delivery.Guard + + // dummyVerifications counts the equivalent-cost verifications + // charged for usernames that do not exist. It exists so a test + // can prove that path runs without measuring wall-clock time. + dummyVerifications atomic.Uint64 } // parsePageTemplate parses a page-specific template set from the @@ -96,7 +119,11 @@ func New( s.db = params.Database s.dbMgr = params.WebhookDBMgr s.session = params.Session + s.mw = params.Middleware s.notifier = params.Notifier + s.evictor = params.Evictor + s.mtr = metrics.Default() + s.ssrf = params.SSRFGuard // Parse all page templates once at startup s.templates = map[string]*template.Template{ @@ -107,6 +134,7 @@ func New( "source_detail.html": parsePageTemplate("source_detail.html"), "source_edit.html": parsePageTemplate("source_edit.html"), "source_logs.html": parsePageTemplate("source_logs.html"), + "target_edit.html": parsePageTemplate("target_edit.html"), } lc.Append(fx.Hook{ @@ -156,6 +184,7 @@ type UserInfo struct { type templateDataWrapper struct { User *UserInfo CSRFToken string + Version string Data any } @@ -206,9 +235,16 @@ func (s *Handlers) renderTemplate( userInfo := s.getUserInfo(r) csrfToken := middleware.CSRFToken(r) + // The footer in base.html renders .Version. Every page reaches it + // through here, so this is the one place that has to supply it; + // left unset, the footer falls back to its literal "dev" and the + // UI reports a build that is not the one running. + version := s.params.Globals.Version + if m, ok := data.(map[string]any); ok { m["User"] = userInfo m["CSRFToken"] = csrfToken + m["Version"] = version s.executeTemplate(w, tmpl, m) return @@ -217,19 +253,29 @@ func (s *Handlers) renderTemplate( wrapper := templateDataWrapper{ User: userInfo, CSRFToken: csrfToken, + Version: version, Data: data, } s.executeTemplate(w, tmpl, wrapper) } -// executeTemplate runs the template and handles errors. +// executeTemplate renders the template into a buffer and writes to +// the response only once rendering has fully succeeded. Executing +// straight into the ResponseWriter commits a partial body and a 200 +// status before a mid-render error can be reported, leaving no way +// to serve a 500. Buffering makes a page's rendered size resident +// memory per concurrent viewer, so every page owes it a bound: the +// event log caps each stored body at maxRenderedBodyBytes for exactly +// this reason. func (s *Handlers) executeTemplate( w http.ResponseWriter, tmpl *template.Template, data any, ) { - err := tmpl.Execute(w, data) + var buf bytes.Buffer + + err := tmpl.Execute(&buf, data) if err != nil { s.log.Error( "failed to execute template", "error", err, @@ -238,5 +284,16 @@ func (s *Handlers) executeTemplate( w, "Internal server error", http.StatusInternalServerError, ) + + return + } + + w.Header().Set("Content-Type", "text/html; charset=utf-8") + + _, err = buf.WriteTo(w) + if err != nil { + s.log.Error( + "failed to write rendered page", "error", err, + ) } } diff --git a/internal/handlers/handlers_test.go b/internal/handlers/handlers_test.go index 8aae54a..3e9874a 100644 --- a/internal/handlers/handlers_test.go +++ b/internal/handlers/handlers_test.go @@ -2,8 +2,11 @@ package handlers_test import ( "context" + "errors" + "html/template" "net/http" "net/http/httptest" + "sync" "testing" "github.com/stretchr/testify/assert" @@ -17,12 +20,62 @@ import ( "sneak.berlin/go/webhooker/internal/handlers" "sneak.berlin/go/webhooker/internal/healthcheck" "sneak.berlin/go/webhooker/internal/logger" + "sneak.berlin/go/webhooker/internal/middleware" "sneak.berlin/go/webhooker/internal/session" ) -type noopNotifier struct{} +// recordingNotifier is a delivery.Notifier that records the tasks it +// was handed, so a test can prove a handler queued the delivery it +// claims to have queued — and, on the refusal paths, that it queued +// nothing. +type recordingNotifier struct { + mu sync.Mutex + tasks []delivery.Task +} -func (n *noopNotifier) Notify([]delivery.Task) {} +func (n *recordingNotifier) Notify(tasks []delivery.Task) { + n.mu.Lock() + defer n.mu.Unlock() + + n.tasks = append(n.tasks, tasks...) +} + +// Tasks returns a copy of the recorded tasks. +func (n *recordingNotifier) Tasks() []delivery.Task { + n.mu.Lock() + defer n.mu.Unlock() + + out := make([]delivery.Task, len(n.tasks)) + copy(out, n.tasks) + + return out +} + +// recordingEvictor is a delivery.WebhookEvictor that records +// the webhook ids it was asked to evict, so a test can prove +// that a deletion path reached the delivery engine. +type recordingEvictor struct { + mu sync.Mutex + evicted []string +} + +func (r *recordingEvictor) EvictWebhook(webhookID string) { + r.mu.Lock() + defer r.mu.Unlock() + + r.evicted = append(r.evicted, webhookID) +} + +// Evicted returns a copy of the recorded webhook ids. +func (r *recordingEvictor) Evicted() []string { + r.mu.Lock() + defer r.mu.Unlock() + + out := make([]string, len(r.evicted)) + copy(out, r.evicted) + + return out +} func newTestApp( t *testing.T, @@ -44,9 +97,20 @@ func newTestApp( database.NewWebhookDBManager, healthcheck.New, session.New, - func() delivery.Notifier { - return &noopNotifier{} + func() *recordingNotifier { + return &recordingNotifier{} }, + func(n *recordingNotifier) delivery.Notifier { + return n + }, + func() *recordingEvictor { + return &recordingEvictor{} + }, + func(r *recordingEvictor) delivery.WebhookEvictor { + return r + }, + middleware.New, + delivery.NewGuard, handlers.New, ), fx.Populate(targets...), @@ -187,6 +251,68 @@ func TestRenderTemplate(t *testing.T) { ) } +// errMidRender is the failure a test template raises partway through +// rendering. +var errMidRender = errors.New("deliberate mid-render failure") + +// midRenderFailure is template data whose first method renders and +// whose second fails, so the template aborts after output has +// already been produced. +type midRenderFailure struct{} + +// Prefix is the output a streaming renderer would flush before the +// failure below aborts the template. +func (midRenderFailure) Prefix() string { return partialPageMarker } + +// Boom aborts template execution. +func (midRenderFailure) Boom() (string, error) { + return "", errMidRender +} + +// partialPageMarker is content the failing template emits before it +// aborts. +const partialPageMarker = "PARTIAL PAGE CONTENT" + +// TestRenderTemplateMidRenderErrorSendsNoPartialBody proves the +// renderer does not commit output it cannot finish: a template that +// fails partway through must yield a 500 and a body carrying none of +// the content emitted before the failure. Against a renderer that +// executes straight into the ResponseWriter this fails on both +// counts, returning 200 with the prefix already flushed. +func TestRenderTemplateMidRenderErrorSendsNoPartialBody(t *testing.T) { + t.Parallel() + + var h *handlers.Handlers + + app := newTestApp(t, &h) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + h.AddTemplateForTest("failing.html", template.Must( + template.New("failing").Parse( + `{{.Data.Prefix}}{{.Data.Boom}}TAIL`, + ), + )) + + req := httptest.NewRequestWithContext( + context.Background(), http.MethodGet, "/", nil) + w := httptest.NewRecorder() + + h.RenderTemplateForTest( + w, req, "failing.html", midRenderFailure{}, + ) + + assert.Equal( + t, http.StatusInternalServerError, w.Code, + "a failed render must report a 500", + ) + assert.Equal( + t, "Internal server error\n", w.Body.String(), + "the response must carry no part of the aborted page", + ) +} + func TestBuildDatabaseTargetConfig_Valid(t *testing.T) { t.Parallel() diff --git a/internal/handlers/logbound_test.go b/internal/handlers/logbound_test.go new file mode 100644 index 0000000..abb78f4 --- /dev/null +++ b/internal/handlers/logbound_test.go @@ -0,0 +1,543 @@ +package handlers_test + +// The handler-side half of the log-field audit. Two slog calls in +// this package reach a value an UNAUTHENTICATED client picks outright +// and of a length it picks outright: +// +// - the unknown-entrypoint DEBUG line on /webhook/{uuid}, whose +// path segment matched no stored entrypoint and so is bounded by +// nothing; +// - the failed-login DEBUG lines, whose username is a form field. +// +// Both are at DEBUG, which is off in production by default. That is +// not a bound: an operator turning DEBUG on to diagnose a flood must +// not thereby hand the flood an unbounded write. Both spend the same +// internal/logfield budget as the access log, and both are held here +// to middleware.MaxAccessLogLineBytes. +// +// The two login lines past the username lookup — "invalid password" +// and "user logged in" — carry the same cap without needing it, since +// by then the value is a stored row rather than the client's. They are +// pinned here too, so the caps cannot be dropped silently. +// +// So is the "password verification capacity exhausted" WARN line, +// whose path chi pins to the constant "/pages/login" on the one route +// that reaches it. Its cap is defensive, and the test below drives the +// handler directly with the path a parameterised route would give it, +// because an unasserted cap is one a later edit removes for free. + +import ( + "bytes" + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/go-chi/chi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/handlers" + "sneak.berlin/go/webhooker/internal/middleware" +) + +// floodRequests is the number of distinct invented values each flood +// drives through the call site under test. +const floodRequests = 32 + +// oversizedFillBytes is the length of the single client-chosen value +// used to show that line size does not track input size. +const oversizedFillBytes = 8192 + +// attackerMarker and tailMarker sit at the END of every oversized +// value, past every budget. Their absence from the log is what +// proves the value was cut rather than merely being short. +const ( + attackerMarker = "QQATTACKERTEXTQQ" + tailMarker = "QQTRUNCATEDTAILQQ" +) + +// escapeFills are the characters the log handlers escape, so a value +// built out of them costs more on the line than it did on the wire. A +// budget counted in raw bytes passes the plain case and fails these. +// +// U+1000C is unassigned, hence non-printable, and strconv.Quote +// spells it as a ten-byte \UXXXXXXXX while the JSON handler passes +// its four UTF-8 bytes through; only the text shape of these tests +// reaches that charge. +func escapeFills() map[string]string { + return map[string]string{ + "plain": "x", + "quote": `"`, + "backslash": `\`, + "tab": "\t", + "newline": "\n", + // A C0 control neither handler has a short escape for, so + // each one costs six bytes on the line against the single + // byte it cost to send: the widest multiplier a client can + // drive, and the case a raw-byte budget breaks on first. + // + // This fill is load-bearing, not decoration. Budgeting raw + // bytes instead of encoded is caught by this fill alone, + // and only under the JSON handler, at 3,072 bytes against + // the 2,560 ceiling. Drop it and that mutation passes. + "control": "\x01", + "astral": "\U0001000C", + } +} + +// logHandlers are the two handlers internal/logger can install. +func logHandlers() map[string]func( + io.Writer, *slog.HandlerOptions, +) slog.Handler { + return map[string]func( + io.Writer, *slog.HandlerOptions, + ) slog.Handler{ + "json": func( + w io.Writer, o *slog.HandlerOptions, + ) slog.Handler { + return slog.NewJSONHandler(w, o) + }, + "text": func( + w io.Writer, o *slog.HandlerOptions, + ) slog.Handler { + return slog.NewTextHandler(w, o) + }, + } +} + +// oversizedFill builds an 8 KB client-chosen value out of +// repetitions of ch, with both markers at its far end. +func oversizedFill(ch string) string { + return "x" + strings.Repeat(ch, oversizedFillBytes) + + attackerMarker + tailMarker +} + +// capturingHandlers builds a Handlers whose log is captured into the +// returned buffer at DEBUG through the named handler. +// +// extra is passed to fx.Populate alongside the Handlers, for the call +// sites that also need the database the client's value is looked up +// in, or the Middleware whose resource has to be exhausted before the +// branch under test is reached. +func capturingHandlers( + t *testing.T, + newHandler func(io.Writer, *slog.HandlerOptions) slog.Handler, + extra ...any, +) (*handlers.Handlers, *bytes.Buffer) { + t.Helper() + + var h *handlers.Handlers + + app := newTestApp(t, append([]any{&h}, extra...)...) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + buf := new(bytes.Buffer) + h.SetLogForTest(slog.New(newHandler( + buf, &slog.HandlerOptions{Level: slog.LevelDebug}, + ))) + + return h, buf +} + +// logLines splits the captured buffer into non-empty lines, holding +// each to the stated per-line ceiling. +func logLines(t *testing.T, buf *bytes.Buffer) []string { + t.Helper() + + var lines []string + + for line := range strings.SplitSeq( + strings.TrimSpace(buf.String()), "\n", + ) { + if line == "" { + continue + } + + require.LessOrEqual( + t, len(line), middleware.MaxAccessLogLineBytes, + "log line exceeded its bound: %s", line, + ) + + lines = append(lines, line) + } + + return lines +} + +// assertNoClientText fails if the far end of the client-chosen input +// survived into the log. +func assertNoClientText(t *testing.T, buf *bytes.Buffer) { + t.Helper() + + assert.NotContains( + t, buf.String(), attackerMarker, + "log carried attacker-chosen text", + ) + assert.NotContains( + t, buf.String(), tailMarker, + "log carried the tail of the attacker-chosen text", + ) +} + +// receiverRouter mounts the real receiver handler at the production +// route pattern. +func receiverRouter(h *handlers.Handlers) *chi.Mux { + router := chi.NewRouter() + router.Post("/webhook/{uuid}", h.HandleWebhook()) + + return router +} + +// postReceiver sends one POST at /webhook/. +// +// RawPath is cleared after parsing so chi routes on the decoded path +// and the handler sees the raw bytes rather than their percent-escaped +// spelling. That is the harder case for the budget: the escaped +// spelling is plain ASCII, which costs one byte per byte, while the +// decoded bytes are what the log handler has to escape. +func postReceiver( + t *testing.T, router *chi.Mux, segment string, +) int { + t.Helper() + + req := httptest.NewRequestWithContext( + context.Background(), + http.MethodPost, + "/webhook/"+url.PathEscape(segment), + strings.NewReader(""), + ) + req.URL.RawPath = "" + + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + return w.Code +} + +// postLogin submits the login form with the given username and a +// non-empty password. +func postLogin( + t *testing.T, h *handlers.Handlers, username string, +) int { + t.Helper() + + return postLoginWithPassword(t, h, username, "not-the-password") +} + +// postLoginWithPassword submits the login form with both credentials +// chosen by the caller, so a test can reach the branches past the +// username lookup. +func postLoginWithPassword( + t *testing.T, h *handlers.Handlers, username, password string, +) int { + t.Helper() + + form := url.Values{ + "username": {username}, + "password": {password}, + } + + req := httptest.NewRequestWithContext( + context.Background(), + http.MethodPost, + "/pages/login", + strings.NewReader(form.Encode()), + ) + req.Header.Set( + "Content-Type", "application/x-www-form-urlencoded", + ) + + w := httptest.NewRecorder() + h.HandleLoginSubmit().ServeHTTP(w, req) + + return w.Code +} + +// TestUnknownEntrypoint_LogLineDoesNotTrackPathSize drives 8 KB of +// client-chosen path at the unauthenticated receiver's +// unknown-entrypoint DEBUG line and holds it to the same ceiling the +// access log states. +func TestUnknownEntrypoint_LogLineDoesNotTrackPathSize(t *testing.T) { + t.Parallel() + + for handlerName, newHandler := range logHandlers() { + for fillName, fill := range escapeFills() { + t.Run(handlerName+"/"+fillName, func(t *testing.T) { + t.Parallel() + + h, buf := capturingHandlers(t, newHandler) + router := receiverRouter(h) + + for i := range floodRequests { + assert.Equal( + t, + http.StatusNotFound, + postReceiver( + t, router, + oversizedFill(fill)+ + strings.Repeat("y", i), + ), + ) + } + + lines := logLines(t, buf) + require.Len(t, lines, floodRequests) + assertNoClientText(t, buf) + + assertBoundedFlood(t, buf.Len()) + }) + } + } +} + +// TestFailedLogin_LogLineDoesNotTrackUsernameSize drives 8 KB of +// client-chosen username at the unauthenticated login endpoint's +// DEBUG line and holds it to the same ceiling. +func TestFailedLogin_LogLineDoesNotTrackUsernameSize(t *testing.T) { + t.Parallel() + + for handlerName, newHandler := range logHandlers() { + for fillName, fill := range escapeFills() { + t.Run(handlerName+"/"+fillName, func(t *testing.T) { + t.Parallel() + + h, buf := capturingHandlers(t, newHandler) + + for i := range floodRequests { + assert.Equal( + t, + http.StatusUnauthorized, + postLogin( + t, h, + oversizedFill(fill)+ + strings.Repeat("y", i), + ), + ) + } + + lines := logLines(t, buf) + require.Len(t, lines, floodRequests) + assertNoClientText(t, buf) + + assertBoundedFlood(t, buf.Len()) + }) + } + } +} + +// storedUserPassword is the password held by the oversize accounts +// the test below creates. +const storedUserPassword = "correct-horse-battery-staple" + +// storedFillBytes is the raw length of the client-chosen value in +// those accounts' usernames. It is well past the 512-byte field +// budget, so the line is still truncated, but short enough that the +// session cookie a successful login writes stays inside +// securecookie's 4 KB limit: the cookie is written BEFORE the +// "user logged in" line, so an 8 KB username answers 500 and never +// reaches it. +const storedFillBytes = 1024 + +// storedFill builds a username fill of storedFillBytes raw bytes out +// of repetitions of ch, with both markers at its far end. +func storedFill(ch string) string { + return "x" + strings.Repeat(ch, storedFillBytes/len(ch)) + + attackerMarker + tailMarker +} + +// TestStoredUsername_LogLinesDoNotTrackUsernameSize pins the two +// login lines that are reached only AFTER the username matched a +// stored row: "invalid password" and "user logged in". Neither +// strictly needs its cap — the value is the operator's own data by +// then, not the client's — but both carry one so that every username +// this unauthenticated endpoint logs is capped, and an unasserted cap +// is one a later edit removes for free. +// +// One app per handler with the accounts created inside it, and no +// parallelism below that level: every account costs an Argon2id hash +// and every attempt costs a verification. +func TestStoredUsername_LogLinesDoNotTrackUsernameSize(t *testing.T) { + t.Parallel() + + for handlerName, newHandler := range logHandlers() { + t.Run(handlerName, func(t *testing.T) { + t.Parallel() + + var db *database.Database + + h, buf := capturingHandlers(t, newHandler, &db) + + hash, err := database.HashPassword(storedUserPassword) + require.NoError(t, err) + + fills := escapeFills() + for fillName, fill := range fills { + username := storedFill(fill) + fillName + + require.NoError(t, db.DB().Create(&database.User{ + Username: username, + Password: hash, + }).Error) + + // Matched the row, wrong secret: "invalid + // password". + assert.Equal( + t, http.StatusUnauthorized, + postLoginWithPassword( + t, h, username, "not-the-password", + ), + ) + + // Matched the row, right secret: "user logged + // in". + assert.Equal( + t, http.StatusSeeOther, + postLoginWithPassword( + t, h, username, storedUserPassword, + ), + ) + } + + lines := logLines(t, buf) + require.Len(t, lines, 2*len(fills)) + assertNoClientText(t, buf) + }) + } +} + +// maxVerificationSlots bounds how many slots the loop below will +// take before it gives up, so a semaphore that never fills fails the +// test instead of hanging it. It is deliberately larger than the +// real concurrency bound, which is not exported to this package. +const maxVerificationSlots = 64 + +// canceledContext returns a context that is already done. A +// verification request carrying one takes the ctx.Done() branch of +// the semaphore's bounded wait immediately, so these cases turn on +// the semaphore being full rather than on a five-second timer firing. +// Nothing here is timing-dependent. +func canceledContext() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + return ctx +} + +// holdEveryVerificationSlot takes verification slots until one is +// refused, and releases them when the test ends. A free slot is +// handed out before any context is consulted, so a canceled context +// cannot make this loop stop early: it stops exactly when the slots +// are gone. +func holdEveryVerificationSlot( + t *testing.T, mw *middleware.Middleware, +) { + t.Helper() + + for range maxVerificationSlots { + release, ok := mw.BeginPasswordVerification(canceledContext()) + if !ok { + return + } + + t.Cleanup(release) + } + + require.Fail(t, "the verification semaphore never filled") +} + +// postLoginAtPath submits the login form at a path of the caller's +// choosing, with a canceled context. +func postLoginAtPath( + t *testing.T, h *handlers.Handlers, path string, +) int { + t.Helper() + + form := url.Values{ + "username": {"someone"}, + "password": {"not-the-password"}, + } + + req := httptest.NewRequestWithContext( + canceledContext(), + http.MethodPost, + path, + strings.NewReader(form.Encode()), + ) + req.Header.Set( + "Content-Type", "application/x-www-form-urlencoded", + ) + + w := httptest.NewRecorder() + h.HandleLoginSubmit().ServeHTTP(w, req) + + return w.Code +} + +// TestVerificationCapacity_LogLineDoesNotTrackPathSize pins the cap +// on the "password verification capacity exhausted" WARN line. +// +// The one route that reaches it is chi's static "/pages/login", so no +// request through the mux can widen the line; the handler is driven +// directly here with the path a parameterised route would give it, +// which is what that cap exists for. Without this test, removing the +// logfield.Truncate there fails nothing. +func TestVerificationCapacity_LogLineDoesNotTrackPathSize( + t *testing.T, +) { + t.Parallel() + + for handlerName, newHandler := range logHandlers() { + for fillName, fill := range escapeFills() { + t.Run(handlerName+"/"+fillName, func(t *testing.T) { + t.Parallel() + + var mw *middleware.Middleware + + h, buf := capturingHandlers(t, newHandler, &mw) + + holdEveryVerificationSlot(t, mw) + + assert.Equal( + t, + http.StatusServiceUnavailable, + postLoginAtPath( + t, h, + "/source/"+url.PathEscape( + oversizedFill(fill), + )+"/login", + ), + ) + + lines := logLines(t, buf) + require.Len(t, lines, 1) + assertNoClientText(t, buf) + }) + } + } +} + +// assertBoundedFlood holds the whole flood's log output to what the +// stated per-line ceiling allows. The flood sent +// floodRequests * oversizedFillBytes bytes of client-chosen text; +// this is the assertion that the log did not grow with it. +func assertBoundedFlood(t *testing.T, got int) { + t.Helper() + + sent := floodRequests * oversizedFillBytes + + require.Less( + t, got, sent/2, + "log volume tracked the size of the flood's input", + ) + require.LessOrEqual( + t, got, + floodRequests*middleware.MaxAccessLogLineBytes, + ) +} diff --git a/internal/handlers/profile.go b/internal/handlers/profile.go index abf39b9..9d929b4 100644 --- a/internal/handlers/profile.go +++ b/internal/handlers/profile.go @@ -1,6 +1,7 @@ package handlers import ( + "context" "net/http" "github.com/go-chi/chi" @@ -31,9 +32,8 @@ func (h *Handlers) HandlePasswordChange() http.HandlerFunc { return } - // Limit request body to prevent memory exhaustion. - r.Body = http.MaxBytesReader(w, r.Body, 1<]*>([^<]*)`, +) + +// baseURLFixture is one started app plus the webhook whose +// entrypoint URL the BaseURL cases read. +type baseURLFixture struct { + handlers *handlers.Handlers + session *session.Session + webhook string + path string +} + +// newBaseURLFixture starts the app and seeds a webhook with one +// entrypoint. +func newBaseURLFixture(t *testing.T) *baseURLFixture { + t.Helper() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + ) + + app := newTestApp(t, &h, &sess, &db) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + seedEntrypoint(t, db, wh.ID) + + return &baseURLFixture{ + handlers: h, + session: sess, + webhook: wh.ID, + path: "ep-" + wh.ID, + } +} + +// entrypointURL renders the source detail page for the fixture's +// webhook over a request the caller shapes, and returns the +// entrypoint URL as an operator would copy it. +func (f *baseURLFixture) entrypointURL( + t *testing.T, + host string, + shape func(*http.Request), +) string { + t.Helper() + + req := httptest.NewRequestWithContext( + context.Background(), + http.MethodGet, + "/source/"+f.webhook, + nil, + ) + req.Host = host + + shape(req) + + for _, c := range authenticatedCookies( + t, f.session, deleteTestUserID, deleteTestUsername, + ) { + req.AddCookie(c) + } + + rctx := chi.NewRouteContext() + rctx.URLParams.Add(paramSourceID, f.webhook) + req = req.WithContext( + context.WithValue( + req.Context(), chi.RouteCtxKey, rctx, + ), + ) + + w := httptest.NewRecorder() + f.handlers.HandleSourceDetail().ServeHTTP(w, req) + + require.Equal(t, http.StatusOK, w.Code) + + match := entrypointURLPattern.FindStringSubmatch(w.Body.String()) + require.Len( + t, match, 2, + "the page must render exactly one entrypoint URL", + ) + + return match[1] +} + +// forwardedProto returns a request shaper setting +// X-Forwarded-Proto, or leaving the request alone for "". +func forwardedProto(value string) func(*http.Request) { + return func(r *http.Request) { + if value == "" { + return + } + + r.Header.Set("X-Forwarded-Proto", value) + } +} + +// baseURLCase is one X-Forwarded-Proto spelling and the scheme +// the rendered entrypoint URL owes it. +type baseURLCase struct { + name string + header string + scheme string + why string +} + +// baseURLCases enumerate the spellings a proxy really emits. The +// scheme is only ever http or https: the header value itself is +// never a scheme, however it is spelled. +func baseURLCases() []baseURLCase { + return []baseURLCase{ + { + name: "lowercase", + header: schemeHTTPS, + scheme: schemeHTTPS, + why: "the ordinary spelling", + }, + { + name: "uppercase", + header: "HTTPS", + scheme: schemeHTTPS, + why: "the token is case-insensitive; the scheme " + + "in a copyable URL is not", + }, + { + name: "chain with plaintext inner hop", + header: "https, http", + scheme: schemeHTTPS, + why: "a chained proxy appends its hop; the " + + "leftmost element faces the client", + }, + { + name: "chain of two TLS hops", + header: "https,https", + scheme: schemeHTTPS, + why: "appended chain with no space after the comma", + }, + { + name: "trailing space", + header: "https ", + scheme: schemeHTTPS, + why: "whitespace is not part of the token", + }, + { + name: "plaintext", + header: schemeHTTP, + scheme: schemeHTTP, + why: "the negative control: the proxy reports plaintext", + }, + { + name: "no header", + header: "", + scheme: schemeHTTP, + why: "a plaintext request asserting nothing is http", + }, + { + name: "garbage token", + header: "javascript:alert(1)//", + scheme: schemeHTTP, + why: "anything that is not https is not TLS, and " + + "the token never becomes the scheme", + }, + } +} + +// TestSourceDetailBaseURL_ForwardedProtoSpellings is the +// regression test for the entrypoint URL an operator pastes into +// the sending system: a header spelling that used to land in the +// scheme verbatim produced a URL no sender could deliver to. +func TestSourceDetailBaseURL_ForwardedProtoSpellings(t *testing.T) { + t.Parallel() + + const host = "hooks.example.com" + + fixture := newBaseURLFixture(t) + + for _, tc := range baseURLCases() { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + assert.Equal( + t, + tc.scheme+"://"+host+"/webhook/"+fixture.path, + fixture.entrypointURL( + t, host, forwardedProto(tc.header), + ), + "X-Forwarded-Proto %q: %s", tc.header, tc.why, + ) + }) + } +} + +// TestSourceDetailBaseURL_DirectTLSBeatsPlaintextHeader pins the +// precedence the old code had backwards: it let any present +// header overwrite what the connection itself proved, so a +// direct-TLS request behind a proxy reporting http rendered an +// http URL. +func TestSourceDetailBaseURL_DirectTLSBeatsPlaintextHeader( + t *testing.T, +) { + t.Parallel() + + const host = "hooks.example.com" + + fixture := newBaseURLFixture(t) + + got := fixture.entrypointURL(t, host, func(r *http.Request) { + r.TLS = &tls.ConnectionState{} + r.Header.Set("X-Forwarded-Proto", "http") + }) + + assert.Equal( + t, + "https://"+host+"/webhook/"+fixture.path, + got, + "a connection this process terminated with TLS "+ + "outranks a header claiming plaintext", + ) +} + +// TestSourceDetailBaseURL_KeepsHostAuthority pins the host half +// of the URL: it is taken from the request unchanged, so the +// deployments that do not sit on port 443 still get a URL that +// works. Constraining the host would break exactly these. +func TestSourceDetailBaseURL_KeepsHostAuthority(t *testing.T) { + t.Parallel() + + fixture := newBaseURLFixture(t) + + hosts := []string{ + "hooks.example.com:8443", + "[2001:db8::1]:8443", + "internal-host", + } + + for _, host := range hosts { + t.Run(host, func(t *testing.T) { + t.Parallel() + + assert.Equal( + t, + "https://"+host+"/webhook/"+fixture.path, + fixture.entrypointURL( + t, host, forwardedProto("HTTPS"), + ), + "the authority must survive verbatim, port and all", + ) + }) + } +} diff --git a/internal/handlers/source_detail_test.go b/internal/handlers/source_detail_test.go new file mode 100644 index 0000000..fcaf698 --- /dev/null +++ b/internal/handlers/source_detail_test.go @@ -0,0 +1,228 @@ +package handlers_test + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/go-chi/chi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm/clause" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/handlers" + "sneak.berlin/go/webhooker/internal/session" +) + +// The secret path segments of a Slack incoming webhook URL. +// Holding them is enough to post to the channel forever, so +// they must never reach the rendered page. +const ( + slackSecretPath = "/services/T00000000/B00000000/" + + "XXXXXXXXXXXXXXXXXXXXXXXX" + slackWebhookURL = "https://hooks.slack.com" + + slackSecretPath +) + +// seedConfiguredTarget inserts a target with a stored config +// blob and returns it. +func seedConfiguredTarget( + t *testing.T, + db *database.Database, + webhookID string, + targetType database.TargetType, + config string, +) *database.Target { + t.Helper() + + tgt := &database.Target{ + WebhookID: webhookID, + Name: "t-" + string(targetType), + Type: targetType, + Active: true, + Config: config, + } + + require.NoError( + t, + db.DB().Omit(clause.Associations).Create(tgt).Error, + ) + + return tgt +} + +// renderSourceDetailPage runs the real source detail handler +// for a webhook and returns the rendered HTML. +func renderSourceDetailPage( + t *testing.T, + h *handlers.Handlers, + sess *session.Session, + webhookID string, +) string { + t.Helper() + + req := httptest.NewRequestWithContext( + context.Background(), + http.MethodGet, + "/source/"+webhookID, + nil, + ) + + for _, c := range authenticatedCookies( + t, sess, deleteTestUserID, deleteTestUsername, + ) { + req.AddCookie(c) + } + + rctx := chi.NewRouteContext() + rctx.URLParams.Add(paramSourceID, webhookID) + + req = req.WithContext( + context.WithValue( + req.Context(), chi.RouteCtxKey, rctx, + ), + ) + + w := httptest.NewRecorder() + h.HandleSourceDetail().ServeHTTP(w, req) + + require.Equal(t, http.StatusOK, w.Code) + + return w.Body.String() +} + +// TestHandleSourceDetail_MasksSlackWebhookURL is the +// load-bearing regression test for the credential leak: the +// rendered page must show the Slack target without any of the +// secret path segments of its webhook URL. +func TestHandleSourceDetail_MasksSlackWebhookURL(t *testing.T) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + ) + + app := newTestApp(t, &h, &sess, &db) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + seedConfiguredTarget( + t, db, wh.ID, + database.TargetTypeSlack, + `{"webhookUrl":"`+slackWebhookURL+`"}`, + ) + + body := renderSourceDetailPage(t, h, sess, wh.ID) + + assert.NotContains(t, body, slackSecretPath) + assert.NotContains(t, body, "T00000000") + assert.NotContains(t, body, "B00000000") + assert.NotContains( + t, body, "XXXXXXXXXXXXXXXXXXXXXXXX", + ) + assert.NotContains(t, body, "webhookUrl") + + assert.Contains(t, body, "Webhook URL") + assert.Contains(t, body, "https://hooks.slack.com/...") +} + +// TestHandleSourceDetail_MasksHTTPDestinationURL is the +// regression test for the same leak reached through the http +// target: its destination is routinely an incoming-webhook +// endpoint whose path segments are the credential, so the +// rendered page must not contain them. +func TestHandleSourceDetail_MasksHTTPDestinationURL( + t *testing.T, +) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + ) + + app := newTestApp(t, &h, &sess, &db) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + seedConfiguredTarget( + t, db, wh.ID, + database.TargetTypeHTTP, + `{"url":"`+slackWebhookURL+`"}`, + ) + + body := renderSourceDetailPage(t, h, sess, wh.ID) + + assert.NotContains(t, body, slackSecretPath) + assert.NotContains(t, body, "T00000000") + assert.NotContains(t, body, "B00000000") + assert.NotContains( + t, body, "XXXXXXXXXXXXXXXXXXXXXXXX", + ) + + assert.Contains(t, body, "Destination URL") + assert.Contains(t, body, "https://hooks.slack.com/...") +} + +// TestHandleSourceDetail_RendersNamedTargetFields proves the +// other target types render labelled fields rather than the +// stored blob. +func TestHandleSourceDetail_RendersNamedTargetFields( + t *testing.T, +) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + ) + + app := newTestApp(t, &h, &sess, &db) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + + seedConfiguredTarget( + t, db, wh.ID, + database.TargetTypeHTTP, + `{"url":"https://example.com/hook","timeout":30,`+ + `"headers":{"Authorization":"Bearer sekrit"}}`, + ) + seedConfiguredTarget( + t, db, wh.ID, + database.TargetTypeDatabase, + `{"expiry":"720h"}`, + ) + seedConfiguredTarget( + t, db, wh.ID, + database.TargetType("carrier-pigeon"), + `{"beak":"sharp"}`, + ) + + body := renderSourceDetailPage(t, h, sess, wh.ID) + + assert.Contains(t, body, "Destination URL") + assert.Contains(t, body, "https://example.com/...") + assert.Contains(t, body, "Timeout") + assert.Contains(t, body, "1 configured") + assert.NotContains(t, body, "sekrit") + + assert.Contains(t, body, "Archive Expiry") + assert.Contains(t, body, "720h") + + // An unknown type gets the neutral placeholder, never the + // stored blob. + assert.Contains(t, body, "(unavailable)") + assert.NotContains(t, body, "beak") +} diff --git a/internal/handlers/source_logs_deleted_target_test.go b/internal/handlers/source_logs_deleted_target_test.go new file mode 100644 index 0000000..31c7a1f --- /dev/null +++ b/internal/handlers/source_logs_deleted_target_test.go @@ -0,0 +1,144 @@ +package handlers_test + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/handlers" + "sneak.berlin/go/webhooker/internal/session" +) + +// deletedMarker is the suffix the event log appends to the name +// of a target that no longer exists. +const deletedMarker = " (deleted)" + +// deleteTargetThroughHandler removes a target through the real +// deletion handler, so the test soft-deletes exactly the way the +// UI does rather than by writing the timestamp itself. +func deleteTargetThroughHandler( + t *testing.T, + h *handlers.Handlers, + sess *session.Session, + webhookID, targetID string, +) { + t.Helper() + + req := postRequest( + "/source/"+webhookID+"/targets/"+targetID+"/delete", + authenticatedCookies( + t, sess, deleteTestUserID, deleteTestUsername, + ), + map[string]string{ + paramSourceID: webhookID, + paramTargetID: targetID, + }, + ) + w := httptest.NewRecorder() + + h.HandleTargetDelete().ServeHTTP(w, req) + + require.Equal(t, http.StatusSeeOther, w.Code) +} + +// TestHandleSourceLogs_NamesDeletedTarget proves a delivery +// produced by a since-deleted target still names it on the event +// log, marked as deleted. +// +// Deletes are soft and deliveries carry no foreign key to the +// target row, so the history outlives the target. Against a +// scoped lookup the delivery resolves to a zero view and the page +// renders ": delivered" with nothing saying what it was delivered +// to. +func TestHandleSourceLogs_NamesDeletedTarget(t *testing.T) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + tgt := seedTarget(t, db, wh.ID, database.TargetTypeLog) + + seedDeliveredEvent(t, dbMgr, wh.ID, tgt.ID) + + // The control: the name is on the page while the target + // lives, and is not yet marked as deleted. + before := renderSourceLogsPage(t, h, sess, wh.ID) + assert.Contains(t, before, tgt.Name) + assert.NotContains(t, before, tgt.Name+deletedMarker) + + deleteTargetThroughHandler(t, h, sess, wh.ID, tgt.ID) + + after := renderSourceLogsPage(t, h, sess, wh.ID) + + assert.Contains( + t, after, tgt.Name+deletedMarker, + "a delivery from a deleted target must keep its name, "+ + "marked as no longer existing", + ) + assert.Contains( + t, after, "delivered", + "the delivery history itself must survive the delete", + ) +} + +// TestHandleSourceLogs_MasksDeletedTargetConfig proves that +// naming a deleted target does not widen what the page shows of +// it: its stored configuration stays masked by exactly the rules +// a live target's is. +// +// The lookup behind the name reads soft-deleted rows, so it +// carries a full target row — credential blob included — into the +// place a zero value used to sit. The projection to TargetView is +// what keeps that blob away from the template, and it must hold +// for a deleted row too. +func TestHandleSourceLogs_MasksDeletedTargetConfig(t *testing.T) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + tgt := seedConfiguredTarget( + t, db, wh.ID, + database.TargetTypeSlack, + `{"webhookUrl":"`+slackWebhookURL+`"}`, + ) + + seedDeliveredEvent(t, dbMgr, wh.ID, tgt.ID) + deleteTargetThroughHandler(t, h, sess, wh.ID, tgt.ID) + + body := renderSourceLogsPage(t, h, sess, wh.ID) + + assert.NotContains(t, body, slackSecretPath) + assert.NotContains(t, body, "T00000000") + assert.NotContains(t, body, "B00000000") + assert.NotContains( + t, body, "XXXXXXXXXXXXXXXXXXXXXXXX", + ) + assert.NotContains(t, body, "webhookUrl") + + // The name is there; only the credential is not. + assert.Contains(t, body, tgt.Name+deletedMarker) +} diff --git a/internal/handlers/source_logs_test.go b/internal/handlers/source_logs_test.go new file mode 100644 index 0000000..90f5a60 --- /dev/null +++ b/internal/handlers/source_logs_test.go @@ -0,0 +1,155 @@ +package handlers_test + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/go-chi/chi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm/clause" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/handlers" + "sneak.berlin/go/webhooker/internal/session" +) + +// contentTypeJSON is the content type the seeded events in this +// package carry. Shared across the seed helpers so the literal +// appears once. +const contentTypeJSON = "application/json" + +// seedDeliveredEvent records an event and a delivery for it in +// the webhook's own database, so the log page has a delivery +// to render against the target. +func seedDeliveredEvent( + t *testing.T, + dbMgr *database.WebhookDBManager, + webhookID, targetID string, +) { + t.Helper() + + webhookDB, err := dbMgr.GetDB(webhookID) + require.NoError(t, err) + + event := &database.Event{ + WebhookID: webhookID, + Method: http.MethodPost, + Body: `{"test":true}`, + ContentType: contentTypeJSON, + } + + require.NoError(t, webhookDB.Omit( + clause.Associations, + ).Create(event).Error) + + dlv := &database.Delivery{ + EventID: event.ID, + TargetID: targetID, + Status: database.DeliveryStatusDelivered, + } + + require.NoError(t, webhookDB.Omit( + clause.Associations, + ).Create(dlv).Error) +} + +// renderSourceLogsPage runs the real event log handler for a +// webhook and returns the rendered HTML. +func renderSourceLogsPage( + t *testing.T, + h *handlers.Handlers, + sess *session.Session, + webhookID string, +) string { + t.Helper() + + return renderSourceLogsPageWithQuery( + t, h, sess, webhookID, "", + ) +} + +// renderSourceLogsPageWithQuery is renderSourceLogsPage over a +// caller-supplied query string, for the page state a redirect back to +// the log carries in one. +func renderSourceLogsPageWithQuery( + t *testing.T, + h *handlers.Handlers, + sess *session.Session, + webhookID, query string, +) string { + t.Helper() + + req := httptest.NewRequestWithContext( + context.Background(), + http.MethodGet, + "/source/"+webhookID+"/logs"+query, + nil, + ) + + for _, c := range authenticatedCookies( + t, sess, deleteTestUserID, deleteTestUsername, + ) { + req.AddCookie(c) + } + + rctx := chi.NewRouteContext() + rctx.URLParams.Add(paramSourceID, webhookID) + + req = req.WithContext( + context.WithValue( + req.Context(), chi.RouteCtxKey, rctx, + ), + ) + + w := httptest.NewRecorder() + h.HandleSourceLogs().ServeHTTP(w, req) + + require.Equal(t, http.StatusOK, w.Code) + + return w.Body.String() +} + +// TestHandleSourceLogs_MasksSlackWebhookURL proves the event +// log page is handed a display-safe projection of each target +// rather than the stored row, so the credential cannot be +// rendered from its template data. +func TestHandleSourceLogs_MasksSlackWebhookURL(t *testing.T) { + t.Parallel() + + var ( + h *handlers.Handlers + sess *session.Session + db *database.Database + dbMgr *database.WebhookDBManager + ) + + app := newTestApp(t, &h, &sess, &db, &dbMgr) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + wh := seedWebhook(t, db) + tgt := seedConfiguredTarget( + t, db, wh.ID, + database.TargetTypeSlack, + `{"webhookUrl":"`+slackWebhookURL+`"}`, + ) + + seedDeliveredEvent(t, dbMgr, wh.ID, tgt.ID) + + body := renderSourceLogsPage(t, h, sess, wh.ID) + + assert.NotContains(t, body, slackSecretPath) + assert.NotContains(t, body, "T00000000") + assert.NotContains(t, body, "B00000000") + assert.NotContains( + t, body, "XXXXXXXXXXXXXXXXXXXXXXXX", + ) + assert.NotContains(t, body, "webhookUrl") + + // The page still identifies the delivery's target. + assert.Contains(t, body, tgt.Name) + assert.Contains(t, body, "delivered") +} diff --git a/internal/handlers/source_management.go b/internal/handlers/source_management.go index eaaf7e9..4abb7fc 100644 --- a/internal/handlers/source_management.go +++ b/internal/handlers/source_management.go @@ -4,13 +4,16 @@ import ( "encoding/json" "errors" "net/http" + "slices" "strconv" "strings" "github.com/go-chi/chi" "github.com/google/uuid" + "gorm.io/gorm" "sneak.berlin/go/webhooker/internal/database" "sneak.berlin/go/webhooker/internal/delivery" + "sneak.berlin/go/webhooker/internal/reqtls" ) // WebhookListItem holds data for the webhook list view. @@ -25,11 +28,107 @@ type WebhookListItem struct { // errMissingURL signals that a required URL was not provided. var errMissingURL = errors.New("missing URL") -// EventWithDeliveries holds an event and its deliveries. -type EventWithDeliveries struct { - database.Event +// errInvalidRetention signals a retention_days form value that is not +// a non-negative whole number. +var errInvalidRetention = errors.New("invalid retention days") - Deliveries []database.Delivery +// errRetentionTooLarge signals a retention_days form value that is a +// whole number but larger than the reaper's cutoff arithmetic can +// represent. It is distinguished from errInvalidRetention so the form +// can tell the user the actual ceiling instead of implying their input +// was not a number. +var errRetentionTooLarge = errors.New("retention days out of range") + +// retentionErrorMessage returns the message the create and edit forms +// show the user for a rejected retention_days value. Any error other +// than errRetentionTooLarge falls back to the generic wording, so an +// unrecognised parse failure still produces a sensible 400 rather than +// an empty alert. +func retentionErrorMessage(err error) string { + if errors.Is(err, errRetentionTooLarge) { + return "Retention must be at most " + + strconv.Itoa(database.MaxFiniteRetentionDays) + + " days, or 0 to retain events forever." + } + + return "Retention must be a whole number of days, or 0 to " + + "retain events forever." +} + +// parseRetentionDays interprets a retention_days form value. +// +// An empty value yields fallback, which lets the create path apply the +// default and the edit path leave the stored value unchanged. A value +// of 0 is returned as 0 and is rewritten to the retain-forever +// sentinel by database.Webhook's BeforeSave hook. Anything unparseable +// or negative is an error rather than a silently substituted default. +// +// The upper bound is not cosmetic. The reaper computes its cutoff as a +// time.Duration, an int64 nanosecond count, so a day count above +// database.MaxFiniteRetentionDays overflows, puts the cutoff in the +// future, and deletes every event the webhook has. A finite value +// above that ceiling is therefore a 400. +// +// A value at or above the retain-forever sentinel is not out of range: +// it is what the edit form pre-fills for a retain-forever webhook, so +// submitting the form back unchanged has to keep meaning "forever" +// rather than being rejected. +func parseRetentionDays(raw string, fallback int) (int, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return fallback, nil + } + + v, err := strconv.Atoi(raw) + if err != nil || v < 0 { + return 0, errInvalidRetention + } + + if v >= database.RetentionForeverDays { + return database.RetentionForeverDays, nil + } + + if v > database.MaxFiniteRetentionDays { + return 0, errRetentionTooLarge + } + + return v, nil +} + +// DeliveryView is the display-safe projection of a delivery +// for the event log page. Its target is a TargetView, so the +// stored configuration blob — which holds the target's +// credential — has no path to the template. +type DeliveryView struct { + ID string + Status database.DeliveryStatus + Target delivery.TargetView + + // Results is this delivery's attempts in attempt order, + // bounded by maxRenderedAttempts. Without them a failure + // renders as the status word alone and says nothing about + // why. + Results []DeliveryResultView + + // AttemptCount is how many attempts were recorded, which + // is more than len(Results) once the middle was dropped. + AttemptCount int + + // AttemptsOmitted is how many attempts were dropped from + // the middle of Results. The page must show it, or the + // bound would hide history rather than fold it. + AttemptsOmitted int +} + +// eventLogTarget is what the event log needs to know about +// one target: the display-safe view its template renders, and +// the redactor that keeps that target's own credential out of +// the text its remote peer chose. The two are kept together +// so a caller cannot pick up one without the other, and apart +// from TargetView so the secrets never reach a template. +type eventLogTarget struct { + View delivery.TargetView + Redactor delivery.Redactor } // HandleSourceList shows a list of user's webhooks. @@ -106,11 +205,30 @@ func (h *Handlers) buildWebhookListItems( // HandleSourceCreate shows the form to create a new webhook. func (h *Handlers) HandleSourceCreate() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - data := map[string]any{ - tmplKeyError: "", - } + h.renderTemplate( + w, r, "sources_new.html", + newSourceFormData("", "", ""), + ) + } +} - h.renderTemplate(w, r, "sources_new.html", data) +// newSourceFormData builds the template data for the webhook creation +// form. +// +// It carries the retention default so the pre-filled value comes from +// database.DefaultRetentionDays rather than being a third hardcoded +// copy of the same policy, and it carries the submitted name and +// description so that re-rendering the form after a validation failure +// gives the user their input back instead of a blank form. The edit +// form already behaves that way; create now matches it. +func newSourceFormData( + errMsg, name, description string, +) map[string]any { + return map[string]any{ + tmplKeyError: errMsg, + "Name": name, + "Description": description, + "DefaultRetentionDays": database.DefaultRetentionDays, } } @@ -127,10 +245,8 @@ func (h *Handlers) HandleSourceCreateSubmit() http.HandlerFunc { return } - r.Body = http.MaxBytesReader( - w, r.Body, 1< 0 { - retentionDays = v - } + return } h.createWebhookWithEntrypoint( @@ -304,23 +428,28 @@ func (h *Handlers) renderSourceDetail( } } - host := r.Host - scheme := "https" - - if r.TLS == nil { - scheme = "http" + scheme := "http" + if reqtls.IsTLS(r) { + scheme = "https" } - if fwdProto := r.Header.Get("X-Forwarded-Proto"); fwdProto != "" { - scheme = fwdProto - } + // The host is the client's Host header, unvalidated. It is + // inert only because source_detail.html renders BaseURL as + // text inside a element; putting it in an href or any + // other URL context needs it constrained first. + baseURL := scheme + "://" + r.Host + // The template calls Webhook methods, which take pointer + // receivers; html/template cannot address a value stored in a map. data := map[string]any{ - tmplKeyWebhook: webhook, - "Entrypoints": entrypoints, - "Targets": targets, - "Events": events, - "BaseURL": scheme + "://" + host, + tmplKeyWebhook: &webhook, + // Targets are projected to a display-safe view: a + // target's stored config blob holds a credential, and it + // must never reach a template. + "Entrypoints": NewEntrypointViews(entrypoints), + "Targets": delivery.NewTargetViews(targets), + "Events": events, + "BaseURL": baseURL, } h.renderTemplate(w, r, "source_detail.html", data) @@ -352,7 +481,7 @@ func (h *Handlers) HandleSourceEdit() http.HandlerFunc { } data := map[string]any{ - tmplKeyWebhook: webhook, + tmplKeyWebhook: &webhook, tmplKeyError: "", } @@ -386,10 +515,8 @@ func (h *Handlers) HandleSourceEditSubmit() http.HandlerFunc { return } - r.Body = http.MaxBytesReader( - w, r.Body, 1< 0 { - webhook.RetentionDays = v - } -} - // HandleSourceDelete handles webhook deletion. func (h *Handlers) HandleSourceDelete() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -496,164 +622,334 @@ func (h *Handlers) deleteWebhookResources( webhook database.Webhook, userID string, ) { - tx := h.db.DB().Begin() - if tx.Error != nil { - h.log.Error( - "failed to begin transaction", - "error", tx.Error, - ) - http.Error( - w, "Internal server error", - http.StatusInternalServerError, - ) + // The configuration delete commits before the event database + // is touched. No transaction spans the main database and the + // filesystem, so one side has to go first: committing the + // configuration first means a later failure leaves an unused + // event database file on disk, while removing the event + // database first would mean a failed commit destroys the + // history of a webhook that still exists. A leftover file can + // be removed by hand; deleted history cannot be recovered. + err := h.commitWebhookDeletion(&webhook) + if err != nil { + h.serverError(w, "failed to delete webhook", err) return } - tx.Where( - "webhook_id = ?", webhook.ID, - ).Delete(&database.Entrypoint{}) - - tx.Where( - "webhook_id = ?", webhook.ID, - ).Delete(&database.Target{}) - - tx.Delete(&webhook) - - err := tx.Commit().Error - if err != nil { - h.log.Error( - "failed to commit deletion", "error", err, - ) - http.Error( - w, "Internal server error", - http.StatusInternalServerError, - ) - - return - } - - err = h.dbMgr.DeleteDB(webhook.ID) - if err != nil { - h.log.Error( - "failed to delete webhook event database", - "webhook_id", webhook.ID, - "error", err, - ) - } - h.log.Info( "webhook deleted", "webhook_id", webhook.ID, "user_id", userID, ) + // Release the delivery engine's per-webhook archiving state + // so a deleted webhook's archive writer (and any handle open + // within its debounce window) does not linger for the + // process lifetime. The archive file itself is deliberately + // left on disk; see evictArchiveWriter. + h.evictArchiveWriter(webhook.ID) + + err = h.dbMgr.DeleteDB(webhook.ID) + if err != nil { + // The configuration is committed, so the webhook is gone, + // but its event database file is still on disk with + // nothing referencing it. Report the failure rather than + // redirecting as though everything succeeded: the file + // needs removing by hand, and the logged error names it. + h.serverError( + w, "failed to delete webhook event database", err, + ) + + return + } + http.Redirect(w, r, "/sources", http.StatusSeeOther) } +// commitWebhookDeletion soft-deletes a webhook's entrypoints, +// targets and the webhook row in one transaction. Every +// statement is checked and any failure rolls the whole +// transaction back, so a caller that gets an error knows the +// configuration is untouched and the event database must be +// left alone. +func (h *Handlers) commitWebhookDeletion( + webhook *database.Webhook, +) error { + tx := h.db.DB().Begin() + if tx.Error != nil { + return tx.Error + } + + err := tx.Where( + "webhook_id = ?", webhook.ID, + ).Delete(&database.Entrypoint{}).Error + if err != nil { + tx.Rollback() + + return err + } + + err = tx.Where( + "webhook_id = ?", webhook.ID, + ).Delete(&database.Target{}).Error + if err != nil { + tx.Rollback() + + return err + } + + err = tx.Delete(webhook).Error + if err != nil { + tx.Rollback() + + return err + } + + return tx.Commit().Error +} + +// evictArchiveWriter asks the delivery engine to drop its +// cached archive writer for a webhook, closing the archive file +// handle. +// +// The archive database file is NOT deleted. Unlike the event +// database — which is per-webhook working storage and is +// hard-deleted with the webhook — an archive is explicitly +// long-term storage that an operator may want to keep or move +// away for offline retention. Destroying it as a side effect of +// deleting a webhook would be a surprising and unrecoverable +// data loss, so the file is left for the operator to handle. +func (h *Handlers) evictArchiveWriter(webhookID string) { + if h.evictor == nil { + return + } + + h.evictor.EvictWebhook(webhookID) +} + +// evictArchiveWriterIfUnused releases a webhook's archive +// writer once the webhook has no database target left to feed +// it. +// +// It is called after any child resource of a webhook is +// deleted, and is correct without knowing which kind was: it +// evicts only when no database target remains, so deleting one +// of several database targets — or deleting an unrelated +// target type — leaves a still-needed writer alone. When no +// database target ever existed there is no writer and eviction +// is a no-op. Soft-deleted targets are excluded by GORM's +// default scope, so the row just deleted is not counted. +func (h *Handlers) evictArchiveWriterIfUnused(webhookID string) { + var remaining int64 + + err := h.db.DB(). + Model(&database.Target{}). + Where( + "webhook_id = ? AND type = ?", + webhookID, database.TargetTypeDatabase, + ). + Count(&remaining).Error + if err != nil { + h.log.Error( + "failed to count remaining database targets", + "webhook_id", webhookID, + "error", err, + ) + + return + } + + if remaining > 0 { + return + } + + h.evictArchiveWriter(webhookID) +} + +// ownedWebhook resolves the request's sourceID parameter to a +// webhook the session's user owns. +// +// Ownership and existence are decided by one query, so a +// webhook belonging to another user is indistinguishable from +// one that does not exist: both are a 404, and neither confirms +// the id. Callers that reach further into a webhook's data — +// the event log page and the event body download — share this +// one check rather than restating it, so the download cannot +// come to authorize differently from the page that links to it. +// +// It reports false once it has written the response, which is a +// redirect to the login page for an unauthenticated request and +// a 404 otherwise. The caller returns without writing more. +func (h *Handlers) ownedWebhook( + w http.ResponseWriter, + r *http.Request, +) (database.Webhook, bool) { + var webhook database.Webhook + + userID, ok := h.getUserID(r) + if !ok { + http.Redirect( + w, r, "/pages/login", http.StatusSeeOther, + ) + + return database.Webhook{}, false + } + + sourceID := chi.URLParam(r, "sourceID") + + err := h.db.DB().Where( + "id = ? AND user_id = ?", sourceID, userID, + ).First(&webhook).Error + if err != nil { + http.NotFound(w, r) + + return database.Webhook{}, false + } + + return webhook, true +} + // HandleSourceLogs shows the request/response logs for a // webhook. func (h *Handlers) HandleSourceLogs() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - userID, ok := h.getUserID(r) + webhook, ok := h.ownedWebhook(w, r) if !ok { - http.Redirect( - w, r, "/pages/login", http.StatusSeeOther, - ) - return } - sourceID := chi.URLParam(r, "sourceID") - - var webhook database.Webhook - - err := h.db.DB().Where( - "id = ? AND user_id = ?", sourceID, userID, - ).First(&webhook).Error + targets, err := h.loadTargetMap(webhook.ID) if err != nil { - http.NotFound(w, r) + // Without the map every delivery renders through a + // zero redactor, so failing the page is the only + // safe answer. + h.serverError(w, "failed to load targets", err) return } - targets := h.loadTargetMap(webhook.ID) page := h.parsePage(r) - evts, total := h.loadEventsWithDeliveries( + evts, total, ok := h.loadEventsWithDeliveries( w, webhook, targets, page, ) + if !ok { + return + } totalPages := int(total) / paginationPerPage if int(total)%paginationPerPage != 0 { totalPages++ } + // The banner a replay or resubmit POST redirected back + // with. The message comes from a fixed set keyed by the + // outcome code, never from the query string itself. + replayMsg, replayOK := replayOutcome( + r.URL.Query().Get(replayOutcomeParam), + ) + + resubmitMsg, resubmitOK := resubmitOutcome( + r.URL.Query().Get(resubmitOutcomeParam), + ) + data := map[string]any{ - tmplKeyWebhook: webhook, - "Events": evts, - "Page": page, - "TotalPages": totalPages, - "TotalEvents": total, - "HasPrev": page > 1, - "HasNext": page < totalPages, - "PrevPage": page - 1, - "NextPage": page + 1, + tmplKeyWebhook: &webhook, + "Events": evts, + "ReplayMessage": replayMsg, + "ReplayQueued": replayOK, + "ResubmitMessage": resubmitMsg, + "ResubmitQueued": resubmitOK, + "Page": page, + "TotalPages": totalPages, + "TotalEvents": total, + "HasPrev": page > 1, + "HasNext": page < totalPages, + "PrevPage": page - 1, + "NextPage": page + 1, } h.renderTemplate(w, r, "source_logs.html", data) } } -// loadTargetMap loads targets into a map keyed by target ID. +// loadTargetMap loads targets into a map of display-safe +// views keyed by target ID, each paired with its redactor. +// The projection happens here so that no caller can hand a +// raw target, configuration blob and all, to a template: the +// raw rows do not leave this function. +// +// The load is Unscoped because deleting a target only soft +// deletes the row while its deliveries survive in the +// per-webhook database. Both halves of the map need those rows: +// a scoped load leaves an old delivery with a zero redactor, +// which renders its response bodies unredacted, and with a zero +// view, which renders its target as a blank name. +// +// This map is historical display only. It is built for the event +// log page and reaches nothing but DeliveryView.Target: the +// target list on the source detail page, the edit form and the +// replay path each resolve targets themselves, and a deleted row +// is refused there as before. func (h *Handlers) loadTargetMap( webhookID string, -) map[string]database.Target { +) (map[string]eventLogTarget, error) { var targets []database.Target - h.db.DB().Where( + err := h.db.DB().Unscoped().Where( "webhook_id = ?", webhookID, - ).Find(&targets) - - targetMap := make( - map[string]database.Target, len(targets), - ) - - for _, t := range targets { - targetMap[t.ID] = t + ).Find(&targets).Error + if err != nil { + return nil, err } - return targetMap + targetMap := make( + map[string]eventLogTarget, len(targets), + ) + + for i := range targets { + targetMap[targets[i].ID] = eventLogTarget{ + Redactor: delivery.NewRedactor(&targets[i]), + } + } + + // The views come from NewTargetViews rather than being + // rebuilt here, so the masking rules stay in one place and a + // deleted target's configuration is masked by the same code + // that masks a live one's. + for _, v := range delivery.NewTargetViews(targets) { + entry := targetMap[v.ID] + entry.View = v + targetMap[v.ID] = entry + } + + return targetMap, nil } // parsePage extracts a page number from the query string. func (h *Handlers) parsePage(r *http.Request) int { - page := 1 - - if p := r.URL.Query().Get("page"); p != "" { - v, err := strconv.Atoi(p) - if err == nil && v > 0 { - page = v - } - } - - return page + return pageOrFirst(r.URL.Query().Get("page")) } // loadEventsWithDeliveries loads paginated events and their -// deliveries from the per-webhook database. +// deliveries from the per-webhook database. Events come back +// as capped projections rather than database.Event rows: see +// eventLogColumns for why the cut happens in SQL. +// +// The bool reports whether the load succeeded. It is false +// once this has answered the request with an error, and the +// caller must then render nothing further. func (h *Handlers) loadEventsWithDeliveries( w http.ResponseWriter, webhook database.Webhook, - targetMap map[string]database.Target, + targetMap map[string]eventLogTarget, page int, -) ([]EventWithDeliveries, int64) { - var totalEvents int64 - - var result []EventWithDeliveries +) ([]EventLogView, int64, bool) { + var result []EventLogView if !h.dbMgr.DBExists(webhook.ID) { - return result, totalEvents + return result, 0, true } webhookDB, err := h.dbMgr.GetDB(webhook.ID) @@ -662,42 +958,250 @@ func (h *Handlers) loadEventsWithDeliveries( w, "failed to get webhook database", err, ) - return nil, 0 + return nil, 0, false } - webhookDB.Model(&database.Event{}).Where( - "webhook_id = ?", webhook.ID, - ).Count(&totalEvents) + rows, totalEvents := loadEventLogRows( + webhookDB, webhook.ID, page, + ) - offset := (page - 1) * paginationPerPage + result = make([]EventLogView, len(rows)) + eventDeliveries := make([][]database.Delivery, len(rows)) - var events []database.Event + var deliveryIDs []string - webhookDB.Where( - "webhook_id = ?", webhook.ID, - ).Order("created_at DESC").Offset(offset).Limit( - paginationPerPage, - ).Find(&events) + eventIDs := make([]string, len(rows)) - result = make([]EventWithDeliveries, len(events)) - - for i := range events { - result[i].Event = events[i] + for i := range rows { + result[i] = rows[i].view() + eventIDs[i] = rows[i].ID webhookDB.Where( - "event_id = ?", events[i].ID, - ).Find(&result[i].Deliveries) + "event_id = ?", rows[i].ID, + ).Find(&eventDeliveries[i]) - for j := range result[i].Deliveries { - tid := result[i].Deliveries[j].TargetID - - if target, ok := targetMap[tid]; ok { - result[i].Deliveries[j].Target = target - } + for j := range eventDeliveries[i] { + deliveryIDs = append( + deliveryIDs, eventDeliveries[i][j].ID, + ) } } - return result, totalEvents + attempts, err := h.loadDeliveryResults( + webhookDB, deliveryIDs, + ) + if err != nil { + h.serverError( + w, "failed to load delivery attempts", err, + ) + + return nil, 0, false + } + + resubmits, err := resubmitCounts(webhookDB, eventIDs) + if err != nil { + h.serverError( + w, "failed to count event resubmissions", err, + ) + + return nil, 0, false + } + + for i := range rows { + result[i].Deliveries = newDeliveryViews( + eventDeliveries[i], targetMap, attempts, + ) + result[i].ResubmitCount = resubmits[rows[i].ID] + } + + return result, totalEvents, true +} + +// loadEventLogRows reads one page of the event log projection, newest +// first, and the total number of events the pager counts against. +func loadEventLogRows( + webhookDB *gorm.DB, webhookID string, page int, +) ([]eventLogRow, int64) { + var totalEvents int64 + + webhookDB.Model(&database.Event{}).Where( + "webhook_id = ?", webhookID, + ).Count(&totalEvents) + + var rows []eventLogRow + + webhookDB.Model(&database.Event{}).Select( + eventLogColumns, maxRenderedBodyBytes, + ).Where( + "webhook_id = ?", webhookID, + ).Order("created_at DESC").Offset( + (page - 1) * paginationPerPage, + ).Limit(paginationPerPage).Find(&rows) + + return rows, totalEvents +} + +// resubmitCounts reports, for each of the page's events, how many +// events have been resubmitted from it. +// +// One grouped query covers the page rather than one query per event. +// A page holds paginationPerPage ids, far below SQLite's bound +// parameter ceiling, so it needs no chunking as the delivery result +// load does. +func resubmitCounts( + webhookDB *gorm.DB, eventIDs []string, +) (map[string]int, error) { + counts := make(map[string]int, len(eventIDs)) + + if len(eventIDs) == 0 { + return counts, nil + } + + var rows []struct { + ResubmittedFromID string + Total int + } + + err := webhookDB.Model(&database.Event{}). + Select("resubmitted_from_id, count(*) AS total"). + Where("resubmitted_from_id IN ?", eventIDs). + Group("resubmitted_from_id"). + Find(&rows).Error + if err != nil { + return nil, err + } + + for _, row := range rows { + counts[row.ResubmittedFromID] = row.Total + } + + return counts, nil +} + +// deliveryIDChunkSize bounds how many delivery IDs go into one +// IN clause. SQLite refuses a statement carrying more than +// SQLITE_MAX_VARIABLE_NUMBER (32766) bound parameters, and a +// page holds one delivery per target per event, so a webhook +// with enough targets would turn the whole query into an error +// and the page into zero attempts. +const deliveryIDChunkSize = 500 + +// loadDeliveryResults loads the recorded attempts for the +// page's deliveries, keyed by delivery ID. +// +// Each response body is cut by SQLite rather than in Go, for +// the reason deliveryResultColumns gives. How many attempts a +// delivery has is the target's MaxRetries, which the +// authenticated operator sets; how many of them reach the page +// is bounded again by maxRenderedAttempts. +func (h *Handlers) loadDeliveryResults( + webhookDB *gorm.DB, + deliveryIDs []string, +) (map[string][]deliveryResultRow, error) { + byDelivery := make(map[string][]deliveryResultRow) + + for chunk := range slices.Chunk( + deliveryIDs, deliveryIDChunkSize, + ) { + var rows []deliveryResultRow + + err := webhookDB.Model( + &database.DeliveryResult{}, + ).Select( + deliveryResultColumns, maxRenderedResponseBytes, + ).Where( + "delivery_id IN ?", chunk, + ).Order("attempt_num ASC").Find(&rows).Error + if err != nil { + // Returning what was loaded so far renders the + // deliveries in the failed chunk as never having run, + // which is indistinguishable from ones that really + // never ran. The page fails instead. + return nil, err + } + + for i := range rows { + byDelivery[rows[i].DeliveryID] = append( + byDelivery[rows[i].DeliveryID], rows[i], + ) + } + } + + return byDelivery, nil +} + +// newDeliveryViews projects deliveries for rendering, +// resolving each one's target to its display-safe view and +// each one's attempts through that target's redactor. +func newDeliveryViews( + deliveries []database.Delivery, + targetMap map[string]eventLogTarget, + attempts map[string][]deliveryResultRow, +) []DeliveryView { + views := make([]DeliveryView, len(deliveries)) + + for i := range deliveries { + target := targetMap[deliveries[i].TargetID] + rows := attempts[deliveries[i].ID] + + results, omitted := renderedAttempts( + rows, target.Redactor, + ) + + views[i] = DeliveryView{ + ID: deliveries[i].ID, + Status: deliveries[i].Status, + Target: target.View, + Results: results, + AttemptCount: len(rows), + AttemptsOmitted: omitted, + } + } + + return views +} + +// maxRenderedAttempts bounds how many of one delivery's +// attempts the page renders. Past it the middle is dropped and +// counted, keeping the first attempts and the last ones: how +// the delivery started failing and how it ended are what a +// reader needs, and the count says plainly that the rest was +// dropped rather than never recorded. +const ( + renderedAttemptsHead = 10 + renderedAttemptsTail = 10 + maxRenderedAttempts = renderedAttemptsHead + + renderedAttemptsTail +) + +// renderedAttempts projects a delivery's attempts through the +// target's redactor, at most maxRenderedAttempts of them, and +// reports how many it dropped. +func renderedAttempts( + rows []deliveryResultRow, + redactor delivery.Redactor, +) ([]DeliveryResultView, int) { + omitted := 0 + + if len(rows) > maxRenderedAttempts { + omitted = len(rows) - maxRenderedAttempts + + kept := make( + []deliveryResultRow, 0, maxRenderedAttempts, + ) + kept = append(kept, rows[:renderedAttemptsHead]...) + kept = append( + kept, rows[len(rows)-renderedAttemptsTail:]..., + ) + rows = kept + } + + views := make([]DeliveryResultView, len(rows)) + for i := range rows { + views[i] = rows[i].view(redactor) + } + + return views, omitted } // HandleEntrypointCreate handles adding a new entrypoint. @@ -725,10 +1229,8 @@ func (h *Handlers) HandleEntrypointCreate() http.HandlerFunc { return } - r.Body = http.MaxBytesReader( - w, r.Body, 1<= 0 { - return v - } - - return 0 + return v } -// buildTargetConfig builds the JSON config string for a target. -// The expiry form value is read by the caller (which bounds the -// request body) and applies to database targets only. +// targetFormInput carries the raw form values describing a target's +// configuration. Both the create and the edit path fill one and hand +// it to buildTargetConfig, so neither can come to validate a +// destination differently from the other. +type targetFormInput struct { + // URL is the destination for an HTTP target and the webhook URL + // for a Slack target. + URL string + // Headers is an HTTP target's headers, one "Name: value" per + // line. + Headers string + // Timeout is an HTTP target's per-request timeout in seconds. + Timeout string + // Expiry is a database (archive) target's row expiry. + Expiry string +} + +// targetFormInputFrom reads the configuration fields from a request +// body. The body size cap is enforced by the MaxBodySize middleware, +// which runs before CSRF parses the form. +// +// Every field is read with PostFormValue, not FormValue. FormValue +// falls back to the query string, which would let +// `POST /source/{id}/targets?url=https://hooks.slack.com/...` +// configure a target from a value the request line carries — and the +// request line, unlike the body, is what logs, proxies, Referer +// headers and error trackers record. The headers field is under the +// same rule and for the same reason: its values are authorization +// tokens. +func targetFormInputFrom(r *http.Request) targetFormInput { + return targetFormInput{ + URL: r.PostFormValue("url"), + Headers: r.PostFormValue("headers"), + Timeout: r.PostFormValue("timeout"), + Expiry: r.PostFormValue("expiry"), + } +} + +// buildTargetConfig builds the JSON config string for a target from +// the submitted form values, writing its own 4xx response on +// rejection. Which fields of in apply depends on the target type. func (h *Handlers) buildTargetConfig( w http.ResponseWriter, r *http.Request, targetType database.TargetType, - targetURL, expiry string, + in targetFormInput, ) (string, error) { switch targetType { case database.TargetTypeHTTP: - return h.buildURLTargetConfig( - w, r, targetURL, "url", - "URL is required for HTTP targets", - ) + return h.buildHTTPTargetConfig(w, r, in) case database.TargetTypeSlack: - return h.buildURLTargetConfig( - w, r, targetURL, "webhookUrl", - "Webhook URL is required for Slack targets", - ) + return h.buildSlackTargetConfig(w, r, in.URL) case database.TargetTypeDatabase: - return h.buildDatabaseTargetConfig(w, expiry) + return h.buildDatabaseTargetConfig(w, in.Expiry) case database.TargetTypeLog: return "", nil default: @@ -927,14 +1471,83 @@ func (h *Handlers) buildTargetConfig( } } -// buildURLTargetConfig builds config JSON for a target whose -// configuration is a single SSRF-validated URL stored under -// configKey. missingMsg is the error shown when no URL is given. -func (h *Handlers) buildURLTargetConfig( +// buildHTTPTargetConfig builds config JSON for an HTTP target: an +// SSRF-validated destination plus the optional headers and timeout +// the delivery path honours. +func (h *Handlers) buildHTTPTargetConfig( w http.ResponseWriter, r *http.Request, - targetURL, configKey, missingMsg string, + in targetFormInput, ) (string, error) { + err := h.validateTargetURL( + w, r, in.URL, "URL is required for HTTP targets", + ) + if err != nil { + return "", err + } + + headers, err := delivery.ParseTargetHeaders(in.Headers) + if err != nil { + http.Error( + w, + "Invalid headers: "+err.Error(), + http.StatusBadRequest, + ) + + return "", err + } + + timeout, err := delivery.ParseTargetTimeout(in.Timeout) + if err != nil { + http.Error( + w, + "Invalid timeout: "+err.Error(), + http.StatusBadRequest, + ) + + return "", err + } + + return marshalTargetConfig(w, delivery.HTTPTargetConfig{ + URL: in.URL, + Headers: headers, + Timeout: timeout, + }) +} + +// buildSlackTargetConfig builds config JSON for a Slack target, +// whose whole configuration is one SSRF-validated webhook URL. +func (h *Handlers) buildSlackTargetConfig( + w http.ResponseWriter, + r *http.Request, + targetURL string, +) (string, error) { + err := h.validateTargetURL( + w, r, targetURL, + "Webhook URL is required for Slack targets", + ) + if err != nil { + return "", err + } + + return marshalTargetConfig(w, delivery.SlackTargetConfig{ + WebhookURL: targetURL, + }) +} + +// validateTargetURL rejects an empty or SSRF-blocked destination, +// writing the 400 itself. missingMsg is the error shown when no URL +// is given. +// +// It is the single point at which a user-supplied destination enters +// the SSRF guard, on create and on edit alike. An edit path that +// reached storage without passing through here would reopen the hole +// the guard closes. +func (h *Handlers) validateTargetURL( + w http.ResponseWriter, + r *http.Request, + targetURL, missingMsg string, +) error { if targetURL == "" { http.Error( w, @@ -942,16 +1555,19 @@ func (h *Handlers) buildURLTargetConfig( http.StatusBadRequest, ) - return "", errMissingURL + return errMissingURL } - err := delivery.ValidateTargetURL( + err := h.ssrf.ValidateTargetURL( r.Context(), targetURL, ) if err != nil { + // The submitted URL can be a credential (a Slack + // incoming webhook URL is a bearer token), so the log + // records only its scheme and host. h.log.Warn( "target URL blocked by SSRF protection", - "url", targetURL, + "url", delivery.MaskURL(targetURL), "error", err, ) http.Error( @@ -960,11 +1576,18 @@ func (h *Handlers) buildURLTargetConfig( http.StatusBadRequest, ) - return "", err + return err } - cfg := map[string]any{configKey: targetURL} + return nil +} +// marshalTargetConfig serialises a target configuration for storage, +// writing a 500 itself if it cannot. +func marshalTargetConfig( + w http.ResponseWriter, + cfg any, +) (string, error) { configBytes, err := json.Marshal(cfg) if err != nil { http.Error( @@ -1004,19 +1627,9 @@ func (h *Handlers) buildDatabaseTargetConfig( return "", err } - cfg := map[string]any{"expiry": expiry} - - configBytes, err := json.Marshal(cfg) - if err != nil { - http.Error( - w, "Internal server error", - http.StatusInternalServerError, - ) - - return "", err - } - - return string(configBytes), nil + return marshalTargetConfig( + w, map[string]any{"expiry": expiry}, + ) } // HandleEntrypointDelete handles deleting an entrypoint. @@ -1024,23 +1637,31 @@ func (h *Handlers) HandleEntrypointDelete() http.HandlerFunc { return h.deleteChildResource( "entrypointID", &database.Entrypoint{}, "failed to delete entrypoint", + nil, ) } -// HandleTargetDelete handles deleting a target. +// HandleTargetDelete handles deleting a target. Deleting the +// last database target of a webhook leaves its archive writer +// with nothing to write, so the writer is evicted and its +// handle closed; the archive file is left on disk. func (h *Handlers) HandleTargetDelete() http.HandlerFunc { return h.deleteChildResource( "targetID", &database.Target{}, "failed to delete target", + h.evictArchiveWriterIfUnused, ) } // deleteChildResource returns a handler that deletes a child -// resource (entrypoint or target) belonging to a webhook. +// resource (entrypoint or target) belonging to a webhook. The +// optional afterDelete hook runs with the webhook's id once the +// delete has succeeded, before the redirect. func (h *Handlers) deleteChildResource( idParam string, model any, errMsg string, + afterDelete func(webhookID string), ) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { userID, ok := h.getUserID(r) @@ -1080,6 +1701,10 @@ func (h *Handlers) deleteChildResource( return } + if afterDelete != nil { + afterDelete(webhook.ID) + } + http.Redirect( w, r, "/source/"+webhook.ID, diff --git a/internal/handlers/source_management_test.go b/internal/handlers/source_management_test.go new file mode 100644 index 0000000..508af5d --- /dev/null +++ b/internal/handlers/source_management_test.go @@ -0,0 +1,589 @@ +package handlers_test + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "testing" + + "github.com/go-chi/chi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm/clause" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/handlers" + "sneak.berlin/go/webhooker/internal/session" +) + +const ( + // sourceTestUserID is the session user id used by the webhook + // management tests. + sourceTestUserID = "source-test-user" + // sourceIDParam is the chi URL parameter naming a webhook. + sourceIDParam = "sourceID" +) + +// formRequest builds an urlencoded POST to path carrying the given +// cookies, plus any chi URL parameters the handler reads. +func formRequest( + path string, + cookies []*http.Cookie, + form url.Values, + urlParams map[string]string, +) *http.Request { + req := httptest.NewRequestWithContext( + context.Background(), + http.MethodPost, + path, + strings.NewReader(form.Encode()), + ) + req.Header.Set( + "Content-Type", "application/x-www-form-urlencoded", + ) + + for _, c := range cookies { + req.AddCookie(c) + } + + rctx := chi.NewRouteContext() + for k, v := range urlParams { + rctx.URLParams.Add(k, v) + } + + return req.WithContext( + context.WithValue(req.Context(), chi.RouteCtxKey, rctx), + ) +} + +// getRequest builds a GET to path carrying the given cookies, plus any +// chi URL parameters the handler reads. +func getRequest( + t *testing.T, + path string, + cookies []*http.Cookie, + urlParams map[string]string, +) *http.Request { + t.Helper() + + req := httptest.NewRequestWithContext( + context.Background(), http.MethodGet, path, nil, + ) + + for _, c := range cookies { + req.AddCookie(c) + } + + rctx := chi.NewRouteContext() + for k, v := range urlParams { + rctx.URLParams.Add(k, v) + } + + return req.WithContext( + context.WithValue(req.Context(), chi.RouteCtxKey, rctx), + ) +} + +// submitCreate posts the webhook creation form with the given +// retention_days value (omitted entirely when retention is nil) and +// returns the recorder. +func submitCreate( + t *testing.T, + h *handlers.Handlers, + cookies []*http.Cookie, + name string, + retention *string, +) *httptest.ResponseRecorder { + t.Helper() + + form := url.Values{} + form.Set("name", name) + + if retention != nil { + form.Set("retention_days", *retention) + } + + req := formRequest("/sources/new", cookies, form, nil) + w := httptest.NewRecorder() + + h.HandleSourceCreateSubmit().ServeHTTP(w, req) + + return w +} + +// onlyWebhook loads the single webhook belonging to the test user. +func onlyWebhook( + t *testing.T, + db *database.Database, +) database.Webhook { + t.Helper() + + var webhooks []database.Webhook + + require.NoError( + t, + db.DB().Where("user_id = ?", sourceTestUserID). + Find(&webhooks).Error, + ) + require.Len(t, webhooks, 1) + + return webhooks[0] +} + +// seedWebhookWithRetention inserts a webhook owned by the test user +// with an exact stored retention value, bypassing Webhook.BeforeSave +// via a column-level update so that legacy rows can be planted too. +func seedWebhookWithRetention( + t *testing.T, + db *database.Database, + retentionDays int, +) database.Webhook { + t.Helper() + + wh := &database.Webhook{ + UserID: sourceTestUserID, + Name: "seeded", + RetentionDays: retentionDays, + } + require.NoError( + t, + db.DB().Omit(clause.Associations).Create(wh).Error, + ) + require.NoError( + t, + db.DB().Model(wh). + Update("retention_days", retentionDays).Error, + ) + + wh.RetentionDays = retentionDays + + return *wh +} + +// storedRetentionDays reads the retention_days column for a webhook. +func storedRetentionDays( + t *testing.T, + db *database.Database, + id string, +) int { + t.Helper() + + var got int + + require.NoError( + t, + db.DB().Model(&database.Webhook{}). + Where("id = ?", id). + Pluck("retention_days", &got).Error, + ) + + return got +} + +// sourceTestEnv bundles the handler, session, and database a webhook +// management test drives. +type sourceTestEnv struct { + handlers *handlers.Handlers + db *database.Database + cookies []*http.Cookie +} + +func setupSourceTest(t *testing.T) *sourceTestEnv { + t.Helper() + + var h *handlers.Handlers + + var sess *session.Session + + var db *database.Database + + app := newTestApp(t, &h, &sess, &db) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + return &sourceTestEnv{ + handlers: h, + db: db, + cookies: authenticatedCookies( + t, sess, sourceTestUserID, "sourceuser", + ), + } +} + +// TestHandleSourceCreateSubmit_ZeroRetentionPersistsForever is the core +// regression test for the bug: the create form's 0 must reach the +// database as the retain-forever sentinel rather than being replaced by +// the column's default of 30. +func TestHandleSourceCreateSubmit_ZeroRetentionPersistsForever( + t *testing.T, +) { + t.Parallel() + + env := setupSourceTest(t) + zero := "0" + + w := submitCreate(t, env.handlers, env.cookies, "forever", &zero) + require.Equal(t, http.StatusSeeOther, w.Code) + + wh := onlyWebhook(t, env.db) + assert.Equal( + t, + database.RetentionForeverDays, + storedRetentionDays(t, env.db, wh.ID), + ) + assert.True(t, wh.RetainsForever()) +} + +func TestHandleSourceCreateSubmit_OmittedRetentionUsesDefault( + t *testing.T, +) { + t.Parallel() + + env := setupSourceTest(t) + + w := submitCreate(t, env.handlers, env.cookies, "defaulted", nil) + require.Equal(t, http.StatusSeeOther, w.Code) + + wh := onlyWebhook(t, env.db) + assert.Equal( + t, + database.DefaultRetentionDays, + storedRetentionDays(t, env.db, wh.ID), + ) +} + +// TestHandleSourceCreate_PrefillsDefaultFromConstant keeps the create +// form's pre-filled retention from becoming a third hardcoded copy of +// the 30-day policy. +func TestHandleSourceCreate_PrefillsDefaultFromConstant(t *testing.T) { + t.Parallel() + + env := setupSourceTest(t) + + w := httptest.NewRecorder() + env.handlers.HandleSourceCreate().ServeHTTP( + w, getRequest(t, "/sources/new", env.cookies, nil), + ) + + require.Equal(t, http.StatusOK, w.Code) + + body := w.Body.String() + + assert.Contains( + t, body, + `value="`+strconv.Itoa(database.DefaultRetentionDays)+`"`, + ) + assert.NotContains( + t, body, `max="365"`, + "a max below the sentinel would block retain-forever", + ) + assert.Contains(t, body, `min="0"`) +} + +func TestHandleSourceCreateSubmit_InvalidRetentionIsRejected( + t *testing.T, +) { + t.Parallel() + + for _, raw := range []string{"abc", "-1", "3.5"} { + t.Run(raw, func(t *testing.T) { + t.Parallel() + + env := setupSourceTest(t) + + w := submitCreate( + t, env.handlers, env.cookies, "bad", &raw, + ) + + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.Contains( + t, w.Body.String(), "Retention must be", + ) + + var count int64 + + require.NoError( + t, + env.db.DB().Model(&database.Webhook{}). + Where("user_id = ?", sourceTestUserID). + Count(&count).Error, + ) + assert.Zero( + t, count, + "no webhook may be created from a rejected form", + ) + }) + } +} + +// TestHandleSourceCreateSubmit_OverflowingRetentionIsRejected covers +// the data-loss path directly: a finite retention above the largest one +// the reaper's cutoff arithmetic can represent must never reach the +// database, because the sweep would compute a future cutoff and delete +// every event the webhook has. +func TestHandleSourceCreateSubmit_OverflowingRetentionIsRejected( + t *testing.T, +) { + t.Parallel() + + tooBig := strconv.Itoa(database.MaxFiniteRetentionDays + 1) + + env := setupSourceTest(t) + + w := submitCreate(t, env.handlers, env.cookies, "huge", &tooBig) + + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.Contains( + t, w.Body.String(), + strconv.Itoa(database.MaxFiniteRetentionDays), + "the form tells the user the actual ceiling", + ) + + var count int64 + + require.NoError( + t, + env.db.DB().Model(&database.Webhook{}). + Where("user_id = ?", sourceTestUserID). + Count(&count).Error, + ) + assert.Zero( + t, count, + "no webhook may be created from a rejected form", + ) +} + +// TestHandleSourceCreateSubmit_SentinelIsAcceptedAsForever guards the +// boundary between "too large to represent" and "retain forever": the +// sentinel is above MaxFiniteRetentionDays, but it is the value the +// edit form pre-fills, so it must be accepted rather than rejected as +// out of range. +func TestHandleSourceCreateSubmit_SentinelIsAcceptedAsForever( + t *testing.T, +) { + t.Parallel() + + env := setupSourceTest(t) + sentinel := strconv.Itoa(database.RetentionForeverDays) + + w := submitCreate(t, env.handlers, env.cookies, "forever", &sentinel) + require.Equal(t, http.StatusSeeOther, w.Code) + + wh := onlyWebhook(t, env.db) + assert.Equal( + t, + database.RetentionForeverDays, + storedRetentionDays(t, env.db, wh.ID), + ) +} + +// TestHandleSourceCreateSubmit_RejectedFormKeepsUserInput checks that a +// validation failure hands the user's typing back, matching what the +// edit form already does. Losing a long description to a mistyped +// retention value is the kind of thing that makes people give up on a +// form. +func TestHandleSourceCreateSubmit_RejectedFormKeepsUserInput( + t *testing.T, +) { + t.Parallel() + + env := setupSourceTest(t) + + const ( + name = "kept-name" + description = "a description worth not losing" + ) + + form := url.Values{} + form.Set("name", name) + form.Set("description", description) + form.Set("retention_days", "nonsense") + + req := formRequest("/sources/new", env.cookies, form, nil) + w := httptest.NewRecorder() + + env.handlers.HandleSourceCreateSubmit().ServeHTTP(w, req) + + require.Equal(t, http.StatusBadRequest, w.Code) + + body := w.Body.String() + + assert.Contains(t, body, `value="`+name+`"`) + assert.Contains(t, body, description) +} + +// submitEdit posts the webhook edit form for the given webhook. +func submitEdit( + t *testing.T, + env *sourceTestEnv, + wh database.Webhook, + retention string, +) *httptest.ResponseRecorder { + t.Helper() + + form := url.Values{} + form.Set("name", wh.Name) + form.Set("description", wh.Description) + form.Set("retention_days", retention) + + req := formRequest( + "/source/"+wh.ID+"/edit", + env.cookies, + form, + map[string]string{sourceIDParam: wh.ID}, + ) + w := httptest.NewRecorder() + + env.handlers.HandleSourceEditSubmit().ServeHTTP(w, req) + + return w +} + +func TestHandleSourceEditSubmit_ZeroRetentionPersistsForever( + t *testing.T, +) { + t.Parallel() + + env := setupSourceTest(t) + wh := seedWebhookWithRetention( + t, env.db, database.DefaultRetentionDays, + ) + + w := submitEdit(t, env, wh, "0") + require.Equal(t, http.StatusSeeOther, w.Code) + + assert.Equal( + t, + database.RetentionForeverDays, + storedRetentionDays(t, env.db, wh.ID), + ) +} + +func TestHandleSourceEditSubmit_InvalidRetentionIsRejected( + t *testing.T, +) { + t.Parallel() + + env := setupSourceTest(t) + wh := seedWebhookWithRetention( + t, env.db, database.DefaultRetentionDays, + ) + + w := submitEdit(t, env, wh, "not-a-number") + + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.Contains(t, w.Body.String(), "Retention must be") + assert.Equal( + t, + database.DefaultRetentionDays, + storedRetentionDays(t, env.db, wh.ID), + "a rejected form must not change the stored retention", + ) +} + +func TestHandleSourceEditSubmit_EmptyRetentionLeavesValueUnchanged( + t *testing.T, +) { + t.Parallel() + + env := setupSourceTest(t) + wh := seedWebhookWithRetention(t, env.db, 7) + + w := submitEdit(t, env, wh, "") + require.Equal(t, http.StatusSeeOther, w.Code) + + assert.Equal(t, 7, storedRetentionDays(t, env.db, wh.ID)) +} + +// TestSourceEditForm_ForeverWebhookRoundTrips walks the exact path that +// the removed max="365" cap used to break: render the edit form for a +// retain-forever webhook, confirm the pre-filled sentinel is not capped +// by browser validation, then submit that pre-filled value straight +// back and confirm the retention policy survives untouched. +func TestSourceEditForm_ForeverWebhookRoundTrips(t *testing.T) { + t.Parallel() + + env := setupSourceTest(t) + wh := seedWebhookWithRetention( + t, env.db, database.RetentionForeverDays, + ) + + req := getRequest( + t, "/source/"+wh.ID+"/edit", env.cookies, + map[string]string{sourceIDParam: wh.ID}, + ) + w := httptest.NewRecorder() + env.handlers.HandleSourceEdit().ServeHTTP(w, req) + + require.Equal(t, http.StatusOK, w.Code) + + sentinel := strconv.Itoa(database.RetentionForeverDays) + body := w.Body.String() + + assert.Contains( + t, body, `value="`+sentinel+`"`, + "the edit form pre-fills the stored retention", + ) + assert.NotContains( + t, body, `max="365"`, + "a max below the sentinel would block saving any edit", + ) + // "Currently forever." is the rendered RetentionLabel, not the + // static hint below the input, which says "Enter 0 to retain events + // forever." A bare Contains of "forever" would pass for any + // webhook and would assert nothing about this one. + assert.Contains( + t, body, "Currently forever.", + "the form reports this webhook's policy as forever", + ) + + // Submit the pre-filled value back, exactly as a browser would. + post := submitEdit(t, env, wh, sentinel) + require.Equal(t, http.StatusSeeOther, post.Code) + + assert.Equal( + t, + database.RetentionForeverDays, + storedRetentionDays(t, env.db, wh.ID), + ) +} + +// TestSourceListAndDetail_ShowForeverNotTheSentinelNumber checks that +// the retain-forever value is never rendered to the user as a raw day +// count on either read-only view. +func TestSourceListAndDetail_ShowForeverNotTheSentinelNumber( + t *testing.T, +) { + t.Parallel() + + env := setupSourceTest(t) + wh := seedWebhookWithRetention( + t, env.db, database.RetentionForeverDays, + ) + sentinel := strconv.Itoa(database.RetentionForeverDays) + + listW := httptest.NewRecorder() + env.handlers.HandleSourceList().ServeHTTP( + listW, getRequest(t, "/sources", env.cookies, nil), + ) + + require.Equal(t, http.StatusOK, listW.Code) + assert.Contains(t, listW.Body.String(), "Retention: forever") + assert.NotContains(t, listW.Body.String(), sentinel) + + detailW := httptest.NewRecorder() + env.handlers.HandleSourceDetail().ServeHTTP( + detailW, + getRequest( + t, "/source/"+wh.ID, env.cookies, + map[string]string{sourceIDParam: wh.ID}, + ), + ) + + require.Equal(t, http.StatusOK, detailW.Code) + assert.Contains(t, detailW.Body.String(), "Retention: forever") + assert.NotContains(t, detailW.Body.String(), sentinel) +} diff --git a/internal/handlers/target_create_query_test.go b/internal/handlers/target_create_query_test.go new file mode 100644 index 0000000..84b0b5e --- /dev/null +++ b/internal/handlers/target_create_query_test.go @@ -0,0 +1,206 @@ +package handlers_test + +import ( + "bytes" + "context" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/go-chi/chi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/config" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/middleware" +) + +// targetSecretSegments are the path segments of an incoming-webhook +// URL. For Slack, Discord and Teams the path IS the bearer credential, +// so this string must not reach storage or the access log by way of +// the request line. +const targetSecretSegments = "T00000000/B00000000/QQTARGETSECRETQQ" + +// targetSecretURL is a destination whose secret lives in its path. It +// uses a literal public address rather than a hostname so the SSRF +// check resolves nothing: with a hostname, a sandbox without DNS would +// reject the URL for the wrong reason and the test would pass even +// with the defect reintroduced. +const targetSecretURL = "https://93.184.216.34/services/" + + targetSecretSegments + +// targetsForWebhook returns every target stored against a webhook. +func targetsForWebhook( + t *testing.T, + db *database.Database, + webhookID string, +) []database.Target { + t.Helper() + + var targets []database.Target + + require.NoError( + t, + db.DB().Where("webhook_id = ?", webhookID). + Find(&targets).Error, + ) + + return targets +} + +// postTargetCreate drives HandleTargetCreate through the production +// access-log middleware and a chi route, so the logged url field is +// produced exactly as it ships, and returns the recorder plus the +// captured log. +func postTargetCreate( + t *testing.T, + env *sourceTestEnv, + webhookID string, + query string, + form url.Values, +) (*httptest.ResponseRecorder, string) { + t.Helper() + + logBuf := new(bytes.Buffer) + mw := middleware.NewForTest( + slog.New(slog.NewJSONHandler( + logBuf, &slog.HandlerOptions{Level: slog.LevelInfo}, + )), + &config.Config{Environment: config.EnvironmentDev}, + nil, + ) + + router := chi.NewRouter() + router.Use(mw.Logging()) + router.Post( + "/source/{sourceID}/targets", + env.handlers.HandleTargetCreate(), + ) + + target := "/source/" + webhookID + "/targets" + if query != "" { + target += "?" + query + } + + body := "" + if form != nil { + body = form.Encode() + } + + req := httptest.NewRequestWithContext( + context.Background(), + http.MethodPost, + target, + strings.NewReader(body), + ) + req.Header.Set( + "Content-Type", "application/x-www-form-urlencoded", + ) + + for _, c := range env.cookies { + req.AddCookie(c) + } + + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + return w, logBuf.String() +} + +// TestHandleTargetCreate_QueryStringURLDoesNotConfigureATarget is the +// regression test for the ingress leak. r.FormValue falls back to the +// query string when a field is absent from the POST body, so +// +// POST /source/{id}/targets?url=https://hooks.slack.com/services/... +// +// with an empty url field used to create a working target from a value +// carried on the request line — where logs, proxies, Referer headers +// and error trackers record it. The handler reads the body only, so +// the request is rejected for a missing URL and stores nothing. +// +// name and type are sent in the BODY on purpose: the request has to +// get past those two validations for the assertion to be about the url +// read specifically. +func TestHandleTargetCreate_QueryStringURLDoesNotConfigureATarget( + t *testing.T, +) { + t.Parallel() + + env := setupSourceTest(t) + webhook := seedWebhookWithRetention(t, env.db, 30) + + body := url.Values{} + body.Set("name", "leaky") + body.Set("type", string(database.TargetTypeSlack)) + + w, logged := postTargetCreate( + t, env, webhook.ID, + "url="+url.QueryEscape(targetSecretURL), + body, + ) + + assert.Equal(t, http.StatusBadRequest, w.Code) + + targets := targetsForWebhook(t, env.db, webhook.ID) + assert.Empty( + t, targets, + "a query-string value must not populate a target config", + ) + + assert.NotContains(t, logged, targetSecretSegments) + assert.NotContains(t, logged, "93.184.216.34") + assert.NotEmpty(t, logged, "the access log line must still be written") +} + +// TestHandleTargetCreate_BodyURLStillCreatesTheTarget is the positive +// control for the test above: the rejection has to come from where the +// value was read, not from the handler being broken. +func TestHandleTargetCreate_BodyURLStillCreatesTheTarget(t *testing.T) { + t.Parallel() + + env := setupSourceTest(t) + webhook := seedWebhookWithRetention(t, env.db, 30) + + body := url.Values{} + body.Set("name", "legit") + body.Set("type", string(database.TargetTypeSlack)) + body.Set("url", targetSecretURL) + + w, logged := postTargetCreate(t, env, webhook.ID, "", body) + + assert.Equal(t, http.StatusSeeOther, w.Code) + + targets := targetsForWebhook(t, env.db, webhook.ID) + require.Len(t, targets, 1) + assert.Contains(t, targets[0].Config, targetSecretSegments) + + // The body carried the credential, so the access log must still + // not have it: the log records the request line only. + assert.NotContains(t, logged, targetSecretSegments) +} + +// TestHandleTargetCreate_QueryStringCannotSupplyNameOrType covers the +// rest of the converted reads on this handler in one request: with an +// empty body, nothing the query carries is visible to it. +func TestHandleTargetCreate_QueryStringCannotSupplyNameOrType( + t *testing.T, +) { + t.Parallel() + + env := setupSourceTest(t) + webhook := seedWebhookWithRetention(t, env.db, 30) + + w, _ := postTargetCreate( + t, env, webhook.ID, + "name=leaky&type=slack&max_retries=9&expiry=30d&url="+ + url.QueryEscape(targetSecretURL), + url.Values{}, + ) + + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.Contains(t, w.Body.String(), "Name is required") + assert.Empty(t, targetsForWebhook(t, env.db, webhook.ID)) +} diff --git a/internal/handlers/target_edit.go b/internal/handlers/target_edit.go new file mode 100644 index 0000000..8ff03b5 --- /dev/null +++ b/internal/handlers/target_edit.go @@ -0,0 +1,229 @@ +package handlers + +import ( + "net/http" + + "github.com/go-chi/chi" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" +) + +// targetEditTemplate is the page the target edit form renders. +const targetEditTemplate = "target_edit.html" + +// tmplKeyTarget is the template data key for the target being +// edited, and tmplKeyMaxTimeout for the timeout ceiling the form +// tells the user about. +const ( + tmplKeyTarget = "Target" + tmplKeyMaxTimeout = "MaxTimeout" +) + +// configUnreadableMessage is shown when a target's stored +// configuration does not parse. It says plainly that saving replaces +// the stored value rather than preserving it, because the form +// cannot pre-fill what it could not read. +const configUnreadableMessage = "The stored configuration for this " + + "target could not be read. Enter the values below; saving " + + "replaces the stored configuration." + +// targetEditView is the display model for the target edit page. +// +// It carries the target's row fields alongside its UNMASKED +// configuration, and deliberately omits database.Target's raw +// Config blob: the form renders named fields, and giving the +// template the blob as well would put an unreviewed second path to +// the credential on the page. +type targetEditView struct { + ID string + Name string + Type database.TargetType + Active bool + MaxRetries int + Config delivery.TargetConfigForm +} + +// HandleTargetEdit shows the form to edit a target. +// +// This page is the one place the full destination URL and header +// values are shown. It is reachable only through the +// /source/{sourceID} route group, which supplies RequireAuth and +// NoCache, and only for a target of a webhook the session's user +// owns; masking (delivery.TargetView) is unchanged everywhere else. +func (h *Handlers) HandleTargetEdit() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + webhook, target, ok := h.ownedTarget(w, r) + if !ok { + return + } + + cfg, err := delivery.NewTargetConfigForm(target) + msg := "" + + if err != nil { + // The error carries the parse failure, never the + // blob, so it is safe to log against the target id. + h.log.Warn( + "stored target config could not be read for editing", + "target_id", target.ID, + "error", err, + ) + + msg = configUnreadableMessage + } + + h.renderTargetEdit(w, r, webhook, target, cfg, msg) + } +} + +// HandleTargetEditSubmit handles the target edit form submission. +func (h *Handlers) HandleTargetEditSubmit() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + webhook, target, ok := h.ownedTarget(w, r) + if !ok { + return + } + + // The body size cap is enforced by the MaxBodySize + // middleware, which runs before CSRF parses the form. + err := r.ParseForm() + if err != nil { + http.Error( + w, "Bad request", http.StatusBadRequest, + ) + + return + } + + h.applyTargetEdit(w, r, webhook, target) + } +} + +// applyTargetEdit validates and saves target edits. +// +// The submitted configuration goes through buildTargetConfig, the +// same builder the create path uses, so an edited destination is +// SSRF-validated exactly as a new one is. +// +// The target's type is not editable. Each type stores a different +// configuration shape and its delivery history is recorded against +// the target row, so changing the type of an existing target is +// really the creation of a different one. The stored type decides +// which fields the form offers and which builder runs. +func (h *Handlers) applyTargetEdit( + w http.ResponseWriter, + r *http.Request, + webhook database.Webhook, + target *database.Target, +) { + name := r.PostFormValue("name") + if name == "" { + http.Error( + w, "Name is required", http.StatusBadRequest, + ) + + return + } + + configJSON, err := h.buildTargetConfig( + w, r, target.Type, targetFormInputFrom(r), + ) + if err != nil { + // buildTargetConfig has already written the response. + return + } + + // Retries are offered only by the forms for target types that + // retry, so an absent field means "this form does not edit + // retries" rather than "set them to zero". Reading it + // unconditionally would silently disable retries on any target + // saved from a form that does not render the input. + // + // A field that IS submitted but does not parse is a 400, through + // the same validator the create path uses. It is rejected before + // anything is written, so a typo cannot destroy the retry count + // the target is already delivering with. + if r.PostForm.Has("max_retries") { + retries, ok := targetMaxRetries(w, r, target.MaxRetries) + if !ok { + return + } + + target.MaxRetries = retries + } + + target.Name = name + target.Config = configJSON + + err = h.db.DB().Save(target).Error + if err != nil { + h.serverError(w, "failed to update target", err) + + return + } + + http.Redirect( + w, r, "/source/"+webhook.ID, http.StatusSeeOther, + ) +} + +// renderTargetEdit renders the target edit page with an optional +// error message. +func (h *Handlers) renderTargetEdit( + w http.ResponseWriter, + r *http.Request, + webhook database.Webhook, + target *database.Target, + cfg delivery.TargetConfigForm, + errMsg string, +) { + // The template calls Webhook methods, which take pointer + // receivers; html/template cannot address a value stored in a + // map. + data := map[string]any{ + tmplKeyWebhook: &webhook, + tmplKeyTarget: targetEditView{ + ID: target.ID, + Name: target.Name, + Type: target.Type, + Active: target.Active, + MaxRetries: target.MaxRetries, + Config: cfg, + }, + tmplKeyMaxTimeout: delivery.MaxTargetTimeoutSeconds, + tmplKeyError: errMsg, + } + + h.renderTemplate(w, r, targetEditTemplate, data) +} + +// ownedTarget resolves the request's sourceID and targetID +// parameters to a target of a webhook the session's user owns. +// +// Ownership is decided by the webhook, and the target is then +// scoped to that webhook, so a target id belonging to someone +// else's webhook is a 404 rather than an edit of their target. It +// reports false once it has written the response. +func (h *Handlers) ownedTarget( + w http.ResponseWriter, + r *http.Request, +) (database.Webhook, *database.Target, bool) { + webhook, ok := h.ownedWebhook(w, r) + if !ok { + return database.Webhook{}, nil, false + } + + var target database.Target + + err := h.db.DB().Where( + "id = ? AND webhook_id = ?", + chi.URLParam(r, "targetID"), webhook.ID, + ).First(&target).Error + if err != nil { + http.NotFound(w, r) + + return database.Webhook{}, nil, false + } + + return webhook, &target, true +} diff --git a/internal/handlers/target_edit_test.go b/internal/handlers/target_edit_test.go new file mode 100644 index 0000000..64a93e4 --- /dev/null +++ b/internal/handlers/target_edit_test.go @@ -0,0 +1,637 @@ +package handlers_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/go-chi/chi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm/clause" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" +) + +// The destinations the target edit tests configure. Both are literal +// public addresses rather than hostnames so the SSRF check resolves +// nothing: with a hostname, a sandbox without DNS would reject the +// URL for the wrong reason and a test asserting rejection would pass +// even with the guard removed. +const ( + editOriginalURL = "https://93.184.216.34/hooks/original" + editReplacedURL = "https://93.184.216.34/hooks/replaced" + // editBlockedURL resolves to loopback, which the SSRF guard + // refuses. It is what proves the guard runs on the edit path. + editBlockedURL = "http://127.0.0.1/hooks/internal" +) + +// editAuthHeader carries a bearer credential, the case the headers +// field exists for. +const ( + editBearerSecret = "QQEDITSECRETQQ" + editAuthHeader = "Authorization: Bearer " + editBearerSecret +) + +// targetRouter mounts the target create and edit routes on a chi +// router so the handlers see the URL parameters they read. +func targetRouter(env *sourceTestEnv) *chi.Mux { + router := chi.NewRouter() + router.Post( + "/source/{sourceID}/targets", + env.handlers.HandleTargetCreate(), + ) + router.Get( + "/source/{sourceID}/targets/{targetID}/edit", + env.handlers.HandleTargetEdit(), + ) + router.Post( + "/source/{sourceID}/targets/{targetID}/edit", + env.handlers.HandleTargetEditSubmit(), + ) + + return router +} + +// serveTarget drives one request through the target routes as the +// authenticated test user. +func serveTarget( + env *sourceTestEnv, + method, path string, + form url.Values, +) *httptest.ResponseRecorder { + body := "" + if form != nil { + body = form.Encode() + } + + req := httptest.NewRequestWithContext( + context.Background(), method, path, + strings.NewReader(body), + ) + if form != nil { + req.Header.Set( + "Content-Type", + "application/x-www-form-urlencoded", + ) + } + + for _, c := range env.cookies { + req.AddCookie(c) + } + + w := httptest.NewRecorder() + targetRouter(env).ServeHTTP(w, req) + + return w +} + +// seedHTTPTarget creates a webhook and an HTTP target on it through +// the real create handler, so every case starts from a target the +// production path produced rather than a hand-written row. +// +// Standing the fx app up is what a handler test mostly costs, and +// internal/handlers is already the slowest package in the suite, so +// the tests below share one env per test function and give each case +// its own webhook rather than its own app. +func seedHTTPTarget( + t *testing.T, + env *sourceTestEnv, + headers, timeout string, +) (database.Webhook, database.Target) { + t.Helper() + + webhook := seedWebhookWithRetention(t, env.db, 30) + + form := url.Values{} + form.Set("name", "original-name") + form.Set("type", string(database.TargetTypeHTTP)) + form.Set("url", editOriginalURL) + form.Set("headers", headers) + form.Set("timeout", timeout) + form.Set("max_retries", "3") + + w := serveTarget( + env, http.MethodPost, + "/source/"+webhook.ID+"/targets", form, + ) + require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String()) + + targets := targetsForWebhook(t, env.db, webhook.ID) + require.Len(t, targets, 1) + + return webhook, targets[0] +} + +// storedTarget reloads a target row. +func storedTarget( + t *testing.T, + env *sourceTestEnv, + targetID string, +) database.Target { + t.Helper() + + var target database.Target + + require.NoError( + t, + env.db.DB().Where("id = ?", targetID). + First(&target).Error, + ) + + return target +} + +// storedHTTPConfig reloads a target and parses its stored HTTP +// configuration. +func storedHTTPConfig( + t *testing.T, + env *sourceTestEnv, + targetID string, +) delivery.HTTPTargetConfig { + t.Helper() + + var cfg delivery.HTTPTargetConfig + + require.NoError( + t, + json.Unmarshal( + []byte(storedTarget(t, env, targetID).Config), &cfg, + ), + ) + + return cfg +} + +// editForm is the fully populated edit submission for an HTTP +// target. +func editForm(targetURL, headers, timeout string) url.Values { + form := url.Values{} + form.Set("name", "edited-name") + form.Set("url", targetURL) + form.Set("headers", headers) + form.Set("timeout", timeout) + form.Set("max_retries", "5") + + return form +} + +// submitTargetEdit posts the edit form for a target. +func submitTargetEdit( + env *sourceTestEnv, + webhookID, targetID string, + form url.Values, +) *httptest.ResponseRecorder { + return serveTarget( + env, http.MethodPost, + "/source/"+webhookID+"/targets/"+targetID+"/edit", + form, + ) +} + +// TestHandleTargetCreate_Configuration covers the half of the gap +// that is not about editing at all: HTTPTargetConfig has carried +// Headers and Timeout, and the delivery path has honoured them, but +// the create form wrote {"url":...} and nothing else, so a +// destination needing an Authorization header could not be +// configured through the UI at all. +func TestHandleTargetCreate_Configuration(t *testing.T) { + t.Parallel() + + env := setupSourceTest(t) + + t.Run("stores headers and timeout", func(t *testing.T) { + t.Parallel() + assertCreateStoresHeadersAndTimeout(t, env) + }) + + t.Run("without them keeps a url-only config", func(t *testing.T) { + t.Parallel() + assertCreateKeepsURLOnlyConfig(t, env) + }) +} + +func assertCreateStoresHeadersAndTimeout( + t *testing.T, env *sourceTestEnv, +) { + t.Helper() + + _, target := seedHTTPTarget( + t, env, editAuthHeader+"\nX-Tenant: acme\n", "12", + ) + + cfg := storedHTTPConfig(t, env, target.ID) + + assert.Equal(t, editOriginalURL, cfg.URL) + assert.Equal(t, 12, cfg.Timeout) + assert.Equal( + t, + map[string]string{ + "Authorization": "Bearer " + editBearerSecret, + "X-Tenant": "acme", + }, + cfg.Headers, + ) +} + +// Without the new fields the stored shape must be the same +// {"url":...} the create form wrote before they existed, so no +// existing target's configuration is rewritten by this change. +func assertCreateKeepsURLOnlyConfig( + t *testing.T, env *sourceTestEnv, +) { + t.Helper() + + _, target := seedHTTPTarget(t, env, "", "") + + assert.JSONEq( + t, `{"url":"`+editOriginalURL+`"}`, target.Config, + ) +} + +// TestHandleTargetEditSubmit_Saves is the round trip the issue asks +// for: create a target, edit it, and confirm the stored config +// changed. +func TestHandleTargetEditSubmit_Saves(t *testing.T) { + t.Parallel() + + env := setupSourceTest(t) + + t.Run("changes the destination URL", func(t *testing.T) { + t.Parallel() + assertEditChangesDestination(t, env) + }) + + t.Run("round trips headers and timeout", func(t *testing.T) { + t.Parallel() + assertEditRoundTripsHeadersAndTimeout(t, env) + }) + + t.Run("clearing them removes them", func(t *testing.T) { + t.Parallel() + assertEditClearingRemovesThem(t, env) + }) + + t.Run("absent max_retries is not zeroed", func(t *testing.T) { + t.Parallel() + assertEditKeepsAbsentMaxRetries(t, env) + }) +} + +func assertEditChangesDestination( + t *testing.T, env *sourceTestEnv, +) { + t.Helper() + + webhook, target := seedHTTPTarget(t, env, "", "") + + w := submitTargetEdit( + env, webhook.ID, target.ID, + editForm(editReplacedURL, "", ""), + ) + require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String()) + + assert.Equal( + t, + editReplacedURL, + storedHTTPConfig(t, env, target.ID).URL, + ) + + reloaded := storedTarget(t, env, target.ID) + assert.Equal(t, "edited-name", reloaded.Name) + assert.Equal(t, 5, reloaded.MaxRetries) + assert.Equal( + t, database.TargetTypeHTTP, reloaded.Type, + "the edit form must not change a target's type", + ) +} + +// The two previously unreachable fields must survive create, +// pre-fill and save. +func assertEditRoundTripsHeadersAndTimeout( + t *testing.T, env *sourceTestEnv, +) { + t.Helper() + + webhook, target := seedHTTPTarget(t, env, editAuthHeader, "7") + + w := submitTargetEdit( + env, webhook.ID, target.ID, + editForm( + editOriginalURL, + "Authorization: Bearer rotated\nX-Trace: on", + "21", + ), + ) + require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String()) + + cfg := storedHTTPConfig(t, env, target.ID) + + assert.Equal(t, 21, cfg.Timeout) + assert.Equal( + t, + map[string]string{ + "Authorization": "Bearer rotated", + "X-Trace": "on", + }, + cfg.Headers, + ) +} + +// The direction a naive "only set what was submitted" implementation +// gets wrong: an emptied field must remove the stored value, not +// leave the previous one in place. +func assertEditClearingRemovesThem( + t *testing.T, env *sourceTestEnv, +) { + t.Helper() + + webhook, target := seedHTTPTarget(t, env, editAuthHeader, "7") + + w := submitTargetEdit( + env, webhook.ID, target.ID, + editForm(editOriginalURL, "", ""), + ) + require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String()) + + cfg := storedHTTPConfig(t, env, target.ID) + + assert.Empty(t, cfg.Headers) + assert.Zero(t, cfg.Timeout) +} + +// Retries are offered only by the forms for target types that retry. +// An absent field means the form does not edit retries, not that +// they should be turned off. +func assertEditKeepsAbsentMaxRetries( + t *testing.T, env *sourceTestEnv, +) { + t.Helper() + + webhook, target := seedHTTPTarget(t, env, "", "") + require.Equal(t, 3, target.MaxRetries) + + form := editForm(editOriginalURL, "", "") + form.Del("max_retries") + + w := submitTargetEdit(env, webhook.ID, target.ID, form) + require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String()) + + assert.Equal( + t, 3, storedTarget(t, env, target.ID).MaxRetries, + ) +} + +// TestHandleTargetEdit_PrefillsTheStoredValuesUnmasked covers the +// deliberate exception to the masking rule. The operator cannot +// correct a value they cannot see, so this page — and only this page +// — renders the destination and the header values in full. +func TestHandleTargetEdit_PrefillsTheStoredValuesUnmasked( + t *testing.T, +) { + t.Parallel() + + env := setupSourceTest(t) + + webhook, target := seedHTTPTarget(t, env, editAuthHeader, "7") + + w := serveTarget( + env, http.MethodGet, + "/source/"+webhook.ID+"/targets/"+target.ID+"/edit", + nil, + ) + require.Equal(t, http.StatusOK, w.Code) + + page := w.Body.String() + + assert.Contains(t, page, editOriginalURL) + assert.Contains(t, page, "Bearer "+editBearerSecret) + assert.Contains(t, page, `value="7"`) + assert.Contains(t, page, "original-name") +} + +// TestHandleTargetEditSubmit_Rejects covers every submission that +// must not reach storage. +// +// The SSRF case is the most important assertion on this change: the +// edited destination goes through the same guard the create path +// uses. An edit that stored an unvalidated URL would reopen a closed +// hole, since a target could then be created public and edited to +// point at loopback. +// +// The header and timeout cases keep input that could not be +// delivered as written out of storage: a stored value that provably +// never reaches the wire reports a configuration that did not take +// effect. +func TestHandleTargetEditSubmit_Rejects(t *testing.T) { + t.Parallel() + + env := setupSourceTest(t) + + t.Run("an SSRF-blocked destination", func(t *testing.T) { + t.Parallel() + assertEditRejectsBlockedDestination(t, env) + }) + + t.Run("a query-string destination", func(t *testing.T) { + t.Parallel() + assertEditIgnoresQueryString(t, env) + }) + + headerCases := map[string]string{ + "no colon": "Authorization Bearer token", + "empty name": ": value", + "invalid name": "X Bad Name: value", + "reserved header": "User-Agent: curl/8", + "duplicate name": "X-A: one\nx-a: two", + } + + for name, headers := range headerCases { + t.Run("headers: "+name, func(t *testing.T) { + t.Parallel() + assertEditRejectsHeaders(t, env, headers) + }) + } + + timeoutCases := map[string]string{ + "not a number": "soon", + "negative": "-1", + "over ceiling": "100000", + } + + for name, timeout := range timeoutCases { + t.Run("timeout: "+name, func(t *testing.T) { + t.Parallel() + assertEditRejectsTimeout(t, env, timeout) + }) + } +} + +func assertEditRejectsBlockedDestination( + t *testing.T, env *sourceTestEnv, +) { + t.Helper() + + webhook, target := seedHTTPTarget(t, env, "", "") + + w := submitTargetEdit( + env, webhook.ID, target.ID, + editForm(editBlockedURL, "", ""), + ) + + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.Contains(t, w.Body.String(), "Invalid target URL") + assert.Equal( + t, editOriginalURL, + storedHTTPConfig(t, env, target.ID).URL, + "a rejected edit must leave the stored config alone", + ) +} + +// The ingress rule the create path already follows applies to the +// edit path too: reading a field with FormValue would let the request +// line carry the credential, and the request line is what logs, +// proxies and Referer headers record. +func assertEditIgnoresQueryString( + t *testing.T, env *sourceTestEnv, +) { + t.Helper() + + webhook, target := seedHTTPTarget(t, env, "", "") + + form := url.Values{} + form.Set("name", "edited-name") + + w := serveTarget( + env, http.MethodPost, + "/source/"+webhook.ID+"/targets/"+target.ID+ + "/edit?url="+url.QueryEscape(editReplacedURL)+ + "&headers="+url.QueryEscape(editAuthHeader), + form, + ) + + assert.Equal(t, http.StatusBadRequest, w.Code) + + cfg := storedHTTPConfig(t, env, target.ID) + assert.Equal(t, editOriginalURL, cfg.URL) + assert.Empty(t, cfg.Headers) +} + +func assertEditRejectsHeaders( + t *testing.T, env *sourceTestEnv, headers string, +) { + t.Helper() + + webhook, target := seedHTTPTarget(t, env, "", "") + + w := submitTargetEdit( + env, webhook.ID, target.ID, + editForm(editOriginalURL, headers, ""), + ) + + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.Contains(t, w.Body.String(), "Invalid headers") + assert.Empty( + t, storedHTTPConfig(t, env, target.ID).Headers, + "a rejected header must not be stored", + ) +} + +func assertEditRejectsTimeout( + t *testing.T, env *sourceTestEnv, timeout string, +) { + t.Helper() + + webhook, target := seedHTTPTarget(t, env, "", "9") + + w := submitTargetEdit( + env, webhook.ID, target.ID, + editForm(editOriginalURL, "", timeout), + ) + + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.Contains(t, w.Body.String(), "Invalid timeout") + assert.Equal( + t, 9, storedHTTPConfig(t, env, target.ID).Timeout, + "a rejected timeout must leave the stored one alone", + ) +} + +// TestHandleTargetEdit_Scoping keeps the edit routes scoped the way +// the delete and toggle routes are: ownership is decided by the +// webhook, and the target is then scoped to it. +func TestHandleTargetEdit_Scoping(t *testing.T) { + t.Parallel() + + env := setupSourceTest(t) + + t.Run("a target of another webhook", func(t *testing.T) { + t.Parallel() + assertTargetOfAnotherWebhook404s(t, env) + }) + + t.Run("a webhook of another user", func(t *testing.T) { + t.Parallel() + assertWebhookOfAnotherUser404s(t, env) + }) +} + +// A target id from elsewhere must not become editable by pairing it +// with a webhook the user does own. +func assertTargetOfAnotherWebhook404s( + t *testing.T, env *sourceTestEnv, +) { + t.Helper() + + mine := seedWebhookWithRetention(t, env.db, 30) + _, target := seedHTTPTarget(t, env, "", "") + + get := serveTarget( + env, http.MethodGet, + "/source/"+mine.ID+"/targets/"+target.ID+"/edit", nil, + ) + assert.Equal(t, http.StatusNotFound, get.Code) + + post := submitTargetEdit( + env, mine.ID, target.ID, + editForm(editReplacedURL, "", ""), + ) + assert.Equal(t, http.StatusNotFound, post.Code) + + assert.Equal( + t, editOriginalURL, + storedHTTPConfig(t, env, target.ID).URL, + ) +} + +func assertWebhookOfAnotherUser404s( + t *testing.T, env *sourceTestEnv, +) { + t.Helper() + + other := &database.Webhook{ + UserID: "some-other-user", + Name: "not mine", + RetentionDays: 30, + } + require.NoError( + t, + env.db.DB().Omit(clause.Associations).Create(other).Error, + ) + + target := seedConfiguredTarget( + t, env.db, other.ID, database.TargetTypeHTTP, + `{"url":"`+editOriginalURL+`"}`, + ) + + w := serveTarget( + env, http.MethodGet, + "/source/"+other.ID+"/targets/"+target.ID+"/edit", nil, + ) + + assert.Equal(t, http.StatusNotFound, w.Code) +} diff --git a/internal/handlers/target_retries.go b/internal/handlers/target_retries.go new file mode 100644 index 0000000..9f571c5 --- /dev/null +++ b/internal/handlers/target_retries.go @@ -0,0 +1,119 @@ +package handlers + +import ( + "errors" + "net/http" + "strconv" + "strings" +) + +// maxTargetRetries bounds a target's max_retries. +// +// Both target forms already declare max="20" on the input, so this +// enforces server-side what the UI has always advertised rather than +// introducing a new limit. +// +// The number is not cosmetic. Every attempt writes a delivery_results +// row that the event log then loads and renders, and the engine backs +// off by 2^(n-1) seconds, so attempt 20 is already about six days +// after the first. A value beyond this buys no additional durability +// and only costs rows. +const maxTargetRetries = 20 + +// Errors returned when a max_retries form value cannot be turned into +// a retry count. +var ( + // errRetriesInvalid signals a max_retries form value that is not + // a non-negative whole number. + errRetriesInvalid = errors.New( + "retries must be a whole number of attempts", + ) + + // errRetriesTooLarge signals a max_retries form value that is a + // whole number but above maxTargetRetries. It is distinguished + // from errRetriesInvalid so the message can name the ceiling + // instead of implying the input was not a number. + errRetriesTooLarge = errors.New("retries out of range") +) + +// parseMaxRetries interprets a max_retries form value. +// +// An ABSENT value — the field empty or not submitted — yields +// fallback, which lets the create path apply its default and the edit +// path leave the stored value alone. A value that is SET BUT INVALID +// is an error: unparseable, negative, or above maxTargetRetries. +// +// The distinction is the whole point of this function. max_retries=0 +// means fire-and-forget, so returning 0 for input the operator typed +// but that did not parse silently disables retries on a +// store-and-forward proxy — and on the edit path it destroys a +// working retry configuration over a typo. A default answers a +// question that was not asked; it never answers one that was asked +// badly. +// +// A target stored with a count above the ceiling before this +// validation existed keeps rendering and keeps delivering — nothing +// clamps the row. Re-saving it from the edit form does have to bring +// it into range, because the form submits the pre-filled value back +// and accepting it would be the ceiling not applying to the edit +// path. The 400 names the ceiling, so the fix is one field. +func parseMaxRetries(raw string, fallback int) (int, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return fallback, nil + } + + v, err := strconv.Atoi(raw) + if err != nil || v < 0 { + return 0, errRetriesInvalid + } + + if v > maxTargetRetries { + return 0, errRetriesTooLarge + } + + return v, nil +} + +// retriesErrorMessage returns the message the create and edit forms +// show for a rejected max_retries value. Any error other than +// errRetriesTooLarge falls back to the generic wording, so an +// unrecognised parse failure still produces a sensible 400. +func retriesErrorMessage(err error) string { + if errors.Is(err, errRetriesTooLarge) { + return errRetriesTooLarge.Error() + + ": at most " + strconv.Itoa(maxTargetRetries) + + " retries" + } + + return errRetriesInvalid.Error() + + ", or 0 for fire-and-forget" +} + +// targetMaxRetries reads and validates max_retries from a target form +// submission, answering the request with a 400 and reporting false +// when the value is set but invalid. +// +// Both the create and the edit path go through here, so the two +// cannot come to disagree about what a valid retry count is. The +// wording matches the timeout control on the same submission. +func targetMaxRetries( + w http.ResponseWriter, + r *http.Request, + fallback int, +) (int, bool) { + retries, err := parseMaxRetries( + r.PostFormValue("max_retries"), fallback, + ) + if err != nil { + http.Error( + w, + "Invalid max retries: "+retriesErrorMessage(err), + http.StatusBadRequest, + ) + + return 0, false + } + + return retries, true +} diff --git a/internal/handlers/target_retries_test.go b/internal/handlers/target_retries_test.go new file mode 100644 index 0000000..51840c5 --- /dev/null +++ b/internal/handlers/target_retries_test.go @@ -0,0 +1,402 @@ +package handlers_test + +import ( + "net/http" + "net/url" + "strconv" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/handlers" +) + +// retriesTargetURL is the destination the retry-validation targets +// point at. It is a literal public address rather than a hostname so +// the SSRF check resolves nothing and a sandbox without DNS cannot +// make these cases pass or fail for the wrong reason. +const retriesTargetURL = "https://93.184.216.34/hooks/retries" + +const ( + // wayAboveCeiling is the typo'd-extra-zero case from the report. + wayAboveCeiling = "999999999" + + // notANumber is the plainest garbage an operator can type, and + // the value the report submitted on the edit form. + notANumber = "abc" + + // workingRetries is the retry count a seeded target is already + // delivering with, which a rejected submission must not disturb. + workingRetries = 2 +) + +// aboveCeiling is the smallest rejected whole number. +func aboveCeiling() string { + return strconv.Itoa(handlers.MaxTargetRetriesForTest + 1) +} + +// overCeilingRetries is whole-number input past the limit, which is +// rejected with the limit named. +func overCeilingRetries() []string { + return []string{aboveCeiling(), wayAboveCeiling} +} + +// unparseableRetries is input an operator can type into the field +// that is not a retry count. Each must be REJECTED: silently reading +// any of them as 0 turns a store-and-forward proxy into +// fire-and-forget without saying so. +// +// The twenty-digit case is here because it parses as digits but +// overflows int, which is the one failure the field's own min/max +// attributes cannot describe. +func unparseableRetries() []string { + return []string{ + notANumber, + "2.7", + "-5", + "12345678901234567890", + "1e3", + } +} + +// createRetriesForm is a complete, otherwise-valid HTTP target +// creation, so the only thing any case below varies is max_retries. +func createRetriesForm(retries string) url.Values { + form := url.Values{} + form.Set("name", "retries-target") + form.Set("type", string(database.TargetTypeHTTP)) + form.Set("url", retriesTargetURL) + + if retries != absentField { + form.Set("max_retries", retries) + } + + return form +} + +// absentField marks a field the form does not submit at all, which is +// the case that legitimately takes a default and must stay distinct +// from a field submitted with garbage in it. +const absentField = "\x00absent" + +// absentRetries is every way of saying "the operator did not set +// this", each of which takes the default rather than a 400. Blank and +// whitespace-only count as absent here because they do in the timeout +// and retention controls on the same forms; a rule the fields do not +// share would be its own surprise. +func absentRetries() []string { + return []string{absentField, "", " "} +} + +// createWithRetries posts the target create form for a fresh webhook +// and returns the webhook and the response. +func createWithRetries( + t *testing.T, + env *sourceTestEnv, + retries string, +) (database.Webhook, int, string) { + t.Helper() + + webhook := seedWebhookWithRetention(t, env.db, 30) + + w := serveTarget( + env, http.MethodPost, + "/source/"+webhook.ID+"/targets", + createRetriesForm(retries), + ) + + return webhook, w.Code, w.Body.String() +} + +// TestTargetCreate_RetriesAboveCeilingRejected proves the create form +// enforces a ceiling at all, and that the 400 names it — a rejection +// that does not say what the limit is leaves the operator guessing. +func TestTargetCreate_RetriesAboveCeilingRejected(t *testing.T) { + t.Parallel() + + env := setupSourceTest(t) + ceiling := strconv.Itoa(handlers.MaxTargetRetriesForTest) + + for _, retries := range overCeilingRetries() { + webhook, code, body := createWithRetries(t, env, retries) + + assert.Equal(t, http.StatusBadRequest, code, + "max_retries=%s should be rejected", retries) + assert.Contains(t, body, ceiling, + "the rejection for %s should name the ceiling", + retries) + assert.Empty(t, + targetsForWebhook(t, env.db, webhook.ID), + "no target should be created for %s", retries) + } +} + +// TestTargetCreate_UnparseableRetriesRejected is the core of the +// defect: each of these was accepted with HTTP 200 and stored as 0. +func TestTargetCreate_UnparseableRetriesRejected(t *testing.T) { + t.Parallel() + + env := setupSourceTest(t) + + for _, retries := range unparseableRetries() { + webhook, code, body := createWithRetries(t, env, retries) + + assert.Equal(t, http.StatusBadRequest, code, + "max_retries=%q should be rejected, not coerced", + retries) + assert.Contains(t, body, "whole number", + "the rejection for %q should say why", retries) + assert.Empty(t, + targetsForWebhook(t, env.db, webhook.ID), + "no target should be created for %q", retries) + } +} + +// TestTargetCreate_ValidRetriesStored covers the accepting half, +// including the ceiling itself: a bound that rejects its own limit +// would make the advertised maximum unreachable. +func TestTargetCreate_ValidRetriesStored(t *testing.T) { + t.Parallel() + + env := setupSourceTest(t) + + for _, want := range []int{0, 3, handlers.MaxTargetRetriesForTest} { + webhook, code, body := createWithRetries( + t, env, strconv.Itoa(want), + ) + + require.Equal(t, http.StatusSeeOther, code, body) + + targets := targetsForWebhook(t, env.db, webhook.ID) + require.Len(t, targets, 1) + assert.Equal(t, want, targets[0].MaxRetries) + } +} + +// TestTargetCreate_AbsentRetriesTakesDefault keeps the two cases +// distinct. An omitted field is not an operator asking for something +// invalid, so it still gets the fire-and-forget default rather than a +// 400 — otherwise the fix above would make the form unusable. +func TestTargetCreate_AbsentRetriesTakesDefault(t *testing.T) { + t.Parallel() + + env := setupSourceTest(t) + + for _, retries := range absentRetries() { + webhook, code, body := createWithRetries(t, env, retries) + + require.Equal(t, http.StatusSeeOther, code, body) + + targets := targetsForWebhook(t, env.db, webhook.ID) + require.Len(t, targets, 1) + assert.Equal(t, 0, targets[0].MaxRetries, + "an absent max_retries should take the default") + } +} + +// seedRetriesTarget creates an HTTP target already delivering with +// workingRetries retries, through the real create handler. +func seedRetriesTarget( + t *testing.T, + env *sourceTestEnv, +) (database.Webhook, database.Target) { + t.Helper() + + webhook, code, body := createWithRetries( + t, env, strconv.Itoa(workingRetries), + ) + require.Equal(t, http.StatusSeeOther, code, body) + + targets := targetsForWebhook(t, env.db, webhook.ID) + require.Len(t, targets, 1) + require.Equal(t, workingRetries, targets[0].MaxRetries) + + return webhook, targets[0] +} + +// editRetriesForm is a complete edit submission that changes the +// target's name as well, so a rejected submission can be shown to +// have written nothing at all rather than merely to have left +// max_retries alone. +func editRetriesForm(retries string) url.Values { + form := url.Values{} + form.Set("name", "renamed-by-edit") + form.Set("url", retriesTargetURL) + + if retries != absentField { + form.Set("max_retries", retries) + } + + return form +} + +// assertEditRejectedAndUnchanged submits an edit expected to fail and +// checks both halves of the requirement: the 400 explains itself, and +// the target it was submitted against is untouched. +func assertEditRejectedAndUnchanged( + t *testing.T, + env *sourceTestEnv, + retries, wantReason string, +) { + t.Helper() + + webhook, target := seedRetriesTarget(t, env) + + w := submitTargetEdit( + env, webhook.ID, target.ID, editRetriesForm(retries), + ) + + assert.Equal(t, http.StatusBadRequest, w.Code, + "max_retries=%q should be rejected on edit", retries) + assert.Contains(t, w.Body.String(), wantReason, + "the rejection for %q should say why", retries) + + stored := storedTarget(t, env, target.ID) + assert.Equal(t, workingRetries, stored.MaxRetries, + "a rejected edit must not destroy the working retry "+ + "count with %q", retries) + assert.Equal(t, "retries-target", stored.Name, + "a rejected edit must write nothing at all") +} + +// TestTargetEdit_UnparseableRetriesRejected is the damaging half of +// the defect. A target delivering with two retries, re-saved with a +// typo in the field, returned 200 and was left with retries disabled +// and nothing said. +func TestTargetEdit_UnparseableRetriesRejected(t *testing.T) { + t.Parallel() + + env := setupSourceTest(t) + + for _, retries := range unparseableRetries() { + assertEditRejectedAndUnchanged( + t, env, retries, "whole number", + ) + } +} + +// TestTargetEdit_RetriesAboveCeilingRejected proves the ceiling +// applies to the edit path too, naming itself, so the two paths +// cannot disagree about what is storable. +func TestTargetEdit_RetriesAboveCeilingRejected(t *testing.T) { + t.Parallel() + + env := setupSourceTest(t) + ceiling := strconv.Itoa(handlers.MaxTargetRetriesForTest) + + for _, retries := range overCeilingRetries() { + assertEditRejectedAndUnchanged(t, env, retries, ceiling) + } +} + +// TestTargetEdit_ValidRetriesStored covers the accepting half of the +// edit path, so the ceiling cannot be enforced by simply refusing +// every submission. +func TestTargetEdit_ValidRetriesStored(t *testing.T) { + t.Parallel() + + env := setupSourceTest(t) + + for _, want := range []int{0, 9, handlers.MaxTargetRetriesForTest} { + webhook, target := seedRetriesTarget(t, env) + + w := submitTargetEdit( + env, webhook.ID, target.ID, + editRetriesForm(strconv.Itoa(want)), + ) + require.Equal(t, + http.StatusSeeOther, w.Code, w.Body.String(), + ) + + assert.Equal(t, want, + storedTarget(t, env, target.ID).MaxRetries) + } +} + +// TestTargetEdit_AbsentRetriesLeavesStoredValue is the edit path's +// absent-versus-invalid case. Retries are only offered by the forms +// for types that retry, so a submission without the field must leave +// the stored count alone rather than be rejected or zeroed. +func TestTargetEdit_AbsentRetriesLeavesStoredValue(t *testing.T) { + t.Parallel() + + env := setupSourceTest(t) + + for _, retries := range absentRetries() { + webhook, target := seedRetriesTarget(t, env) + + w := submitTargetEdit( + env, webhook.ID, target.ID, + editRetriesForm(retries), + ) + require.Equal(t, + http.StatusSeeOther, w.Code, w.Body.String(), + ) + + assert.Equal(t, workingRetries, + storedTarget(t, env, target.ID).MaxRetries, + "an absent max_retries must leave the stored "+ + "count alone (%q)", retries) + } +} + +// TestTargetRetries_CreateAndEditAgreeOnEveryCase proves the two +// paths cannot disagree, which is what let the create form and the +// edit form drift apart in the first place. Every input is submitted +// to both and the accept/reject verdicts are compared. +func TestTargetRetries_CreateAndEditAgreeOnEveryCase(t *testing.T) { + t.Parallel() + + env := setupSourceTest(t) + + accepted := []string{ + "0", "1", + strconv.Itoa(handlers.MaxTargetRetriesForTest), + } + overCeiling := overCeilingRetries() + unparseable := unparseableRetries() + + cases := make( + []string, 0, + len(accepted)+len(overCeiling)+len(unparseable), + ) + cases = append(cases, accepted...) + cases = append(cases, overCeiling...) + cases = append(cases, unparseable...) + + for _, retries := range cases { + _, createCode, _ := createWithRetries(t, env, retries) + + webhook, target := seedRetriesTarget(t, env) + editCode := submitTargetEdit( + env, webhook.ID, target.ID, + editRetriesForm(retries), + ).Code + + assert.Equal(t, + createCode == http.StatusBadRequest, + editCode == http.StatusBadRequest, + "create and edit must agree on max_retries=%q "+ + "(create %d, edit %d)", + retries, createCode, editCode, + ) + } +} + +// TestPageOrFirst_CoercesRatherThanRejects pins the one place a +// non-numeric form value legitimately falls back. A page number says +// where to send the browser after an action that has already +// happened, so it is not configuration and rejecting it would report +// a failure that did not occur. +func TestPageOrFirst_CoercesRatherThanRejects(t *testing.T) { + t.Parallel() + + for _, s := range []string{"", "abc", "0", "-1", "2.7", " "} { + assert.Equal(t, 1, handlers.PageOrFirstForTest(s), + "%q should fall back to the first page", s) + } + + assert.Equal(t, 4, handlers.PageOrFirstForTest("4")) + assert.Equal(t, 4, handlers.PageOrFirstForTest(" 4 ")) +} diff --git a/internal/handlers/ui_copy_test.go b/internal/handlers/ui_copy_test.go new file mode 100644 index 0000000..da9b6d6 --- /dev/null +++ b/internal/handlers/ui_copy_test.go @@ -0,0 +1,302 @@ +package handlers_test + +import ( + "context" + "net/http" + "net/http/httptest" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "sneak.berlin/go/webhooker/internal/database" + "sneak.berlin/go/webhooker/internal/delivery" + "sneak.berlin/go/webhooker/internal/handlers" + "sneak.berlin/go/webhooker/internal/session" +) + +// Template data keys the page templates read. The handlers package has +// its own unexported constants for these; this is the external test +// package, so it needs its own. +const ( + dataKeyWebhook = "Webhook" + dataKeyError = "Error" +) + +// testWebhookID is the identifier given to the webhook under test on +// pages that render one. +const testWebhookID = "wh-1" + +// renderPage renders a page template through the real template set as +// an authenticated user and returns the resulting HTML. +func renderPage( + t *testing.T, + h *handlers.Handlers, + sess *session.Session, + page string, + data map[string]any, +) string { + t.Helper() + + cookies := authenticatedCookies(t, sess, "test-user-id", "testuser") + + req := httptest.NewRequestWithContext( + context.Background(), http.MethodGet, "/", nil, + ) + for _, c := range cookies { + req.AddCookie(c) + } + + w := httptest.NewRecorder() + h.RenderTemplateForTest(w, req, page, data) + + return w.Body.String() +} + +// TestNavbarUsesWebhookTerminology pins the user-visible navigation +// label to "Webhooks". The /sources route is deliberately unchanged, so +// the assertion targets the link text rather than the href. +func TestNavbarUsesWebhookTerminology(t *testing.T) { + t.Parallel() + + var h *handlers.Handlers + + var sess *session.Session + + app := newTestApp(t, &h, &sess) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + // One item, so the list body renders too: it calls + // WebhookListItem.RetentionLabel, promoted from the embedded + // Webhook and therefore a pointer method. An empty list would + // skip that call and hide a template error behind the + // navigation assertions below. + item := handlers.WebhookListItem{} + item.Name = "wh" + item.ID = testWebhookID + item.RetentionDays = 14 + + body := renderPage(t, h, sess, "sources_list.html", map[string]any{ + "Webhooks": []handlers.WebhookListItem{item}, + }) + + assert.Contains(t, body, "Retention: 14 days") + assert.Contains(t, body, `class="btn-text">Webhooks`) + assert.Contains( + t, body, `class="btn-text w-full text-left">Webhooks`, + ) + assert.Contains( + t, body, + `

Webhooks

`, + ) + assert.NotContains( + t, body, ">Sources<", + "no user-visible element may still be labelled Sources", + ) + assert.Contains( + t, body, `href="/sources"`, + "the /sources route itself must not change", + ) +} + +// TestEditPageUsesWebhookTerminology pins the edit page's heading and +// its back link. The link's href still points at /source/{id}, which is +// intentional: only user-visible copy changes. +func TestEditPageUsesWebhookTerminology(t *testing.T) { + t.Parallel() + + var h *handlers.Handlers + + var sess *session.Session + + app := newTestApp(t, &h, &sess) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + // The webhook goes in as a pointer because source_edit.html calls + // Webhook.RetentionLabel, a pointer method: a map element is not + // addressable, so a value here renders an error instead of the + // page. + webhook := &database.Webhook{Name: "wh", RetentionDays: 14} + webhook.ID = testWebhookID + + body := renderPage(t, h, sess, "source_edit.html", map[string]any{ + dataKeyWebhook: webhook, + dataKeyError: "", + }) + + assert.Contains(t, body, "Edit Webhook") + assert.NotContains(t, body, ">Sources<") + assert.Contains(t, body, `href="/source/wh-1"`) +} + +// TestCreateFormRetentionCopyMatchesBehaviour pins the create form's +// retention copy to what the code does: the reaper permanently deletes +// events past the cutoff, an empty field falls back to +// DefaultRetentionDays, and 0 is rewritten to the retain-forever +// sentinel by Webhook.BeforeSave. +func TestCreateFormRetentionCopyMatchesBehaviour(t *testing.T) { + t.Parallel() + + var h *handlers.Handlers + + var sess *session.Session + + app := newTestApp(t, &h, &sess) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + body := renderPage(t, h, sess, "sources_new.html", map[string]any{ + "Name": "", + "Description": "", + "DefaultRetentionDays": database.DefaultRetentionDays, + dataKeyError: "", + }) + + assert.Contains( + t, body, + "permanently deletes events older than this", + "the form must say retention is enforced by deletion", + ) + assert.Contains(t, body, "Enter 0 to retain events forever") + assert.Contains( + t, body, + "leave blank to use the default of "+ + strconv.Itoa(database.DefaultRetentionDays)+" days", + "blank means the default, not forever", + ) +} + +// TestEditFormRetentionCopyMatchesBehaviour pins the edit form's +// retention copy, including that it states the stored policy via +// RetentionLabel and that an empty field leaves that policy unchanged +// rather than meaning forever. +func TestEditFormRetentionCopyMatchesBehaviour(t *testing.T) { + t.Parallel() + + var h *handlers.Handlers + + var sess *session.Session + + app := newTestApp(t, &h, &sess) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + finite := &database.Webhook{Name: "wh", RetentionDays: 14} + finite.ID = testWebhookID + + body := renderPage(t, h, sess, "source_edit.html", map[string]any{ + dataKeyWebhook: finite, + dataKeyError: "", + }) + + assert.Contains(t, body, "Currently 14 days.") + assert.Contains( + t, body, + "permanently deletes events older than this", + ) + assert.Contains(t, body, "Enter 0 to retain events forever") + assert.Contains( + t, body, + "leave blank to keep the current setting", + "blank means unchanged, not forever", + ) + + forever := &database.Webhook{ + Name: "wh", + RetentionDays: database.RetentionForeverDays, + } + forever.ID = "wh-2" + + foreverBody := renderPage( + t, h, sess, "source_edit.html", map[string]any{ + dataKeyWebhook: forever, + dataKeyError: "", + }, + ) + + assert.Contains( + t, foreverBody, "Currently forever.", + "a retain-forever webhook must not read as a day count", + ) + assert.Contains( + t, foreverBody, + "No events are deleted while retention is set to forever", + ) + assert.NotContains( + t, foreverBody, + "permanently deletes events older than this", + "the reaper skips retain-forever webhooks, so the form "+ + "must not claim it deletes their events", + ) +} + +// TestEntrypointCopyButtonIsProgressiveEnhancement proves the copy +// affordance degrades: the button ships with the hidden attribute, so a +// browser that never runs app.js shows no dead control, and the URL is +// rendered as ordinary selectable text either way. +func TestEntrypointCopyButtonIsProgressiveEnhancement(t *testing.T) { + t.Parallel() + + var h *handlers.Handlers + + var sess *session.Session + + app := newTestApp(t, &h, &sess) + app.RequireStart() + + t.Cleanup(app.RequireStop) + + entrypoint := database.Entrypoint{Path: "abc123"} + entrypoint.ID = "ep-1" + + // The webhook goes in as a pointer because source_detail.html + // calls Webhook.RetentionLabel, a pointer method: a map element + // is not addressable, so a value here aborts execution partway + // down the page, after the copy button has already been flushed + // to the response. + webhook := &database.Webhook{Name: "wh", RetentionDays: 14} + webhook.ID = testWebhookID + webhook.CreatedAt = time.Date( + 2026, time.January, 2, 3, 4, 5, 0, time.UTC, + ) + + body := renderPage(t, h, sess, "source_detail.html", map[string]any{ + dataKeyWebhook: webhook, + // The handler passes projected views, never raw rows — a + // target carries its stored credential — so the test data + // has that same shape. + "Entrypoints": handlers.NewEntrypointViews( + []database.Entrypoint{entrypoint}, + ), + "Targets": delivery.NewTargetViews(nil), + "Events": []database.Event{}, + "BaseURL": "https://hooks.example.com", + }) + + assert.Contains( + t, body, + `