Compare commits
1 Commits
next
...
595d352d6e
| Author | SHA1 | Date | |
|---|---|---|---|
| 595d352d6e |
@@ -1,13 +1,5 @@
|
|||||||
# .ci-fingerprint is deliberately NOT excluded: it is the CI cache barrier
|
|
||||||
# that keeps the check stages from replaying a cached pass. See the lint
|
|
||||||
# stage of the Dockerfile.
|
|
||||||
.git/
|
.git/
|
||||||
bin/
|
bin/
|
||||||
# Third-party browser assets are fetched and hash-verified inside the build by
|
|
||||||
# script/fetch-assets. Excluding any host copy keeps a developer's working tree
|
|
||||||
# from supplying the bytes that get shipped. The script and its
|
|
||||||
# static/vendor.sha256 manifest stay in the context.
|
|
||||||
static/js/alpine.min.js
|
|
||||||
*.md
|
*.md
|
||||||
LICENSE
|
LICENSE
|
||||||
.editorconfig
|
.editorconfig
|
||||||
|
|||||||
@@ -11,33 +11,5 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 2024-10-23
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 2024-10-23
|
||||||
with:
|
|
||||||
# The fingerprint step below needs history to find the last commit
|
|
||||||
# that touched the Docker build context, and the superseded-status
|
|
||||||
# step needs it to walk ancestors (it aborts on a shallow clone).
|
|
||||||
fetch-depth: 0
|
|
||||||
|
|
||||||
- name: Mark superseded run statuses
|
|
||||||
# Gitea cancels the in-flight run when another commit is pushed to the
|
|
||||||
# same branch and records the cancellation as `failure`, so a commit
|
|
||||||
# that was never tested reads as a test result. The script rewrites
|
|
||||||
# those statuses to say what happened. See its header for why the
|
|
||||||
# state stays `failure` and not `skipped`.
|
|
||||||
env:
|
|
||||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
|
||||||
run: script/ci-mark-superseded
|
|
||||||
|
|
||||||
- name: Fingerprint the build context
|
|
||||||
# `.dockerignore` keeps docs out of the build context, so a docs-only
|
|
||||||
# commit legitimately replays the whole image from cache and stays
|
|
||||||
# cheap. Every other commit writes a new fingerprint into the context,
|
|
||||||
# which invalidates the `COPY . .` layer of both check stages: a
|
|
||||||
# commit that was never linted, formatted-checked, tested and built
|
|
||||||
# cannot report success from cache.
|
|
||||||
run: |
|
|
||||||
set -eu
|
|
||||||
fp="$(git log -1 --format=%H -- . ':!*.md' ':!LICENSE' ':!.editorconfig')"
|
|
||||||
printf '%s\n' "${fp:-$GITHUB_SHA}" > .ci-fingerprint
|
|
||||||
|
|
||||||
- name: Build Docker image (runs make check)
|
- name: Build Docker image (runs make check)
|
||||||
run: script/cibuild
|
run: script/cibuild
|
||||||
|
|||||||
8
.gitignore
vendored
8
.gitignore
vendored
@@ -42,11 +42,3 @@ data/
|
|||||||
# Temporary files
|
# Temporary files
|
||||||
tmp/
|
tmp/
|
||||||
temp/
|
temp/
|
||||||
|
|
||||||
# CI cache barrier, written into the build context by the check workflow
|
|
||||||
.ci-fingerprint
|
|
||||||
|
|
||||||
# Third-party browser assets, fetched and hash-verified by
|
|
||||||
# script/fetch-assets against static/vendor.sha256. Not committed:
|
|
||||||
# REPO_POLICIES.md forbids minified bundles in version control.
|
|
||||||
/static/js/alpine.min.js
|
|
||||||
30
Dockerfile
30
Dockerfile
@@ -12,21 +12,12 @@ WORKDIR /src
|
|||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
|
|
||||||
# Copy source code. In CI the context also carries .ci-fingerprint, whose
|
# Copy source code
|
||||||
# value changes with every commit that touches the build context (see
|
|
||||||
# .gitea/workflows/check.yml). That invalidates this layer, so the checks
|
|
||||||
# below cannot report success by replaying a cached pass. Do not add it to
|
|
||||||
# .dockerignore.
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# Run formatting check and linter. golangci-lint is invoked directly rather
|
# Run formatting check and linter
|
||||||
# than through `make lint`: this stage is already the pinned linter image, and
|
|
||||||
# script/lint is a wrapper that builds Dockerfile.lint, so calling it here
|
|
||||||
# would need a docker daemon inside the build. Keep these steps in step with
|
|
||||||
# Dockerfile.lint, including --network=none (see its header for why).
|
|
||||||
RUN make fmt-check
|
RUN make fmt-check
|
||||||
RUN --network=none golangci-lint config verify --config .golangci.yml
|
RUN make lint
|
||||||
RUN --network=none golangci-lint run --config .golangci.yml ./...
|
|
||||||
|
|
||||||
# Build stage
|
# Build stage
|
||||||
# golang:1.26.1-bookworm (Debian-based), 2026-03-17
|
# golang:1.26.1-bookworm (Debian-based), 2026-03-17
|
||||||
@@ -37,9 +28,7 @@ FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a349228
|
|||||||
# Depend on lint stage passing
|
# Depend on lint stage passing
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
|
|
||||||
# jq is a runtime dependency of script/ci-mark-superseded, which the test
|
RUN apt-get update && apt-get install -y --no-install-recommends make && rm -rf /var/lib/apt/lists/*
|
||||||
# suite executes.
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends make curl ca-certificates jq && rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
WORKDIR /build
|
WORKDIR /build
|
||||||
|
|
||||||
@@ -47,18 +36,9 @@ WORKDIR /build
|
|||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
|
|
||||||
# Copy source code, including the .ci-fingerprint cache barrier described in
|
# Copy source code
|
||||||
# the lint stage above.
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# Fetch the third-party browser assets the UI serves. They are not committed
|
|
||||||
# (REPO_POLICIES.md forbids minified bundles in version control) and
|
|
||||||
# .dockerignore keeps any host copy out of the build context, so this step is
|
|
||||||
# the only way they enter the image. Each download is checked against a
|
|
||||||
# hardcoded sha256 and the build fails on mismatch; make test re-checks the
|
|
||||||
# hashes against the bytes go:embed actually put in the binary.
|
|
||||||
RUN script/fetch-assets
|
|
||||||
|
|
||||||
# Run tests and build
|
# Run tests and build
|
||||||
RUN make test
|
RUN make test
|
||||||
RUN make build
|
RUN make build
|
||||||
|
|||||||
@@ -1,37 +0,0 @@
|
|||||||
# Lint-only image, built by script/lint. golangci-lint is never installed on
|
|
||||||
# the host: the repo is COPYed into the pinned image and linted as a build
|
|
||||||
# step, so a successful build IS a clean lint. This works even when the docker
|
|
||||||
# daemon is remote and bind mounts are impossible.
|
|
||||||
#
|
|
||||||
# script/lint passes --no-cache-filter=lint. Without it an unchanged tree
|
|
||||||
# replays the lint stage from cache and the build succeeds in under a second
|
|
||||||
# having run no linter at all. Do not drop that flag.
|
|
||||||
#
|
|
||||||
# The lint steps run with --network=none. `golangci-lint config verify` is
|
|
||||||
# documented as fetching its JSON schema over HTTPS, which would make linting
|
|
||||||
# depend on an unpinned remote artifact; this pinned image resolves the schema
|
|
||||||
# without any network, and --network=none enforces that rather than trusting
|
|
||||||
# it. It also proves no linter reaches out at analysis time. If a future image
|
|
||||||
# bump makes either step need the network, this build fails loudly instead of
|
|
||||||
# quietly acquiring an unpinned dependency.
|
|
||||||
|
|
||||||
# golangci/golangci-lint:v2.12.2 (Debian-based), 2026-08-07
|
|
||||||
# Using Debian-based image because mattn/go-sqlite3 (CGO) does not
|
|
||||||
# compile on Alpine musl (off64_t is a glibc type).
|
|
||||||
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS deps
|
|
||||||
|
|
||||||
WORKDIR /src
|
|
||||||
|
|
||||||
# Copy go mod files first for better layer caching. This stage is cacheable;
|
|
||||||
# only the lint stage below is forced to re-execute.
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
|
|
||||||
FROM deps AS lint
|
|
||||||
|
|
||||||
COPY . .
|
|
||||||
|
|
||||||
# `run` silently ignores config keys it does not recognize, so a typo would
|
|
||||||
# disable a setting without a word. `config verify` is what catches that.
|
|
||||||
RUN --network=none golangci-lint config verify --config .golangci.yml
|
|
||||||
RUN --network=none golangci-lint run --config .golangci.yml ./...
|
|
||||||
5
Makefile
5
Makefile
@@ -1,4 +1,4 @@
|
|||||||
.PHONY: bootstrap setup assets test lint fmt fmt-check check build run dev deps docker clean hooks css
|
.PHONY: bootstrap setup test lint fmt fmt-check check build run dev deps docker clean hooks css
|
||||||
|
|
||||||
# Default target
|
# Default target
|
||||||
.DEFAULT_GOAL := check
|
.DEFAULT_GOAL := check
|
||||||
@@ -9,9 +9,6 @@ bootstrap:
|
|||||||
setup:
|
setup:
|
||||||
@script/setup
|
@script/setup
|
||||||
|
|
||||||
assets:
|
|
||||||
@script/fetch-assets
|
|
||||||
|
|
||||||
test:
|
test:
|
||||||
@script/test
|
@script/test
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
---
|
---
|
||||||
title: Repository Policies
|
title: Repository Policies
|
||||||
last_modified: 2026-08-07
|
last_modified: 2026-07-06
|
||||||
---
|
---
|
||||||
|
|
||||||
This document covers repository structure, tooling, and workflow standards. Code
|
This document covers repository structure, tooling, and workflow standards. Code
|
||||||
@@ -189,13 +189,8 @@ style conventions are in separate documents:
|
|||||||
module under test to verify it compiles/parses. There is no excuse for
|
module under test to verify it compiles/parses. There is no excuse for
|
||||||
`make test` to be a no-op.
|
`make test` to be a no-op.
|
||||||
|
|
||||||
- `make test` must complete in under 60 seconds. That is the hard cap, and a
|
- `make test` must complete in under 20 seconds. Add a 30-second timeout in the
|
||||||
suite that exceeds it fails. Under 20 seconds is the target. A suite between
|
Makefile.
|
||||||
20 and 60 seconds is still green, but the overage must be filed as an
|
|
||||||
improvement bug against that repo. Add a 90-second timeout to the test
|
|
||||||
invocation in the Makefile (`go test -timeout 90s`). The backstop deliberately
|
|
||||||
sits above the hard cap so that it catches a genuinely hung test rather than a
|
|
||||||
merely slow one.
|
|
||||||
|
|
||||||
- **`make test` should use the conditional verbose rerun pattern.** Run tests
|
- **`make test` should use the conditional verbose rerun pattern.** Run tests
|
||||||
without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to
|
without `-v` (verbose) first. If tests fail, automatically rerun with `-v` to
|
||||||
@@ -214,9 +209,9 @@ style conventions are in separate documents:
|
|||||||
|
|
||||||
```makefile
|
```makefile
|
||||||
test:
|
test:
|
||||||
@go test -timeout 90s -race -cover ./... || \
|
@go test -timeout 30s -race -cover ./... || \
|
||||||
{ echo "--- Rerunning with -v for details ---"; \
|
{ echo "--- Rerunning with -v for details ---"; \
|
||||||
go test -timeout 90s -race -v ./...; exit 1; }
|
go test -timeout 30s -race -v ./...; exit 1; }
|
||||||
```
|
```
|
||||||
|
|
||||||
Python example:
|
Python example:
|
||||||
@@ -265,10 +260,7 @@ style conventions are in separate documents:
|
|||||||
|
|
||||||
- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only
|
- `.golangci.yml` is standardized and must _NEVER_ be modified by an agent, only
|
||||||
manually by the user. Fetch from
|
manually by the user. Fetch from
|
||||||
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`. The
|
`https://git.eeqj.de/sneak/prompts/raw/branch/main/.golangci.yml`.
|
||||||
canonical golangci-lint version is v2.12.2 (released 2026-05-06), installed
|
|
||||||
commit-pinned via
|
|
||||||
`go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@c0d3ddc9cf3faa61a4e378e879ece580256d76e5`.
|
|
||||||
|
|
||||||
- When pinning images or packages by hash, add a comment above the reference
|
- When pinning images or packages by hash, add a comment above the reference
|
||||||
with the version and date (YYYY-MM-DD).
|
with the version and date (YYYY-MM-DD).
|
||||||
|
|||||||
260
TODO.md
260
TODO.md
@@ -1,243 +1,31 @@
|
|||||||
# Workflow
|
# Workflow
|
||||||
|
|
||||||
One issue per unit of work, one branch and one PR per issue:
|
* branch (from `main`)
|
||||||
|
* do the work in Next Step
|
||||||
* ensure a tracked issue exists with a definition of done
|
* move Next Step to the top of Completed Steps
|
||||||
* branch from `next` (never from `main`)
|
* move the top item of Future Steps into Next Step
|
||||||
* do the work; open a PR based on `next` (never on `main`)
|
* commit (`TODO.md` changes in the same commit as the work)
|
||||||
* pass an independent review, then the manager squash-merges into `next`
|
* merge to `main` if the branch is not protected, otherwise open a PR
|
||||||
* push; nothing stays local-only
|
* push
|
||||||
|
|
||||||
`next` is the branch for the next milestone and must stay green and
|
|
||||||
mergeable to `main` without notice. One `next` -> `main` PR accumulates
|
|
||||||
the milestone; releases are cut from `main` separately.
|
|
||||||
|
|
||||||
Issue branches do NOT touch this file — the manager maintains it on
|
|
||||||
`next`. Every branch editing `TODO.md` conflicts with every other
|
|
||||||
(#112).
|
|
||||||
|
|
||||||
# Status
|
# Status
|
||||||
|
|
||||||
1.0.0 is complete: 55 closed, 0 open. `next` (6874059) is 62 commits
|
pre-1.0. No git tags exist. main (4f5ecb1) is a working webhook proxy
|
||||||
ahead of `main` and a strict fast-forward. No git tags exist yet.
|
with auth, CSRF/SSRF protections, login rate limiting, Slack target,
|
||||||
|
event retention (#63), the database archiving target (#43), the admin
|
||||||
The bar was not "the milestone is empty" but "sneak can deploy this and
|
password change flow (#65), policy compliance (#6), pinned lint tooling
|
||||||
use it in low-volume production". Every gap the deployability audit
|
(#55), and fail-loud configuration parsing (#80). Note: TODO.md was
|
||||||
named against that bar is now closed:
|
deliberately deleted from this repo in f9a9569 (2026-03-01, #6); its
|
||||||
|
content was folded into the README TODO section, which this draft
|
||||||
- `DATA_DIR` locking, so two instances cannot both deliver
|
reconstructs as of 2026-07-06.
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/201)
|
|
||||||
- shutdown on listener failure, rather than a live non-serving process
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/200)
|
|
||||||
- inbound signature verification
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/67)
|
|
||||||
- per-attempt delivery detail in the event log
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/202)
|
|
||||||
- replay of a terminally failed delivery
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/203)
|
|
||||||
- `ALLOWED_EGRESS_CIDRS`, an allowlist escape hatch for the SSRF guard
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/204)
|
|
||||||
- the three credential exposures
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/205,
|
|
||||||
https://git.eeqj.de/sneak/webhooker/issues/206,
|
|
||||||
https://git.eeqj.de/sneak/webhooker/issues/207)
|
|
||||||
|
|
||||||
One caveat on reading a green check: a docs-only commit deliberately
|
|
||||||
replays from the layer cache
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/119), so a green status on
|
|
||||||
such a commit evidences a replay rather than an executed run. A code
|
|
||||||
commit invalidates the `COPY` layer and genuinely executes.
|
|
||||||
|
|
||||||
# Next Step
|
# Next Step
|
||||||
|
|
||||||
Merge the milestone PR (https://git.eeqj.de/sneak/webhooker/pulls/111)
|
Manual event redelivery from the web UI (replay is a core promised
|
||||||
and tag `v1.0.0`. It is `merge-ready` and assigned to sneak; nothing
|
capability in the README rationale).
|
||||||
else gates it.
|
|
||||||
|
|
||||||
Post-1.0 follow-ups are open, none blocking the tag:
|
|
||||||
https://git.eeqj.de/sneak/webhooker/issues/245,
|
|
||||||
https://git.eeqj.de/sneak/webhooker/issues/246,
|
|
||||||
https://git.eeqj.de/sneak/webhooker/issues/247 and
|
|
||||||
https://git.eeqj.de/sneak/webhooker/issues/248. Also still open and
|
|
||||||
unmilestoned: https://git.eeqj.de/sneak/webhooker/issues/193 (a design
|
|
||||||
question, not a defect), https://git.eeqj.de/sneak/webhooker/issues/198
|
|
||||||
(`make test` is past the org 20s target) and
|
|
||||||
https://git.eeqj.de/sneak/webhooker/issues/212 (encrypting target config
|
|
||||||
at rest).
|
|
||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- 2026-08-18 Raise `script/test`'s per-package timeout from 30s to 90s,
|
|
||||||
matching the org-wide backstop. `go test` applies `-timeout` per
|
|
||||||
package, and `internal/handlers` had grown past the old budget: a
|
|
||||||
cache-defeated build failed outright at `GOMAXPROCS=4`, and every run
|
|
||||||
under deliberate host load breached 30s. The measurement table lives
|
|
||||||
in the script (#194)
|
|
||||||
- 2026-08-18 Re-sync `REPO_POLICIES.md` from `prompts`. The local copy
|
|
||||||
was stale and still mandated a 20s test target with a 30s timeout,
|
|
||||||
which the org replaced with a 60s cap and a 90s backstop. A synced
|
|
||||||
copy is not a source; reading it as one nearly produced a PR against
|
|
||||||
`prompts` proposing a change already merged there (#196)
|
|
||||||
- 2026-08-18 Report handler panics through the logger and answer 500.
|
|
||||||
chi v1.5.5's `Recoverer` scans for a `panic(0x` frame the runtime no
|
|
||||||
longer emits, then indexes `pkg[-1:]`, so it panicked inside its own
|
|
||||||
stack printer before writing a byte: the recovery never ran, the
|
|
||||||
client got a dropped connection instead of a 500, and the original
|
|
||||||
panic was lost. A local middleware replaces it, bounded by
|
|
||||||
`MaxPanicLogLineBytes` (#187)
|
|
||||||
- 2026-08-18 Route GORM's logger through `slog` and bound it. Every
|
|
||||||
`gorm.Open` left `logger.Default` in place at `Warn` with
|
|
||||||
`IgnoreRecordNotFoundError` false, so **every record-not-found
|
|
||||||
printed the fully interpolated SQL to stdout** — including the
|
|
||||||
client-chosen path on `/webhook/{uuid}` and the submitted username on
|
|
||||||
the login form, at no level the operator set and outside
|
|
||||||
`internal/logger` entirely. Three call sites, not the two the issue
|
|
||||||
named (#178)
|
|
||||||
- 2026-08-18 Bound every `slog` line against client-chosen text. Eight
|
|
||||||
sites reachable unauthenticated, found by reading every `slog` call in
|
|
||||||
the tree rather than only the one reported; the budget moved to a
|
|
||||||
shared `internal/logfield` so no second truncation exists. `DEBUG`
|
|
||||||
being off by default is not a bound and is not treated as one (#176)
|
|
||||||
- 2026-08-18 Stop a slow host turning a login-guard test into a
|
|
||||||
segfault. A non-fatal `assert` on an acquire result was dereferenced
|
|
||||||
on the next line, so one timing miss killed the whole
|
|
||||||
`internal/middleware` binary and reddened CI for unrelated PRs. The
|
|
||||||
fix also removed a real production race — `acquire` could shed a
|
|
||||||
request with a slot standing free, because Go picks uniformly among
|
|
||||||
ready `select` cases (#186)
|
|
||||||
- 2026-08-18 Send the chi route pattern to Sentry rather than the
|
|
||||||
concrete path. The receiver's path carries the entrypoint capability
|
|
||||||
token, so every Sentry event from `/webhook/{uuid}` shipped a live
|
|
||||||
credential to a third party. Request `Data`, `QueryString`, `Cookies`
|
|
||||||
and `Env` are dropped and headers reduced to an allowlist (#179)
|
|
||||||
- 2026-08-18 Read form fields from the POST body only. `r.FormValue`
|
|
||||||
merges the query string, so a login could be driven by URL parameters
|
|
||||||
— putting the password somewhere that lands in access logs, proxy
|
|
||||||
logs and browser history (#160)
|
|
||||||
- 2026-08-18 Verify login credentials before spending rate-limit
|
|
||||||
budget, so a flood of wrong passwords cannot lock out the account it
|
|
||||||
is guessing at. The manager took this decision rather than stall the
|
|
||||||
queue; it is flagged on the issue for reversal (#150)
|
|
||||||
- 2026-08-18 Run all linting in Docker via `Dockerfile.lint`. Host lint
|
|
||||||
was wrong in both directions from version skew and shared caches.
|
|
||||||
`script/lint` asserts the summary line, because `--no-cache-filter`
|
|
||||||
silently ignores a stage name it does not match — the flag that makes
|
|
||||||
the gate meaningful fails open (#109)
|
|
||||||
- 2026-08-18 Serve an event's full stored body over HTTP. The list
|
|
||||||
query truncates for rendering, and that truncated value was the only
|
|
||||||
way to read a body, so the full payload was unreachable (#157)
|
|
||||||
- 2026-08-18 Bound the access log line against client-chosen text.
|
|
||||||
`internal/logfield` budgets by *encoded* bytes, not runes, so a
|
|
||||||
handler's JSON escaping cannot multiply a field past its allowance
|
|
||||||
(#146)
|
|
||||||
- 2026-08-18 Mark superseded CI commits `failure` rather than
|
|
||||||
`skipped`. A skipped run rolls up green, so a commit that was never
|
|
||||||
tested reported success (#152)
|
|
||||||
- 2026-08-18 Set `fx.StopTimeout` inside the container stop grace, so
|
|
||||||
shutdown hooks are bounded by a deadline the orchestrator will
|
|
||||||
actually honour rather than being killed mid-flush (#134)
|
|
||||||
- 2026-08-17 Bucket IPv6 rate-limit keys by `/64`. A single allocation
|
|
||||||
hands out 2^64 addresses, so per-address keying let one client mint
|
|
||||||
unlimited buckets. Manager decision, recorded on the issue (#125)
|
|
||||||
- 2026-08-17 Correct release-blocking README and startup-warning
|
|
||||||
inaccuracies, including claims about behaviour the code does not have
|
|
||||||
(#151)
|
|
||||||
- 2026-08-17 Fetch and verify Alpine.js at build time against
|
|
||||||
`static/vendor.sha256` instead of committing the minified blob, so
|
|
||||||
the dependency is pinned by hash rather than by trust (#145)
|
|
||||||
- 2026-08-17 Bound the event log's rendered bodies in the query itself,
|
|
||||||
so a large stored payload cannot be read into memory just to be
|
|
||||||
truncated for display (#135)
|
|
||||||
- 2026-08-17 Mask the `http` target's destination URL in the UI: it can
|
|
||||||
carry a bearer credential in its path or query, and was rendered
|
|
||||||
verbatim. Manager decision to mask unconditionally (#115)
|
|
||||||
- 2026-08-14 Bound shutdown hooks by their stop context, so a hook that
|
|
||||||
hangs cannot hold the process past its grace period (#102)
|
|
||||||
- 2026-08-14 Render templates via a buffer rather than the
|
|
||||||
`ResponseWriter`, so a template error part-way through cannot commit
|
|
||||||
a 200 and then fail — the response is written only once it is whole
|
|
||||||
(#123)
|
|
||||||
- 2026-08-14 Align the session codec's max-age with the 7-day absolute
|
|
||||||
cap. The codec accepted cookies the session layer considered expired,
|
|
||||||
so the cap was enforced in one place and not the other (#108)
|
|
||||||
- 2026-08-12 Warn when `TRUSTED_PROXIES` is empty in production, where
|
|
||||||
the safe default silently discards forwarded headers and every client
|
|
||||||
rate-limits as the proxy's address (#149)
|
|
||||||
- 2026-08-12 Bound the receiver rate limit per client IP across the
|
|
||||||
whole `/webhook/*` route. The existing limiter keyed on the request
|
|
||||||
path and `/webhook/{uuid}` matches any single segment, so a client
|
|
||||||
that invented a fresh path per request minted a fresh bucket per
|
|
||||||
request: the limit on the only unauthenticated endpoint bounded
|
|
||||||
nothing in aggregate, and every request still cost an entrypoint
|
|
||||||
lookup before it 404ed. An outer limiter keyed on the client address
|
|
||||||
alone now bounds that, chained in front of the unchanged
|
|
||||||
per-entrypoint limiter (#139)
|
|
||||||
- 2026-08-12 Correct release-blocking documentation inaccuracies: the
|
|
||||||
README promised manual redelivery in the present tense in three
|
|
||||||
places when nothing implements it (the same false claim also sat in
|
|
||||||
the doc comment that was its source text), the env table omitted
|
|
||||||
`RETENTION_SWEEP_INTERVAL`, and `TODO.md` itself omitted five landed
|
|
||||||
units (#141)
|
|
||||||
- 2026-08-12 Make the CI gate execute the checks it reports on. The
|
|
||||||
workflow now writes a build-context fingerprint before calling
|
|
||||||
`script/cibuild`, so a code commit invalidates the `COPY` layer of
|
|
||||||
the lint and builder stages while a docs-only commit still replays
|
|
||||||
from cache; a superseding run also rewrites the `failure` status
|
|
||||||
Gitea leaves on commits it cancelled and never tested. Verified by
|
|
||||||
pushing a deliberately broken test and watching CI go red (#119)
|
|
||||||
- 2026-08-12 Require a positive `RETENTION_SWEEP_INTERVAL`: a
|
|
||||||
non-positive value reached `time.NewTicker` in both the retention
|
|
||||||
reaper and the archive sweeper, panicking two goroutines with no
|
|
||||||
recover after startup had already reported success (#140)
|
|
||||||
- 2026-08-12 Bound the `X-Forwarded-For` scan's allocation to the hop
|
|
||||||
cap: the reverse walk cuts entries with `strings.LastIndexByte`
|
|
||||||
instead of joining and splitting, so a 1 MB header allocates 16 bytes
|
|
||||||
rather than 1.6 MB per request on the unauthenticated receiver.
|
|
||||||
Semantics proven unchanged by differential testing against the
|
|
||||||
previous implementation (#133)
|
|
||||||
- 2026-08-12 Cap the `X-Forwarded-For` hop walk at 64 entries, so an
|
|
||||||
attacker-supplied chain cannot burn unbounded CPU in the rate-limit
|
|
||||||
key function; running off the end falls back to the peer address
|
|
||||||
(#124)
|
|
||||||
- 2026-08-12 Gate forwarded-header trust behind a `TRUSTED_PROXIES` CIDR
|
|
||||||
list: all three rate limiters key on the connection's own address
|
|
||||||
unless the direct peer is a configured proxy, in which case
|
|
||||||
`X-Forwarded-For` is walked right to left for the first non-proxy hop.
|
|
||||||
Default trusts nothing, and a set-but-unparseable value aborts
|
|
||||||
startup. Before this, any client could mint a fresh bucket or drain
|
|
||||||
another's by rotating a spoofed header (#88)
|
|
||||||
- 2026-08-11 Web UI cleanup: nav terminology unified on Webhooks, the
|
|
||||||
Profile settings placeholder removed, a progressive-enhancement copy
|
|
||||||
button for the entrypoint URL, and retention form copy that states the
|
|
||||||
actual policy (deletion by the reaper, 0 retains forever) (#57)
|
|
||||||
- 2026-08-11 Mask the webhook credential in delivery errors and logs:
|
|
||||||
Go embeds the request URL in `*url.Error`, so every transport failure
|
|
||||||
persisted the full Slack webhook URL into the per-webhook event
|
|
||||||
database via `DeliveryResult.Error`, a field a future REST API would
|
|
||||||
have served. `maskURLError` drops path, query and userinfo while
|
|
||||||
preserving the wrapped cause, so `errors.Is`/`As` and `Timeout()`
|
|
||||||
still work and DNS, TLS and timeout failures still read differently
|
|
||||||
(#118)
|
|
||||||
- 2026-08-11 Rate-limit the public webhook receiver endpoint
|
|
||||||
(`RECEIVER_RATE_LIMIT`, default 120/min), keyed on client IP plus
|
|
||||||
entrypoint path so one entrypoint cannot exhaust another's budget;
|
|
||||||
over-limit requests get 429 with `Retry-After`. It was the one
|
|
||||||
unauthenticated, internet-facing endpoint with no limit at all (#64)
|
|
||||||
- 2026-08-11 Enforce the body size limit before CSRF parses the form:
|
|
||||||
`MaxBodySize` is now first in all four form-parsing route groups, so
|
|
||||||
an oversized request is rejected with 413 instead of being read in
|
|
||||||
full by the CSRF middleware before any cap applied (#90)
|
|
||||||
- 2026-08-11 Mask target config on the source detail page, which
|
|
||||||
rendered the stored blob verbatim and so exposed the Slack
|
|
||||||
incoming-webhook URL — a bearer credential that cannot be revoked
|
|
||||||
per-holder. Config reaches the template only as a `TargetView` of
|
|
||||||
labelled fields, and header values are rendered as a count (#113)
|
|
||||||
- 2026-08-11 Allow `retention_days` of 0 to mean retain forever, via a
|
|
||||||
sentinel written in `BeforeSave` so the GORM column default cannot
|
|
||||||
win the race. Also bounds the reaper's cutoff arithmetic: day counts
|
|
||||||
above 106751 overflowed `time.Duration` and wrapped the cutoff into
|
|
||||||
the future, where every row matched and the sweep deleted everything
|
|
||||||
(#79)
|
|
||||||
- 2026-08-09 Inactivity-based session timeout: sliding idle expiry
|
- 2026-08-09 Inactivity-based session timeout: sliding idle expiry
|
||||||
(`SESSION_IDLE_TIMEOUT`, default `24h`) refreshed on authenticated
|
(`SESSION_IDLE_TIMEOUT`, default `24h`) refreshed on authenticated
|
||||||
requests, with the 7-day absolute cap kept as an independent
|
requests, with the 7-day absolute cap kept as an independent
|
||||||
@@ -292,16 +80,11 @@ at rest).
|
|||||||
|
|
||||||
# Future Steps
|
# Future Steps
|
||||||
|
|
||||||
- Delivery status and retry management UI. Replay of a terminally
|
- Delivery status and retry management UI
|
||||||
failed delivery and per-attempt detail already landed
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/203,
|
|
||||||
https://git.eeqj.de/sneak/webhooker/issues/202)
|
|
||||||
- Per-webhook rate limiting in the receiver handler (per-webhook config
|
- Per-webhook rate limiting in the receiver handler (per-webhook config
|
||||||
plus handler enforcement; global limits must not apply to receiver
|
plus handler enforcement; global limits must not apply to receiver
|
||||||
endpoints)
|
endpoints)
|
||||||
- Stripe HMAC signature verification. The GitHub and GitLab schemes
|
- Webhook signature verification for GitHub and Stripe HMAC formats
|
||||||
landed with inbound verification
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/67)
|
|
||||||
- API key authentication for programmatic access (APIKey model exists;
|
- API key authentication for programmatic access (APIKey model exists;
|
||||||
Bearer token middleware does not)
|
Bearer token middleware does not)
|
||||||
- REST API v1
|
- REST API v1
|
||||||
@@ -311,10 +94,7 @@ at rest).
|
|||||||
- OpenAPI specification
|
- OpenAPI specification
|
||||||
- Analytics dashboard: success rates, response times, volume
|
- Analytics dashboard: success rates, response times, volume
|
||||||
- A remember-me option at login
|
- A remember-me option at login
|
||||||
- Password reset flow for a forgotten password over the web. The
|
- Password change and reset flow
|
||||||
authenticated password *change* flow already landed, and a lost
|
|
||||||
password is recoverable from the console with `webhooker resetpw`
|
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/208)
|
|
||||||
- Later, nice to have
|
- Later, nice to have
|
||||||
- email delivery target type
|
- email delivery target type
|
||||||
- SNS and S3 delivery targets
|
- SNS and S3 delivery targets
|
||||||
|
|||||||
@@ -2,58 +2,19 @@
|
|||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/datadir"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
"sneak.berlin/go/webhooker/internal/globals"
|
"sneak.berlin/go/webhooker/internal/globals"
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
"sneak.berlin/go/webhooker/internal/handlers"
|
||||||
"sneak.berlin/go/webhooker/internal/healthcheck"
|
"sneak.berlin/go/webhooker/internal/healthcheck"
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
"sneak.berlin/go/webhooker/internal/middleware"
|
"sneak.berlin/go/webhooker/internal/middleware"
|
||||||
"sneak.berlin/go/webhooker/internal/resetpw"
|
|
||||||
"sneak.berlin/go/webhooker/internal/server"
|
"sneak.berlin/go/webhooker/internal/server"
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
"sneak.berlin/go/webhooker/internal/session"
|
||||||
)
|
)
|
||||||
|
|
||||||
// stopTimeout bounds the whole fx stop sequence, not each hook.
|
|
||||||
//
|
|
||||||
// fx defaults to 15s, which is longer than Docker's 10s default
|
|
||||||
// stop grace: the container would be SIGKILLed before the bound
|
|
||||||
// could fire, so nothing bounded by it would ever be observed.
|
|
||||||
// 5s leaves headroom inside that grace for signal delivery and
|
|
||||||
// process exit; the observed wedge case already exits at ~5.3s,
|
|
||||||
// so a larger bound would trade a rare skipped database close for
|
|
||||||
// a more common hard kill.
|
|
||||||
//
|
|
||||||
// The server's stop hook must fit inside it with room to spare: a
|
|
||||||
// hook that used the whole budget would exhaust it at that instant,
|
|
||||||
// and fx would skip every hook after the server — the delivery
|
|
||||||
// engine, the healthcheck, the webhook DB manager and the database
|
|
||||||
// close. That hook is the 3s HTTP drain plus the Sentry flush that
|
|
||||||
// follows it in the same hook, so the flush is clamped to the stop
|
|
||||||
// context's remaining time less server.TailHookReserve rather than
|
|
||||||
// running for its own fixed 2s; the reserve is what the tail hooks
|
|
||||||
// live on, and they are microsecond-scale in normal operation.
|
|
||||||
// TestStopTimeout_LeavesHeadroomForTailHooks pins the arithmetic
|
|
||||||
// across every drain length.
|
|
||||||
//
|
|
||||||
// This does not make the database close unconditional: the
|
|
||||||
// ArchiveSweeper and RetentionReaper hooks run before the server
|
|
||||||
// and can still consume the whole budget on their own.
|
|
||||||
const stopTimeout = 5 * time.Second
|
|
||||||
|
|
||||||
// exitUsage is the status for a command line this binary cannot make
|
|
||||||
// sense of, kept distinct from the 1 a refusal exits with so that a
|
|
||||||
// caller can tell "called wrong" from "declined".
|
|
||||||
const exitUsage = 2
|
|
||||||
|
|
||||||
// Build-time variables set via -ldflags.
|
// Build-time variables set via -ldflags.
|
||||||
//
|
//
|
||||||
//nolint:gochecknoglobals // Build-time variables injected by the linker.
|
//nolint:gochecknoglobals // Build-time variables injected by the linker.
|
||||||
@@ -66,86 +27,7 @@ func main() {
|
|||||||
globals.Appname = appname
|
globals.Appname = appname
|
||||||
globals.Version = version
|
globals.Version = version
|
||||||
|
|
||||||
os.Exit(dispatch(os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
|
fx.New(
|
||||||
}
|
|
||||||
|
|
||||||
// dispatch routes the command line to a subcommand.
|
|
||||||
//
|
|
||||||
// No arguments runs the server, which is what the image's CMD and
|
|
||||||
// every existing deployment invoke; that path is unchanged, including
|
|
||||||
// where the DATA_DIR lock is taken relative to building the fx graph
|
|
||||||
// and how fx propagates a non-zero exit itself.
|
|
||||||
func dispatch(
|
|
||||||
args []string,
|
|
||||||
stdin io.Reader,
|
|
||||||
stdout, stderr io.Writer,
|
|
||||||
) int {
|
|
||||||
if len(args) == 0 {
|
|
||||||
return run(stderr)
|
|
||||||
}
|
|
||||||
|
|
||||||
switch args[0] {
|
|
||||||
case resetpw.Name:
|
|
||||||
return resetpw.Run(args[1:], stdin, stdout, stderr)
|
|
||||||
case "help", "-h", "-help", "--help":
|
|
||||||
usage(stdout)
|
|
||||||
|
|
||||||
return 0
|
|
||||||
default:
|
|
||||||
_, _ = fmt.Fprintf(
|
|
||||||
stderr, "%s: unknown subcommand %q\n", appname, args[0],
|
|
||||||
)
|
|
||||||
usage(stderr)
|
|
||||||
|
|
||||||
return exitUsage
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// usage lists what the binary can be asked to do.
|
|
||||||
func usage(w io.Writer) {
|
|
||||||
_, _ = fmt.Fprintf(w, `usage: %s [subcommand]
|
|
||||||
|
|
||||||
With no subcommand, runs the webhooker server.
|
|
||||||
|
|
||||||
Subcommands:
|
|
||||||
%s [-generate] <username>
|
|
||||||
Set an existing account's password on a stopped deployment.
|
|
||||||
Recovers an admin account whose bootstrap password was lost.
|
|
||||||
help
|
|
||||||
Print this message.
|
|
||||||
`, appname, resetpw.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
// run takes the exclusive DATA_DIR lock, then runs the application
|
|
||||||
// under it, and returns the process exit status.
|
|
||||||
//
|
|
||||||
// The lock is taken here rather than inside the fx graph because it has
|
|
||||||
// to be held before anything opens a database, and because a refusal
|
|
||||||
// has to reach the operator as a plain line on standard error rather
|
|
||||||
// than as one entry in an fx failure dump. It is released by the defer
|
|
||||||
// on a clean shutdown, and by the kernel closing the descriptor on any
|
|
||||||
// other exit — including the one fx performs itself when a start or
|
|
||||||
// stop hook fails, which skips deferred calls.
|
|
||||||
func run(stderr io.Writer) int {
|
|
||||||
lock, err := datadir.Acquire(config.DataDir())
|
|
||||||
if err != nil {
|
|
||||||
_, _ = fmt.Fprintf(stderr, "%s: %v\n", appname, err)
|
|
||||||
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = lock.Release() }()
|
|
||||||
|
|
||||||
newApp().Run()
|
|
||||||
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// newApp builds the application graph. It is separate from main so
|
|
||||||
// a test can assert the options it carries.
|
|
||||||
func newApp() *fx.App {
|
|
||||||
return fx.New(
|
|
||||||
fx.StopTimeout(stopTimeout),
|
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
@@ -157,10 +39,6 @@ func newApp() *fx.App {
|
|||||||
session.New,
|
session.New,
|
||||||
handlers.New,
|
handlers.New,
|
||||||
middleware.New,
|
middleware.New,
|
||||||
// The one SSRF guard both target-creation validation
|
|
||||||
// and the delivery dialer consult, so they cannot
|
|
||||||
// disagree about a destination.
|
|
||||||
delivery.NewGuard,
|
|
||||||
delivery.New,
|
delivery.New,
|
||||||
delivery.NewArchiveSweeper,
|
delivery.NewArchiveSweeper,
|
||||||
// Wire *delivery.Engine as delivery.Notifier so the
|
// Wire *delivery.Engine as delivery.Notifier so the
|
||||||
@@ -182,5 +60,5 @@ func newApp() *fx.App {
|
|||||||
) {
|
) {
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
)
|
).Run()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,171 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/datadir"
|
|
||||||
"sneak.berlin/go/webhooker/internal/resetpw"
|
|
||||||
"sneak.berlin/go/webhooker/internal/server"
|
|
||||||
)
|
|
||||||
|
|
||||||
// dockerStopGrace is Docker's default `docker stop` grace period.
|
|
||||||
// The Dockerfile sets no STOPSIGNAL or grace override, so this is
|
|
||||||
// the deadline the container is actually held to, and the fx stop
|
|
||||||
// timeout has to fit inside it with room for signal delivery and
|
|
||||||
// process exit.
|
|
||||||
const dockerStopGrace = 10 * time.Second
|
|
||||||
|
|
||||||
// TestNewApp_StopTimeout pins the fx stop timeout. Without the
|
|
||||||
// explicit fx.StopTimeout option the app reads fx's 15s
|
|
||||||
// DefaultTimeout, which exceeds dockerStopGrace: the container is
|
|
||||||
// SIGKILLed before the bound fires and every shutdown hook bounded
|
|
||||||
// by it — including the operator-facing timeout log — becomes
|
|
||||||
// unreachable in the image this repo produces.
|
|
||||||
//
|
|
||||||
// fx.New applies options before it executes invokes, so the timeout
|
|
||||||
// is set whether or not the graph itself can be constructed here.
|
|
||||||
func TestNewApp_StopTimeout(t *testing.T) {
|
|
||||||
t.Setenv("DATA_DIR", t.TempDir())
|
|
||||||
|
|
||||||
got := newApp().StopTimeout()
|
|
||||||
|
|
||||||
require.Equal(t, stopTimeout, got)
|
|
||||||
require.Less(t, got, dockerStopGrace)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunRefusesLockedDataDir pins what an operator's second start
|
|
||||||
// does. The entry point must refuse before it builds the fx graph —
|
|
||||||
// nothing may open a database in a DATA_DIR another process holds —
|
|
||||||
// and must exit non-zero with a message naming the directory rather
|
|
||||||
// than starting a second delivery engine over the same rows.
|
|
||||||
//
|
|
||||||
// flock(2) locks descriptors independently, so holding the lock here
|
|
||||||
// is the same denial a separate process gets; internal/datadir pins
|
|
||||||
// that property and covers the real two-process case.
|
|
||||||
func TestRunRefusesLockedDataDir(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
t.Setenv("DATA_DIR", dir)
|
|
||||||
|
|
||||||
lock, err := datadir.Acquire(dir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
defer func() { _ = lock.Release() }()
|
|
||||||
|
|
||||||
var stderr bytes.Buffer
|
|
||||||
|
|
||||||
code := run(&stderr)
|
|
||||||
|
|
||||||
require.Equal(
|
|
||||||
t, 1, code, "a second instance must exit non-zero",
|
|
||||||
)
|
|
||||||
assert.Contains(
|
|
||||||
t, stderr.String(), dir,
|
|
||||||
"the refusal must name the directory",
|
|
||||||
)
|
|
||||||
assert.Contains(t, stderr.String(), "another instance")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDispatch_NoArgumentsRunsTheServer pins the routing of a bare
|
|
||||||
// invocation, which is what the image's CMD and every deployment use.
|
|
||||||
// Adding subcommands must not move the server off the empty argument
|
|
||||||
// list, and must not move the DATA_DIR lock: this asserts the refusal
|
|
||||||
// arrives with no fx graph built, exactly as run does on its own.
|
|
||||||
func TestDispatch_NoArgumentsRunsTheServer(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
t.Setenv("DATA_DIR", dir)
|
|
||||||
|
|
||||||
lock, err := datadir.Acquire(dir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
defer func() { _ = lock.Release() }()
|
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
|
|
||||||
code := dispatch(nil, strings.NewReader(""), &stdout, &stderr)
|
|
||||||
|
|
||||||
require.Equal(t, 1, code)
|
|
||||||
assert.Contains(t, stderr.String(), "another instance")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDispatch_UnknownSubcommand keeps a mistyped subcommand from
|
|
||||||
// starting a server. Anything else would have `webhooker resetpww`
|
|
||||||
// silently take the DATA_DIR lock and serve.
|
|
||||||
func TestDispatch_UnknownSubcommand(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
|
|
||||||
code := dispatch(
|
|
||||||
[]string{"resetpww", "admin"},
|
|
||||||
strings.NewReader(""), &stdout, &stderr,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.Equal(t, 2, code)
|
|
||||||
assert.Contains(t, stderr.String(), "unknown subcommand")
|
|
||||||
assert.Contains(
|
|
||||||
t, stderr.String(), resetpw.Name,
|
|
||||||
"the usage must name the subcommand that does exist",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDispatch_Help answers on standard output with a zero status, so
|
|
||||||
// `webhooker help` is usable in a pipe.
|
|
||||||
func TestDispatch_Help(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var stdout, stderr bytes.Buffer
|
|
||||||
|
|
||||||
code := dispatch(
|
|
||||||
[]string{"help"}, strings.NewReader(""), &stdout, &stderr,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.Equal(t, 0, code)
|
|
||||||
assert.Empty(t, stderr.String())
|
|
||||||
assert.Contains(t, stdout.String(), resetpw.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
// tailHeadroom is the slack the fx stop budget must keep beyond the
|
|
||||||
// server stop hook. The hooks that run after the server — the
|
|
||||||
// delivery engine, the healthcheck, the webhook DB manager and the
|
|
||||||
// database close — are microsecond-scale in normal operation, so
|
|
||||||
// this is generous for them.
|
|
||||||
const tailHeadroom = 2 * time.Second
|
|
||||||
|
|
||||||
// TestStopTimeout_LeavesHeadroomForTailHooks pins the relationship
|
|
||||||
// between the server's stop hook and the fx stop budget. fx bounds
|
|
||||||
// the whole stop sequence, and returns without running its
|
|
||||||
// remaining hooks once the stop context has expired. If the hook
|
|
||||||
// could use the entire budget, every later hook — the database close
|
|
||||||
// included — would be skipped in exactly the case where the drain
|
|
||||||
// mattered.
|
|
||||||
//
|
|
||||||
// The hook is not just the HTTP drain: a Sentry flush follows it in
|
|
||||||
// the same hook, and sentry.Flush honours no context, so both halves
|
|
||||||
// have to be counted. The sweep walks every drain length the hook
|
|
||||||
// can produce, since a shorter drain leaves the flush more room and
|
|
||||||
// the worst case is not necessarily at either extreme.
|
|
||||||
//
|
|
||||||
// Shrinking either budget, or unbounding the flush again, must fail
|
|
||||||
// here rather than silently recreating a hook that swallows the
|
|
||||||
// whole sequence.
|
|
||||||
func TestStopTimeout_LeavesHeadroomForTailHooks(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
require.Less(t, server.ShutdownTimeout, stopTimeout)
|
|
||||||
|
|
||||||
const step = 10 * time.Millisecond
|
|
||||||
|
|
||||||
for drain := time.Duration(0); drain <= server.ShutdownTimeout; drain += step {
|
|
||||||
hook := drain + server.SentryFlushBudget(stopTimeout-drain)
|
|
||||||
|
|
||||||
require.LessOrEqual(
|
|
||||||
t, hook+tailHeadroom, stopTimeout,
|
|
||||||
"a %s drain leaves the tail hooks short", drain,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
9
go.mod
9
go.mod
@@ -8,18 +8,15 @@ require (
|
|||||||
github.com/go-chi/chi v1.5.5
|
github.com/go-chi/chi v1.5.5
|
||||||
github.com/go-chi/cors v1.2.1
|
github.com/go-chi/cors v1.2.1
|
||||||
github.com/go-chi/httprate v0.15.0
|
github.com/go-chi/httprate v0.15.0
|
||||||
github.com/gofrs/flock v0.13.0
|
|
||||||
github.com/google/uuid v1.6.0
|
github.com/google/uuid v1.6.0
|
||||||
github.com/gorilla/csrf v1.7.3
|
github.com/gorilla/csrf v1.7.3
|
||||||
github.com/gorilla/sessions v1.4.0
|
github.com/gorilla/sessions v1.4.0
|
||||||
github.com/joho/godotenv v1.5.1
|
github.com/joho/godotenv v1.5.1
|
||||||
github.com/prometheus/client_golang v1.18.0
|
github.com/prometheus/client_golang v1.18.0
|
||||||
github.com/prometheus/client_model v0.5.0
|
|
||||||
github.com/slok/go-http-metrics v0.11.0
|
github.com/slok/go-http-metrics v0.11.0
|
||||||
github.com/stretchr/testify v1.11.1
|
github.com/stretchr/testify v1.8.4
|
||||||
go.uber.org/fx v1.20.1
|
go.uber.org/fx v1.20.1
|
||||||
golang.org/x/crypto v0.38.0
|
golang.org/x/crypto v0.38.0
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
|
||||||
gorm.io/driver/sqlite v1.5.4
|
gorm.io/driver/sqlite v1.5.4
|
||||||
gorm.io/gorm v1.25.5
|
gorm.io/gorm v1.25.5
|
||||||
modernc.org/sqlite v1.28.0
|
modernc.org/sqlite v1.28.0
|
||||||
@@ -40,6 +37,7 @@ require (
|
|||||||
github.com/mattn/go-sqlite3 v1.14.17 // indirect
|
github.com/mattn/go-sqlite3 v1.14.17 // indirect
|
||||||
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
|
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||||
|
github.com/prometheus/client_model v0.5.0 // indirect
|
||||||
github.com/prometheus/common v0.45.0 // indirect
|
github.com/prometheus/common v0.45.0 // indirect
|
||||||
github.com/prometheus/procfs v0.12.0 // indirect
|
github.com/prometheus/procfs v0.12.0 // indirect
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||||
@@ -50,10 +48,11 @@ require (
|
|||||||
go.uber.org/zap v1.23.0 // indirect
|
go.uber.org/zap v1.23.0 // indirect
|
||||||
golang.org/x/mod v0.17.0 // indirect
|
golang.org/x/mod v0.17.0 // indirect
|
||||||
golang.org/x/sync v0.14.0 // indirect
|
golang.org/x/sync v0.14.0 // indirect
|
||||||
golang.org/x/sys v0.37.0 // indirect
|
golang.org/x/sys v0.33.0 // indirect
|
||||||
golang.org/x/text v0.25.0 // indirect
|
golang.org/x/text v0.25.0 // indirect
|
||||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
|
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
|
||||||
google.golang.org/protobuf v1.31.0 // indirect
|
google.golang.org/protobuf v1.31.0 // indirect
|
||||||
|
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||||
lukechampine.com/uint128 v1.2.0 // indirect
|
lukechampine.com/uint128 v1.2.0 // indirect
|
||||||
modernc.org/cc/v3 v3.40.0 // indirect
|
modernc.org/cc/v3 v3.40.0 // indirect
|
||||||
modernc.org/ccgo/v3 v3.16.13 // indirect
|
modernc.org/ccgo/v3 v3.16.13 // indirect
|
||||||
|
|||||||
14
go.sum
14
go.sum
@@ -23,8 +23,6 @@ github.com/go-chi/httprate v0.15.0 h1:j54xcWV9KGmPf/X4H32/aTH+wBlrvxL7P+SdnRqxh5
|
|||||||
github.com/go-chi/httprate v0.15.0/go.mod h1:rzGHhVrsBn3IMLYDOZQsSU4fJNWcjui4fWKJcCId1R4=
|
github.com/go-chi/httprate v0.15.0/go.mod h1:rzGHhVrsBn3IMLYDOZQsSU4fJNWcjui4fWKJcCId1R4=
|
||||||
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
||||||
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||||
github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw=
|
|
||||||
github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0=
|
|
||||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||||
@@ -83,11 +81,11 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj
|
|||||||
github.com/slok/go-http-metrics v0.11.0 h1:ABJUpekCZSkQT1wQrFvS4kGbhea/w6ndFJaWJeh3zL0=
|
github.com/slok/go-http-metrics v0.11.0 h1:ABJUpekCZSkQT1wQrFvS4kGbhea/w6ndFJaWJeh3zL0=
|
||||||
github.com/slok/go-http-metrics v0.11.0/go.mod h1:ZGKeYG1ET6TEJpQx18BqAJAvxw9jBAZXCHU7bWQqqAc=
|
github.com/slok/go-http-metrics v0.11.0/go.mod h1:ZGKeYG1ET6TEJpQx18BqAJAvxw9jBAZXCHU7bWQqqAc=
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
github.com/stretchr/objx v0.5.1 h1:4VhoImhV/Bm0ToFkXFi8hXNXwpDRZ/ynw3amt82mzq0=
|
||||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
github.com/stretchr/objx v0.5.1/go.mod h1:/iHQpkQwBD6DLUmQ4pE+s1TXdob1mORJ4/UFdrifcy0=
|
||||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||||
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||||
@@ -111,8 +109,8 @@ golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
|||||||
golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
|
golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
|
||||||
golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
|
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
|
||||||
golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||||
golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
|
golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
|
||||||
golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
|
golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
|
||||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
|
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
|
||||||
|
|||||||
@@ -1,47 +0,0 @@
|
|||||||
// Package banner renders the operator-facing blocks that carry a
|
|
||||||
// plaintext credential.
|
|
||||||
//
|
|
||||||
// A generated password printed as one more structured log line is lost:
|
|
||||||
// a boot writes roughly 45 fx PROVIDE/RUN/HOOK lines around it, and
|
|
||||||
// under `docker run -d` it is one line in a log subject to rotation. A
|
|
||||||
// credential that is shown exactly once has to be findable by eye when
|
|
||||||
// an operator scrolls back, so it is written as a ruled block rather
|
|
||||||
// than as a log record.
|
|
||||||
//
|
|
||||||
// It is deliberately not a log line: it goes straight to the writer the
|
|
||||||
// caller names — standard output for both the first-boot account and
|
|
||||||
// the `resetpw` subcommand — so it is neither levelled, filtered, nor
|
|
||||||
// rendered as JSON by whichever handler internal/logger installed.
|
|
||||||
package banner
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ruleWidth is the length of the horizontal rules, chosen to fit an
|
|
||||||
// 80-column terminal without wrapping.
|
|
||||||
const ruleWidth = 72
|
|
||||||
|
|
||||||
// Credentials writes a ruled block naming an account and its plaintext
|
|
||||||
// password. headline says which event produced it, and note says what
|
|
||||||
// the operator must do about it; both are written verbatim, so a
|
|
||||||
// multi-line note must already be wrapped.
|
|
||||||
func Credentials(
|
|
||||||
w io.Writer,
|
|
||||||
headline, username, password, note string,
|
|
||||||
) error {
|
|
||||||
rule := strings.Repeat("=", ruleWidth)
|
|
||||||
|
|
||||||
_, err := fmt.Fprintf(
|
|
||||||
w,
|
|
||||||
"\n%s\n%s\n\n username: %s\n password: %s\n\n%s\n%s\n\n",
|
|
||||||
rule, headline, username, password, note, rule,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("writing credentials banner: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,59 +0,0 @@
|
|||||||
package banner_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/banner"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestCredentials_IsFindableByEye pins the properties that make the
|
|
||||||
// block worth having: rules above and below it, the two fields on
|
|
||||||
// their own lines, and blank lines separating it from whatever the
|
|
||||||
// surrounding log wrote.
|
|
||||||
func TestCredentials_IsFindableByEye(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
require.NoError(t, banner.Credentials(
|
|
||||||
&out, "HEADLINE", "admin", "s3cret", "NOTE",
|
|
||||||
))
|
|
||||||
|
|
||||||
got := out.String()
|
|
||||||
lines := strings.Split(strings.Trim(got, "\n"), "\n")
|
|
||||||
|
|
||||||
require.GreaterOrEqual(t, len(lines), 3)
|
|
||||||
assert.Equal(t, lines[0], lines[len(lines)-1], "rules must match")
|
|
||||||
assert.Greater(
|
|
||||||
t, len(lines[0]), 40, "the rule must be visible at a glance",
|
|
||||||
)
|
|
||||||
assert.Equal(t, strings.Repeat("=", len(lines[0])), lines[0])
|
|
||||||
|
|
||||||
assert.Contains(t, got, "\n username: admin\n")
|
|
||||||
assert.Contains(t, got, "\n password: s3cret\n")
|
|
||||||
assert.Contains(t, got, "HEADLINE")
|
|
||||||
assert.Contains(t, got, "NOTE")
|
|
||||||
assert.True(t, strings.HasPrefix(got, "\n"))
|
|
||||||
}
|
|
||||||
|
|
||||||
// failingWriter reports the write error a banner must not swallow: it
|
|
||||||
// is the one copy of a password that will never be shown again.
|
|
||||||
type failingWriter struct{}
|
|
||||||
|
|
||||||
func (failingWriter) Write([]byte) (int, error) {
|
|
||||||
return 0, assert.AnError
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCredentials_ReportsAWriteFailure(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
err := banner.Credentials(
|
|
||||||
failingWriter{}, "HEADLINE", "admin", "s3cret", "NOTE",
|
|
||||||
)
|
|
||||||
|
|
||||||
require.ErrorIs(t, err, assert.AnError)
|
|
||||||
}
|
|
||||||
@@ -1,387 +0,0 @@
|
|||||||
package ciscript_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"maps"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gopkg.in/yaml.v3"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// supersededDesc is the description script/ci-mark-superseded
|
|
||||||
// writes, and the one an earlier revision of it wrote alongside a
|
|
||||||
// `skipped` state.
|
|
||||||
supersededDesc = "Superseded by a newer commit; never tested"
|
|
||||||
|
|
||||||
// liveContext is the commit-status context Gitea uses for this
|
|
||||||
// repository's runs, as seen in its API. The script derives it from
|
|
||||||
// the workflow and job names rather than hardcoding it; the
|
|
||||||
// derivation is checked against this value below.
|
|
||||||
liveContext = "check / check (push)"
|
|
||||||
|
|
||||||
scriptPath = "../../script/ci-mark-superseded"
|
|
||||||
workflow = "../../.gitea/workflows/check.yml"
|
|
||||||
|
|
||||||
// failure is the only state that neither folds into a combined
|
|
||||||
// `success` (as `skipped` does) nor blocks the commit forever (as
|
|
||||||
// `pending` does).
|
|
||||||
failure = "failure"
|
|
||||||
)
|
|
||||||
|
|
||||||
// repo is a throwaway git history: parent is the commit a run would be
|
|
||||||
// cancelled on, head the commit that superseded it.
|
|
||||||
type repo struct {
|
|
||||||
dir string
|
|
||||||
head string
|
|
||||||
parent string
|
|
||||||
}
|
|
||||||
|
|
||||||
// scriptEnv is the run identity the Gitea runner exports and the script
|
|
||||||
// builds its context string from.
|
|
||||||
type scriptEnv struct {
|
|
||||||
workflow string
|
|
||||||
job string
|
|
||||||
event string
|
|
||||||
}
|
|
||||||
|
|
||||||
func defaultEnv() scriptEnv {
|
|
||||||
return scriptEnv{workflow: "check", job: "check", event: "push"}
|
|
||||||
}
|
|
||||||
|
|
||||||
func cancelled() commitStatus {
|
|
||||||
return commitStatus{
|
|
||||||
Context: liveContext,
|
|
||||||
Status: failure,
|
|
||||||
Description: "Has been cancelled",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func running() commitStatus {
|
|
||||||
return commitStatus{
|
|
||||||
Context: liveContext,
|
|
||||||
Status: "pending",
|
|
||||||
Description: "Has started running",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMarkSuperseded(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cases := map[string]struct {
|
|
||||||
parent commitStatus
|
|
||||||
wantMark bool
|
|
||||||
}{
|
|
||||||
"a cancelled run is marked": {
|
|
||||||
parent: cancelled(),
|
|
||||||
wantMark: true,
|
|
||||||
},
|
|
||||||
"a laundered skipped status is marked": {
|
|
||||||
parent: commitStatus{
|
|
||||||
Context: liveContext,
|
|
||||||
Status: "skipped",
|
|
||||||
Description: supersededDesc,
|
|
||||||
},
|
|
||||||
wantMark: true,
|
|
||||||
},
|
|
||||||
"a genuine failure is left alone": {
|
|
||||||
parent: commitStatus{
|
|
||||||
Context: liveContext,
|
|
||||||
Status: failure,
|
|
||||||
Description: "Failing after 3m1s",
|
|
||||||
},
|
|
||||||
wantMark: false,
|
|
||||||
},
|
|
||||||
"a passing run is left alone": {
|
|
||||||
parent: commitStatus{
|
|
||||||
Context: liveContext,
|
|
||||||
Status: "success",
|
|
||||||
Description: "Successful in 2m52s",
|
|
||||||
},
|
|
||||||
wantMark: false,
|
|
||||||
},
|
|
||||||
"another context is left alone": {
|
|
||||||
parent: commitStatus{
|
|
||||||
Context: "other / other (push)",
|
|
||||||
Status: failure,
|
|
||||||
Description: "Has been cancelled",
|
|
||||||
},
|
|
||||||
wantMark: false,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, tc := range cases {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
requireTools(t)
|
|
||||||
|
|
||||||
history := newRepo(t)
|
|
||||||
fake, api := newFakeGitea(t)
|
|
||||||
fake.setStatus(history.head, running())
|
|
||||||
fake.setStatus(history.parent, tc.parent)
|
|
||||||
|
|
||||||
out, err := runScript(t, history, api, defaultEnv())
|
|
||||||
require.NoError(t, err, out)
|
|
||||||
|
|
||||||
posted := fake.postedFor(history.parent)
|
|
||||||
if !tc.wantMark {
|
|
||||||
require.Empty(t, posted)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
require.Equal(t, []postedStatus{{
|
|
||||||
Context: liveContext,
|
|
||||||
// Not `skipped`: Gitea's combined status folds
|
|
||||||
// that into `success`, which is what made a
|
|
||||||
// never-tested commit read green.
|
|
||||||
State: failure,
|
|
||||||
Description: supersededDesc,
|
|
||||||
}}, posted)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A second run must not rewrite what the first one wrote, or every
|
|
||||||
// later push would post a duplicate status.
|
|
||||||
func TestMarkSupersededIsIdempotent(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
requireTools(t)
|
|
||||||
|
|
||||||
history := newRepo(t)
|
|
||||||
fake, api := newFakeGitea(t)
|
|
||||||
fake.setStatus(history.head, running())
|
|
||||||
fake.setStatus(history.parent, cancelled())
|
|
||||||
|
|
||||||
for range 2 {
|
|
||||||
out, err := runScript(t, history, api, defaultEnv())
|
|
||||||
require.NoError(t, err, out)
|
|
||||||
}
|
|
||||||
|
|
||||||
require.Len(t, fake.postedFor(history.parent), 1)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Renaming the workflow or the job changes the context string Gitea
|
|
||||||
// uses. The script must say so instead of quietly matching nothing.
|
|
||||||
func TestMarkSupersededRejectsAnUnknownContext(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
requireTools(t)
|
|
||||||
|
|
||||||
history := newRepo(t)
|
|
||||||
fake, api := newFakeGitea(t)
|
|
||||||
fake.setStatus(history.head, running())
|
|
||||||
fake.setStatus(history.parent, cancelled())
|
|
||||||
|
|
||||||
env := defaultEnv()
|
|
||||||
env.job = "renamed"
|
|
||||||
|
|
||||||
out, err := runScript(t, history, api, env)
|
|
||||||
require.Error(t, err)
|
|
||||||
require.Contains(t, out, "renamed")
|
|
||||||
require.Contains(t, out, liveContext)
|
|
||||||
require.Empty(t, fake.postedFor(history.parent))
|
|
||||||
}
|
|
||||||
|
|
||||||
// ANCESTOR_LIMIT is a documented knob. A value that is set but unusable
|
|
||||||
// must abort: handing it to git and discarding the exit status left the
|
|
||||||
// walk empty and the step green, marking nothing.
|
|
||||||
func TestMarkSupersededRejectsAnUnparseableAncestorLimit(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
requireTools(t)
|
|
||||||
|
|
||||||
history := newRepo(t)
|
|
||||||
fake, api := newFakeGitea(t)
|
|
||||||
fake.setStatus(history.head, running())
|
|
||||||
fake.setStatus(history.parent, cancelled())
|
|
||||||
|
|
||||||
out, err := runScript(
|
|
||||||
t, history, api, defaultEnv(), "ANCESTOR_LIMIT=twenty",
|
|
||||||
)
|
|
||||||
require.Error(t, err)
|
|
||||||
require.Contains(t, out, "ANCESTOR_LIMIT")
|
|
||||||
require.Contains(t, out, "twenty")
|
|
||||||
require.Empty(t, fake.postedFor(history.parent))
|
|
||||||
}
|
|
||||||
|
|
||||||
// A status read that fails is not the same as a commit with nothing to
|
|
||||||
// do. Losing curl's exit status through a pipe made the two identical
|
|
||||||
// and left a laundered commit laundered with no signal.
|
|
||||||
func TestMarkSupersededFailsOnAnUnreadableAncestorStatus(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
requireTools(t)
|
|
||||||
|
|
||||||
history := newRepo(t)
|
|
||||||
fake, api := newFakeGitea(t)
|
|
||||||
fake.setStatus(history.head, running())
|
|
||||||
fake.setStatus(history.parent, cancelled())
|
|
||||||
fake.failStatusRead(history.parent)
|
|
||||||
|
|
||||||
out, err := runScript(t, history, api, defaultEnv())
|
|
||||||
require.Error(t, err)
|
|
||||||
require.Contains(t, out, history.parent)
|
|
||||||
require.Contains(t, out, "cannot read commit statuses")
|
|
||||||
require.Empty(t, fake.postedFor(history.parent))
|
|
||||||
}
|
|
||||||
|
|
||||||
// A shallow clone cannot resolve the parent, so it is indistinguishable
|
|
||||||
// from a root commit to rev-parse and the walk would exit 0 having
|
|
||||||
// marked nothing. It must abort instead: dropping `fetch-depth: 0` from
|
|
||||||
// the checkout step is one edit, and a silent no-op there restores the
|
|
||||||
// false-green bug this script exists to prevent.
|
|
||||||
func TestMarkSupersededRejectsAShallowRepository(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
requireTools(t)
|
|
||||||
|
|
||||||
history := shallowClone(t, newRepo(t))
|
|
||||||
fake, api := newFakeGitea(t)
|
|
||||||
fake.setStatus(history.head, running())
|
|
||||||
fake.setStatus(history.parent, cancelled())
|
|
||||||
|
|
||||||
out, err := runScript(t, history, api, defaultEnv())
|
|
||||||
require.Error(t, err)
|
|
||||||
require.Contains(t, out, "shallow repository")
|
|
||||||
require.Empty(t, fake.postedFor(history.parent))
|
|
||||||
require.Empty(t, fake.postedFor(history.head))
|
|
||||||
}
|
|
||||||
|
|
||||||
// shallowClone returns the same history as a depth-1 clone. The `file://`
|
|
||||||
// URL is required: git ignores --depth for a plain local path.
|
|
||||||
func shallowClone(t *testing.T, history repo) repo {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
|
|
||||||
//nolint:gosec // fixed argv, arguments are test-local paths
|
|
||||||
cmd := exec.CommandContext(t.Context(), "git", "clone", "-q",
|
|
||||||
"--depth=1", "file://"+history.dir, dir)
|
|
||||||
|
|
||||||
out, err := cmd.CombinedOutput()
|
|
||||||
require.NoError(t, err, string(out))
|
|
||||||
|
|
||||||
return repo{dir: dir, head: history.head, parent: history.parent}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The derived context must equal the one Gitea actually uses, which is
|
|
||||||
// built from the same workflow and job names.
|
|
||||||
func TestDerivedContextMatchesGitea(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
requireTools(t)
|
|
||||||
|
|
||||||
name, job := workflowIdentity(t)
|
|
||||||
|
|
||||||
history := newRepo(t)
|
|
||||||
fake, api := newFakeGitea(t)
|
|
||||||
fake.setStatus(history.head, running())
|
|
||||||
fake.setStatus(history.parent, cancelled())
|
|
||||||
|
|
||||||
out, err := runScript(t, history, api, scriptEnv{
|
|
||||||
workflow: name,
|
|
||||||
job: job,
|
|
||||||
event: "push",
|
|
||||||
})
|
|
||||||
require.NoError(t, err, out)
|
|
||||||
|
|
||||||
posted := fake.postedFor(history.parent)
|
|
||||||
require.Len(t, posted, 1)
|
|
||||||
require.Equal(t, liveContext, posted[0].Context)
|
|
||||||
}
|
|
||||||
|
|
||||||
// workflowIdentity reads the workflow name and its single job id out of
|
|
||||||
// the checked-in workflow file.
|
|
||||||
func workflowIdentity(t *testing.T) (string, string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
raw, err := os.ReadFile(workflow)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var parsed struct {
|
|
||||||
Name string `yaml:"name"`
|
|
||||||
Jobs map[string]any `yaml:"jobs"`
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, yaml.Unmarshal(raw, &parsed))
|
|
||||||
|
|
||||||
jobs := slices.Collect(maps.Keys(parsed.Jobs))
|
|
||||||
require.Len(t, jobs, 1)
|
|
||||||
|
|
||||||
return parsed.Name, jobs[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
func runScript(
|
|
||||||
t *testing.T, history repo, api string, env scriptEnv,
|
|
||||||
extra ...string,
|
|
||||||
) (string, error) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
script, err := filepath.Abs(scriptPath)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
//nolint:gosec // fixed argv, repo-local script under test
|
|
||||||
cmd := exec.CommandContext(t.Context(), "sh", script)
|
|
||||||
cmd.Dir = history.dir
|
|
||||||
cmd.Env = append(os.Environ(),
|
|
||||||
"GITHUB_API_URL="+api,
|
|
||||||
"GITHUB_REPOSITORY=sneak/webhooker",
|
|
||||||
"GITHUB_SHA="+history.head,
|
|
||||||
"GITHUB_WORKFLOW="+env.workflow,
|
|
||||||
"GITHUB_JOB="+env.job,
|
|
||||||
"GITHUB_EVENT_NAME="+env.event,
|
|
||||||
"GITEA_TOKEN=test-token",
|
|
||||||
)
|
|
||||||
cmd.Env = append(cmd.Env, extra...)
|
|
||||||
|
|
||||||
out, err := cmd.CombinedOutput()
|
|
||||||
|
|
||||||
return string(out), err
|
|
||||||
}
|
|
||||||
|
|
||||||
func newRepo(t *testing.T) repo {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
|
|
||||||
git := func(args ...string) string {
|
|
||||||
//nolint:gosec // fixed argv, arguments are test constants
|
|
||||||
cmd := exec.CommandContext(t.Context(), "git", args...)
|
|
||||||
cmd.Dir = dir
|
|
||||||
|
|
||||||
out, err := cmd.CombinedOutput()
|
|
||||||
require.NoError(t, err, string(out))
|
|
||||||
|
|
||||||
return strings.TrimSpace(string(out))
|
|
||||||
}
|
|
||||||
|
|
||||||
commit := func(message string) string {
|
|
||||||
git(
|
|
||||||
"-c", "user.email=ci@example.invalid",
|
|
||||||
"-c", "user.name=ci",
|
|
||||||
"-c", "commit.gpgsign=false",
|
|
||||||
"commit", "-q", "--allow-empty", "-m", message,
|
|
||||||
)
|
|
||||||
|
|
||||||
return git("rev-parse", "HEAD")
|
|
||||||
}
|
|
||||||
|
|
||||||
git("init", "-q", "-b", "main")
|
|
||||||
|
|
||||||
parent := commit("parent")
|
|
||||||
head := commit("head")
|
|
||||||
|
|
||||||
return repo{dir: dir, head: head, parent: parent}
|
|
||||||
}
|
|
||||||
|
|
||||||
func requireTools(t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for _, tool := range []string{"sh", "git", "curl", "jq"} {
|
|
||||||
_, err := exec.LookPath(tool)
|
|
||||||
if err != nil {
|
|
||||||
t.Skipf("%s is not installed: %v", tool, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,10 +0,0 @@
|
|||||||
// Package ciscript holds the tests for the repository's CI shell
|
|
||||||
// scripts in script/. It carries no runtime code: the scripts run on
|
|
||||||
// the CI runner, not inside the binary, but their behaviour still has
|
|
||||||
// to be verified by the test suite.
|
|
||||||
//
|
|
||||||
// The scripts under test are outside the Go build graph, so `go test`'s
|
|
||||||
// result cache serves a stale PASS when only a script changed: run the
|
|
||||||
// container build, or GOFLAGS=-count=1, to trust a result here after
|
|
||||||
// editing script/.
|
|
||||||
package ciscript
|
|
||||||
@@ -1,162 +0,0 @@
|
|||||||
package ciscript_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// commitStatus is the part of an entry in Gitea's combined-status
|
|
||||||
// response that script/ci-mark-superseded reads.
|
|
||||||
type commitStatus struct {
|
|
||||||
Context string `json:"context"`
|
|
||||||
Status string `json:"status"`
|
|
||||||
Description string `json:"description"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// postedStatus is the part of a create-status request body the script
|
|
||||||
// writes.
|
|
||||||
type postedStatus struct {
|
|
||||||
Context string `json:"context"`
|
|
||||||
State string `json:"state"`
|
|
||||||
Description string `json:"description"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// fakeGitea serves the two endpoints the script talks to. Like Gitea,
|
|
||||||
// the newest status for a context replaces the previous one, so a
|
|
||||||
// second run of the script sees what the first one wrote.
|
|
||||||
type fakeGitea struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
statuses map[string][]commitStatus
|
|
||||||
posted map[string][]postedStatus
|
|
||||||
// failRead is a commit whose combined-status read answers HTTP
|
|
||||||
// 500, standing in for a status API that is down.
|
|
||||||
failRead string
|
|
||||||
}
|
|
||||||
|
|
||||||
// newFakeGitea returns the fake and the base URL to hand the script as
|
|
||||||
// GITHUB_API_URL.
|
|
||||||
func newFakeGitea(t *testing.T) (*fakeGitea, string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
fake := &fakeGitea{
|
|
||||||
mu: sync.Mutex{},
|
|
||||||
statuses: map[string][]commitStatus{},
|
|
||||||
posted: map[string][]postedStatus{},
|
|
||||||
failRead: "",
|
|
||||||
}
|
|
||||||
|
|
||||||
srv := httptest.NewServer(fake.routes())
|
|
||||||
t.Cleanup(srv.Close)
|
|
||||||
|
|
||||||
return fake, srv.URL
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *fakeGitea) routes() http.Handler {
|
|
||||||
mux := http.NewServeMux()
|
|
||||||
|
|
||||||
mux.HandleFunc(
|
|
||||||
"GET /repos/{owner}/{repo}/commits/{sha}/status",
|
|
||||||
f.handleCombined,
|
|
||||||
)
|
|
||||||
mux.HandleFunc(
|
|
||||||
"POST /repos/{owner}/{repo}/statuses/{sha}",
|
|
||||||
f.handleCreate,
|
|
||||||
)
|
|
||||||
|
|
||||||
return mux
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *fakeGitea) handleCombined(
|
|
||||||
w http.ResponseWriter, r *http.Request,
|
|
||||||
) {
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
|
|
||||||
sha := r.PathValue("sha")
|
|
||||||
if f.failRead != "" && f.failRead == sha {
|
|
||||||
http.Error(w, "boom", http.StatusInternalServerError)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
body := struct {
|
|
||||||
Statuses []commitStatus `json:"statuses"`
|
|
||||||
}{Statuses: f.statuses[sha]}
|
|
||||||
|
|
||||||
payload, err := json.Marshal(body)
|
|
||||||
if err != nil {
|
|
||||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
|
|
||||||
_, _ = w.Write(payload)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *fakeGitea) handleCreate(w http.ResponseWriter, r *http.Request) {
|
|
||||||
var got postedStatus
|
|
||||||
|
|
||||||
err := json.NewDecoder(r.Body).Decode(&got)
|
|
||||||
if err != nil {
|
|
||||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
sha := r.PathValue("sha")
|
|
||||||
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
|
|
||||||
f.posted[sha] = append(f.posted[sha], got)
|
|
||||||
f.replaceLocked(sha, commitStatus{
|
|
||||||
Context: got.Context,
|
|
||||||
Status: got.State,
|
|
||||||
Description: got.Description,
|
|
||||||
})
|
|
||||||
|
|
||||||
w.WriteHeader(http.StatusCreated)
|
|
||||||
}
|
|
||||||
|
|
||||||
// failStatusRead makes the combined-status read for one commit answer
|
|
||||||
// HTTP 500.
|
|
||||||
func (f *fakeGitea) failStatusRead(sha string) {
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
|
|
||||||
f.failRead = sha
|
|
||||||
}
|
|
||||||
|
|
||||||
// setStatus gives a commit its latest status for a context.
|
|
||||||
func (f *fakeGitea) setStatus(sha string, status commitStatus) {
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
|
|
||||||
f.replaceLocked(sha, status)
|
|
||||||
}
|
|
||||||
|
|
||||||
// postedFor returns the statuses the script created for a commit.
|
|
||||||
func (f *fakeGitea) postedFor(sha string) []postedStatus {
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
|
|
||||||
return append([]postedStatus(nil), f.posted[sha]...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// replaceLocked requires f.mu.
|
|
||||||
func (f *fakeGitea) replaceLocked(sha string, status commitStatus) {
|
|
||||||
for i, existing := range f.statuses[sha] {
|
|
||||||
if existing.Context == status.Context {
|
|
||||||
f.statuses[sha][i] = status
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
f.statuses[sha] = append(f.statuses[sha], status)
|
|
||||||
}
|
|
||||||
@@ -5,10 +5,8 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/netip"
|
|
||||||
"os"
|
"os"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
@@ -26,10 +24,6 @@ const (
|
|||||||
// EnvironmentProd represents production environment.
|
// EnvironmentProd represents production environment.
|
||||||
EnvironmentProd = "prod"
|
EnvironmentProd = "prod"
|
||||||
|
|
||||||
// DefaultDataDir is where all SQLite databases live when DATA_DIR
|
|
||||||
// is unset. The same default applies in every environment.
|
|
||||||
DefaultDataDir = "/var/lib/webhooker"
|
|
||||||
|
|
||||||
// defaultPort is the default HTTP listen port.
|
// defaultPort is the default HTTP listen port.
|
||||||
defaultPort = 8080
|
defaultPort = 8080
|
||||||
|
|
||||||
@@ -51,11 +45,6 @@ const (
|
|||||||
// maxPort is the highest valid TCP port number. The lower
|
// maxPort is the highest valid TCP port number. The lower
|
||||||
// bound (at least 1) is enforced by envPositiveInt.
|
// bound (at least 1) is enforced by envPositiveInt.
|
||||||
maxPort = 65535
|
maxPort = 65535
|
||||||
|
|
||||||
// mappedV4Offset is the number of leading bits an IPv4-mapped
|
|
||||||
// IPv6 prefix spends on the ::ffff:0:0/96 wrapper, so a /104
|
|
||||||
// covers the same addresses as an IPv4 /8.
|
|
||||||
mappedV4Offset = 96
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrInvalidEnvironment is returned when WEBHOOKER_ENVIRONMENT
|
// ErrInvalidEnvironment is returned when WEBHOOKER_ENVIRONMENT
|
||||||
@@ -70,21 +59,6 @@ var ErrNonPositiveValue = errors.New("value must be positive")
|
|||||||
// TCP port number is set above the valid port range.
|
// TCP port number is set above the valid port range.
|
||||||
var ErrInvalidPort = errors.New("invalid port")
|
var ErrInvalidPort = errors.New("invalid port")
|
||||||
|
|
||||||
// ErrInvalidCIDR is returned when an environment variable holding a
|
|
||||||
// list of CIDR blocks contains an entry that is neither a CIDR block
|
|
||||||
// nor a bare IP address.
|
|
||||||
var ErrInvalidCIDR = errors.New("invalid CIDR")
|
|
||||||
|
|
||||||
// ErrIncompleteMetricsAuth is returned when exactly one of
|
|
||||||
// METRICS_USERNAME and METRICS_PASSWORD carries a value. Neither
|
|
||||||
// fallback is acceptable: serving /metrics on the username alone
|
|
||||||
// publishes an endpoint whose password is the empty string, and
|
|
||||||
// silently leaving it unmounted withholds an endpoint the operator
|
|
||||||
// asked for. Half-set is a configuration error, so startup fails.
|
|
||||||
var ErrIncompleteMetricsAuth = errors.New(
|
|
||||||
"incomplete metrics credentials",
|
|
||||||
)
|
|
||||||
|
|
||||||
//nolint:revive // ConfigParams is a standard fx naming convention.
|
//nolint:revive // ConfigParams is a standard fx naming convention.
|
||||||
type ConfigParams struct {
|
type ConfigParams struct {
|
||||||
fx.In
|
fx.In
|
||||||
@@ -106,7 +80,6 @@ type Config struct {
|
|||||||
SentryDSN string
|
SentryDSN string
|
||||||
|
|
||||||
// RetentionSweepInterval is how often the retention reaper runs.
|
// RetentionSweepInterval is how often the retention reaper runs.
|
||||||
// Always positive: it becomes a time.NewTicker period.
|
|
||||||
RetentionSweepInterval time.Duration
|
RetentionSweepInterval time.Duration
|
||||||
|
|
||||||
// SessionIdleTimeout is the sliding inactivity window after
|
// SessionIdleTimeout is the sliding inactivity window after
|
||||||
@@ -117,33 +90,6 @@ type Config struct {
|
|||||||
// client IP may send to a single webhook receiver entrypoint.
|
// client IP may send to a single webhook receiver entrypoint.
|
||||||
ReceiverRateLimit int
|
ReceiverRateLimit int
|
||||||
|
|
||||||
// TrustedProxies is the set of networks whose members are
|
|
||||||
// allowed to speak for the client with X-Forwarded-For, the
|
|
||||||
// only forwarded header read. It is empty unless
|
|
||||||
// TRUSTED_PROXIES is set, and empty means no peer is
|
|
||||||
// trusted: forwarded headers are then ignored entirely and
|
|
||||||
// clients are identified by the connection's own address.
|
|
||||||
// Members can choose their own rate-limit key, so this must
|
|
||||||
// name proxy hosts only, never a block that also covers
|
|
||||||
// clients.
|
|
||||||
TrustedProxies []netip.Prefix
|
|
||||||
|
|
||||||
// AllowedEgressCIDRs is the set of networks a delivery target
|
|
||||||
// may reach even though the SSRF guard's default blocklist
|
|
||||||
// covers them. It is empty unless ALLOWED_EGRESS_CIDRS is set,
|
|
||||||
// and empty means every private/reserved range stays refused.
|
|
||||||
//
|
|
||||||
// This only ever adds destinations to what the guard would
|
|
||||||
// otherwise refuse. The guard itself is always on: there is no
|
|
||||||
// setting that disables SSRF protection, and delivery's
|
|
||||||
// alwaysBlockedNetworks stays blocked no matter what is listed
|
|
||||||
// here. That set is link-local plus the cloud metadata
|
|
||||||
// endpoints outside it that disclose credentials or user data
|
|
||||||
// at a provider-fixed address; it is not exhaustive of every
|
|
||||||
// cloud's metadata address. See alwaysBlockedNetworks for the
|
|
||||||
// authoritative list and the criterion it is built from.
|
|
||||||
AllowedEgressCIDRs []netip.Prefix
|
|
||||||
|
|
||||||
params *ConfigParams
|
params *ConfigParams
|
||||||
log *slog.Logger
|
log *slog.Logger
|
||||||
}
|
}
|
||||||
@@ -158,40 +104,12 @@ func (c *Config) IsProd() bool {
|
|||||||
return c.Environment == EnvironmentProd
|
return c.Environment == EnvironmentProd
|
||||||
}
|
}
|
||||||
|
|
||||||
// MetricsAuthEnabled reports whether /metrics is served behind basic
|
|
||||||
// auth. It is the only answer to that question in the codebase: the
|
|
||||||
// route mount, the Prometheus recording middleware and the startup
|
|
||||||
// log's hasMetricsAuth field all read this one method, so the log
|
|
||||||
// cannot report auth as off while the route is mounted.
|
|
||||||
//
|
|
||||||
// It requires both credentials rather than the username alone.
|
|
||||||
// loadFromEnv already rejects a half-set pair, but a Config built in
|
|
||||||
// code bypasses that, and the failure mode this guards is an endpoint
|
|
||||||
// mounted with a credential map whose only password is the empty
|
|
||||||
// string.
|
|
||||||
func (c *Config) MetricsAuthEnabled() bool {
|
|
||||||
return c.MetricsUsername != "" && c.MetricsPassword != ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// envString returns the value of the named environment variable,
|
// envString returns the value of the named environment variable,
|
||||||
// or an empty string if not set.
|
// or an empty string if not set.
|
||||||
func envString(key string) string {
|
func envString(key string) string {
|
||||||
return os.Getenv(key)
|
return os.Getenv(key)
|
||||||
}
|
}
|
||||||
|
|
||||||
// DataDir resolves DATA_DIR, applying DefaultDataDir when it is unset
|
|
||||||
// or empty. It is exported so that entry points which must act on the
|
|
||||||
// data directory before the fx graph exists — taking the exclusive
|
|
||||||
// directory lock, above all — resolve it exactly as Config does.
|
|
||||||
func DataDir() string {
|
|
||||||
dir := envString("DATA_DIR")
|
|
||||||
if dir == "" {
|
|
||||||
return DefaultDataDir
|
|
||||||
}
|
|
||||||
|
|
||||||
return dir
|
|
||||||
}
|
|
||||||
|
|
||||||
// envBool returns the value of the named environment variable
|
// envBool returns the value of the named environment variable
|
||||||
// parsed as a boolean. Returns defaultValue if not set. If the
|
// parsed as a boolean. Returns defaultValue if not set. If the
|
||||||
// variable is set but cannot be parsed, it returns a wrapped error
|
// variable is set but cannot be parsed, it returns a wrapped error
|
||||||
@@ -294,123 +212,6 @@ func envDuration(
|
|||||||
return d, nil
|
return d, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// envPositiveDuration returns the value of the named environment
|
|
||||||
// variable parsed as a Go duration that must be greater than zero.
|
|
||||||
// Returns defaultValue if not set. A set value that is unparseable or
|
|
||||||
// non-positive is a hard error naming the key and the bad value.
|
|
||||||
//
|
|
||||||
// This is for durations that reach time.NewTicker, which panics on a
|
|
||||||
// non-positive period, in a goroutine started after startup has
|
|
||||||
// already reported success. It is deliberately not used for durations
|
|
||||||
// where non-positive means "disabled" (SESSION_IDLE_TIMEOUT).
|
|
||||||
func envPositiveDuration(
|
|
||||||
key string,
|
|
||||||
defaultValue time.Duration,
|
|
||||||
) (time.Duration, error) {
|
|
||||||
d, err := envDuration(key, defaultValue)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if d <= 0 {
|
|
||||||
return 0, fmt.Errorf(
|
|
||||||
"%w: %s must be greater than zero, got %s",
|
|
||||||
ErrNonPositiveValue, key, d,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return d, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseCIDR parses one trusted-proxy list entry, which may be a
|
|
||||||
// CIDR block ("10.0.0.0/8") or a bare address ("10.0.0.1", treated
|
|
||||||
// as a single-host block).
|
|
||||||
//
|
|
||||||
// Both forms are unmapped, because peer addresses are unmapped
|
|
||||||
// before they are matched against the list: an IPv4-mapped prefix
|
|
||||||
// left in that form would silently never match.
|
|
||||||
func parseCIDR(entry string) (netip.Prefix, error) {
|
|
||||||
if strings.Contains(entry, "/") {
|
|
||||||
prefix, err := netip.ParsePrefix(entry)
|
|
||||||
if err != nil {
|
|
||||||
return netip.Prefix{}, err //nolint:wrapcheck // wrapped by caller
|
|
||||||
}
|
|
||||||
|
|
||||||
if addr := prefix.Addr(); addr.Is4In6() &&
|
|
||||||
prefix.Bits() >= mappedV4Offset {
|
|
||||||
prefix = netip.PrefixFrom(
|
|
||||||
addr.Unmap(), prefix.Bits()-mappedV4Offset,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return prefix.Masked(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
addr, err := netip.ParseAddr(entry)
|
|
||||||
if err != nil {
|
|
||||||
return netip.Prefix{}, err //nolint:wrapcheck // wrapped by caller
|
|
||||||
}
|
|
||||||
|
|
||||||
return netip.PrefixFrom(addr.Unmap(), addr.Unmap().BitLen()), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// envPrefixList returns the value of the named environment variable
|
|
||||||
// parsed as a comma-separated list of CIDR blocks (bare addresses
|
|
||||||
// allowed). An unset, empty, or blank value yields an empty list. A
|
|
||||||
// set value containing an unparseable entry is a hard error naming
|
|
||||||
// the key and the bad entry, so startup fails loudly rather than
|
|
||||||
// silently running with a list the operator did not intend.
|
|
||||||
func envPrefixList(key string) ([]netip.Prefix, error) {
|
|
||||||
v := strings.TrimSpace(os.Getenv(key))
|
|
||||||
if v == "" {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var prefixes []netip.Prefix
|
|
||||||
|
|
||||||
for entry := range strings.SplitSeq(v, ",") {
|
|
||||||
entry = strings.TrimSpace(entry)
|
|
||||||
if entry == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
prefix, err := parseCIDR(entry)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"%w: %s: %q: %w", ErrInvalidCIDR, key, entry, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
prefixes = append(prefixes, prefix)
|
|
||||||
}
|
|
||||||
|
|
||||||
return prefixes, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolveMetricsAuth reads the /metrics basic-auth credentials and
|
|
||||||
// rejects a half-set pair, naming both variables either way. The
|
|
||||||
// error carries neither value: the password is a secret.
|
|
||||||
func resolveMetricsAuth() (string, string, error) {
|
|
||||||
username := envString("METRICS_USERNAME")
|
|
||||||
password := envString("METRICS_PASSWORD")
|
|
||||||
|
|
||||||
if (username == "") == (password == "") {
|
|
||||||
return username, password, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
set, empty := "METRICS_USERNAME", "METRICS_PASSWORD"
|
|
||||||
if username == "" {
|
|
||||||
set, empty = empty, set
|
|
||||||
}
|
|
||||||
|
|
||||||
return "", "", fmt.Errorf(
|
|
||||||
"%w: %s is set but %s is empty; METRICS_USERNAME and "+
|
|
||||||
"METRICS_PASSWORD must both be set to serve /metrics, "+
|
|
||||||
"or both be empty to leave it unmounted",
|
|
||||||
ErrIncompleteMetricsAuth, set, empty,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolveEnvironment reads WEBHOOKER_ENVIRONMENT, defaulting to
|
// resolveEnvironment reads WEBHOOKER_ENVIRONMENT, defaulting to
|
||||||
// dev, and rejects unrecognised values.
|
// dev, and rejects unrecognised values.
|
||||||
func resolveEnvironment() (string, error) {
|
func resolveEnvironment() (string, error) {
|
||||||
@@ -457,7 +258,7 @@ func loadFromEnv() (*Config, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
retentionSweepInterval, err := envPositiveDuration(
|
retentionSweepInterval, err := envDuration(
|
||||||
"RETENTION_SWEEP_INTERVAL",
|
"RETENTION_SWEEP_INTERVAL",
|
||||||
defaultRetentionSweepInterval,
|
defaultRetentionSweepInterval,
|
||||||
)
|
)
|
||||||
@@ -465,8 +266,6 @@ func loadFromEnv() (*Config, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Non-positive is "disabled" here, not invalid, so this stays on
|
|
||||||
// envDuration.
|
|
||||||
sessionIdleTimeout, err := envDuration(
|
sessionIdleTimeout, err := envDuration(
|
||||||
"SESSION_IDLE_TIMEOUT",
|
"SESSION_IDLE_TIMEOUT",
|
||||||
defaultSessionIdleTimeout,
|
defaultSessionIdleTimeout,
|
||||||
@@ -483,123 +282,21 @@ func loadFromEnv() (*Config, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
trustedProxies, err := envPrefixList("TRUSTED_PROXIES")
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
allowedEgressCIDRs, err := envPrefixList("ALLOWED_EGRESS_CIDRS")
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
metricsUsername, metricsPassword, err := resolveMetricsAuth()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return &Config{
|
return &Config{
|
||||||
DataDir: DataDir(),
|
DataDir: envString("DATA_DIR"),
|
||||||
Debug: debug,
|
Debug: debug,
|
||||||
MaintenanceMode: maintenanceMode,
|
MaintenanceMode: maintenanceMode,
|
||||||
Environment: environment,
|
Environment: environment,
|
||||||
MetricsUsername: metricsUsername,
|
MetricsUsername: envString("METRICS_USERNAME"),
|
||||||
MetricsPassword: metricsPassword,
|
MetricsPassword: envString("METRICS_PASSWORD"),
|
||||||
Port: port,
|
Port: port,
|
||||||
SentryDSN: envString("SENTRY_DSN"),
|
SentryDSN: envString("SENTRY_DSN"),
|
||||||
RetentionSweepInterval: retentionSweepInterval,
|
RetentionSweepInterval: retentionSweepInterval,
|
||||||
SessionIdleTimeout: sessionIdleTimeout,
|
SessionIdleTimeout: sessionIdleTimeout,
|
||||||
ReceiverRateLimit: receiverRateLimit,
|
ReceiverRateLimit: receiverRateLimit,
|
||||||
TrustedProxies: trustedProxies,
|
|
||||||
AllowedEgressCIDRs: allowedEgressCIDRs,
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// PrefixStrings renders a prefix list as its CIDR strings, for
|
|
||||||
// logging a list an operator has to be able to read back.
|
|
||||||
func PrefixStrings(prefixes []netip.Prefix) []string {
|
|
||||||
out := make([]string, 0, len(prefixes))
|
|
||||||
|
|
||||||
for _, prefix := range prefixes {
|
|
||||||
out = append(out, prefix.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// warnEgressAllowlist logs the effective ALLOWED_EGRESS_CIDRS
|
|
||||||
// whenever it is non-empty.
|
|
||||||
//
|
|
||||||
// It prints the blocks themselves rather than a count, because
|
|
||||||
// this is the one setting that lets a delivery target reach the
|
|
||||||
// host's own network: an operator reading the startup log has to
|
|
||||||
// be able to see exactly which hole is open. Silence means the
|
|
||||||
// list is empty and the SSRF guard is refusing every
|
|
||||||
// private/reserved range, which is the default.
|
|
||||||
func (c *Config) warnEgressAllowlist(log *slog.Logger) {
|
|
||||||
if len(c.AllowedEgressCIDRs) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Warn(
|
|
||||||
"ALLOWED_EGRESS_CIDRS lets delivery targets reach these "+
|
|
||||||
"otherwise-blocked private/reserved networks. Anyone "+
|
|
||||||
"who can create a delivery target can now make this "+
|
|
||||||
"process issue requests into them, and read back the "+
|
|
||||||
"response. Link-local and the known cloud instance "+
|
|
||||||
"metadata endpoints outside it stay blocked "+
|
|
||||||
"regardless of what is listed here.",
|
|
||||||
"allowedEgressCIDRs",
|
|
||||||
strings.Join(PrefixStrings(c.AllowedEgressCIDRs), ","),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// warnSharedRateLimitBucket logs a startup warning whenever
|
|
||||||
// TRUSTED_PROXIES is empty, in any environment.
|
|
||||||
//
|
|
||||||
// With no trusted proxies every rate limiter keys on the connecting
|
|
||||||
// peer's address. Whether that is harmless or dangerous depends on
|
|
||||||
// what is in front of the process, which this code cannot observe:
|
|
||||||
// with nothing in front, the peer is the client and the limits are
|
|
||||||
// per-client as intended; behind a reverse proxy the peer is the proxy
|
|
||||||
// for every request, so all clients share one bucket per limiter.
|
|
||||||
//
|
|
||||||
// The login endpoint no longer spends budget on arrival — it verifies
|
|
||||||
// credentials first and charges only failures — so a shared bucket
|
|
||||||
// cannot deny the operator a correct password. What it does collapse
|
|
||||||
// is the failure counting: one client's wrong passwords throttle
|
|
||||||
// everyone else's wrong passwords, and the receiver's limits become
|
|
||||||
// service-wide ceilings.
|
|
||||||
//
|
|
||||||
// The warning is deliberately not gated on WEBHOOKER_ENVIRONMENT. That
|
|
||||||
// variable defaults to dev, so gating on it would silence the warning
|
|
||||||
// for exactly the operator who forgot to configure the deployment —
|
|
||||||
// the case it exists to catch.
|
|
||||||
//
|
|
||||||
// The default of trusting nobody is deliberate — trusting forwarded
|
|
||||||
// headers from arbitrary peers lets any client choose its own bucket —
|
|
||||||
// so this warns rather than failing startup or changing the key.
|
|
||||||
func (c *Config) warnSharedRateLimitBucket(log *slog.Logger) {
|
|
||||||
if len(c.TrustedProxies) > 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Warn(
|
|
||||||
"TRUSTED_PROXIES is empty: every rate limit keys on the "+
|
|
||||||
"connecting peer's address. With nothing proxying to "+
|
|
||||||
"this process that is the client itself and the limits "+
|
|
||||||
"are per-client as intended. Behind a reverse proxy the "+
|
|
||||||
"peer is the proxy on every request, so all clients "+
|
|
||||||
"share one bucket per limit: the receiver limits become "+
|
|
||||||
"service-wide ceilings, and one client's failed logins "+
|
|
||||||
"throttle every other client's failed logins — a "+
|
|
||||||
"correct password still gets in. If anything proxies to "+
|
|
||||||
"this process, set TRUSTED_PROXIES to its address.",
|
|
||||||
"environment", c.Environment,
|
|
||||||
"trustedProxies", len(c.TrustedProxies),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// New creates a Config by reading environment variables.
|
// New creates a Config by reading environment variables.
|
||||||
//
|
//
|
||||||
//nolint:revive // lc parameter is required by fx even if unused.
|
//nolint:revive // lc parameter is required by fx even if unused.
|
||||||
@@ -617,6 +314,14 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) {
|
|||||||
s.log = log
|
s.log = log
|
||||||
s.params = ¶ms
|
s.params = ¶ms
|
||||||
|
|
||||||
|
// Set default DataDir. All SQLite databases (main application
|
||||||
|
// DB and per-webhook event DBs) live here. The same default is
|
||||||
|
// used regardless of environment; override with DATA_DIR if
|
||||||
|
// needed.
|
||||||
|
if s.DataDir == "" {
|
||||||
|
s.DataDir = "/var/lib/webhooker"
|
||||||
|
}
|
||||||
|
|
||||||
if s.Debug {
|
if s.Debug {
|
||||||
params.Logger.EnableDebugLogging()
|
params.Logger.EnableDebugLogging()
|
||||||
}
|
}
|
||||||
@@ -629,19 +334,11 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) {
|
|||||||
"maintenanceMode", s.MaintenanceMode,
|
"maintenanceMode", s.MaintenanceMode,
|
||||||
"dataDir", s.DataDir,
|
"dataDir", s.DataDir,
|
||||||
"retentionSweepInterval", s.RetentionSweepInterval.String(),
|
"retentionSweepInterval", s.RetentionSweepInterval.String(),
|
||||||
// Logged because a perfectly valid non-positive value here
|
|
||||||
// disables idle expiry entirely, and that is worth showing
|
|
||||||
// back to the operator.
|
|
||||||
"sessionIdleTimeout", s.SessionIdleTimeout.String(),
|
|
||||||
"receiverRateLimit", s.ReceiverRateLimit,
|
"receiverRateLimit", s.ReceiverRateLimit,
|
||||||
"trustedProxies", len(s.TrustedProxies),
|
|
||||||
"allowedEgressCIDRs", len(s.AllowedEgressCIDRs),
|
|
||||||
"hasSentryDSN", s.SentryDSN != "",
|
"hasSentryDSN", s.SentryDSN != "",
|
||||||
"hasMetricsAuth", s.MetricsAuthEnabled(),
|
"hasMetricsAuth",
|
||||||
|
s.MetricsUsername != "" && s.MetricsPassword != "",
|
||||||
)
|
)
|
||||||
|
|
||||||
s.warnSharedRateLimitBucket(log)
|
|
||||||
s.warnEgressAllowlist(log)
|
|
||||||
|
|
||||||
return s, nil
|
return s, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,8 +1,6 @@
|
|||||||
package config_test
|
package config_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"log/slog"
|
|
||||||
"os"
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -22,16 +20,6 @@ const (
|
|||||||
caseUnsetUsesDefault = "unset uses default"
|
caseUnsetUsesDefault = "unset uses default"
|
||||||
caseValidValueParsed = "valid value is parsed"
|
caseValidValueParsed = "valid value is parsed"
|
||||||
caseUnparseableFails = "unparseable value fails startup"
|
caseUnparseableFails = "unparseable value fails startup"
|
||||||
|
|
||||||
// cidrPrivateV4 is the sample trusted-proxy block the
|
|
||||||
// TRUSTED_PROXIES cases are built from.
|
|
||||||
cidrPrivateV4 = "10.0.0.0/8"
|
|
||||||
|
|
||||||
// metricsAuthValue is the sample METRICS_PASSWORD the metrics
|
|
||||||
// credential cases are built from. It is asserted absent from
|
|
||||||
// the startup error, so it must not be a substring of either
|
|
||||||
// variable name that error prints.
|
|
||||||
metricsAuthValue = "s3cret"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestEnvironmentConfig(t *testing.T) {
|
func TestEnvironmentConfig(t *testing.T) {
|
||||||
@@ -147,10 +135,6 @@ func TestRetentionSweepInterval(t *testing.T) {
|
|||||||
set bool
|
set bool
|
||||||
value string
|
value string
|
||||||
expectError bool
|
expectError bool
|
||||||
// sentinel, when set, must be wrapped by the startup
|
|
||||||
// error; every error case must additionally name the
|
|
||||||
// variable in its message.
|
|
||||||
sentinel error
|
|
||||||
expected time.Duration
|
expected time.Duration
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
@@ -170,24 +154,6 @@ func TestRetentionSweepInterval(t *testing.T) {
|
|||||||
value: "not-a-duration",
|
value: "not-a-duration",
|
||||||
expectError: true,
|
expectError: true,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
// A non-positive period panics the ticker in the
|
|
||||||
// reaper and archive-sweeper goroutines, long after
|
|
||||||
// startup has reported success, so it has to fail
|
|
||||||
// here instead.
|
|
||||||
name: "zero fails startup",
|
|
||||||
set: true,
|
|
||||||
value: "0s",
|
|
||||||
expectError: true,
|
|
||||||
sentinel: config.ErrNonPositiveValue,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "negative fails startup",
|
|
||||||
set: true,
|
|
||||||
value: "-1h",
|
|
||||||
expectError: true,
|
|
||||||
sentinel: config.ErrNonPositiveValue,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
@@ -205,9 +171,7 @@ func TestRetentionSweepInterval(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if tt.expectError {
|
if tt.expectError {
|
||||||
expectStartupErrorFor(
|
expectStartupError(t)
|
||||||
t, "RETENTION_SWEEP_INTERVAL", tt.sentinel,
|
|
||||||
)
|
|
||||||
} else {
|
} else {
|
||||||
testRetentionSweepIntervalSuccess(t, tt.expected)
|
testRetentionSweepIntervalSuccess(t, tt.expected)
|
||||||
}
|
}
|
||||||
@@ -215,10 +179,9 @@ func TestRetentionSweepInterval(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// startupError builds the app config.New belongs to and returns
|
// expectStartupError asserts that fx refuses to build the app,
|
||||||
// the error fx reports, which is non-nil whenever an environment
|
// which is what a set-but-invalid environment value must cause.
|
||||||
// value is set but invalid.
|
func expectStartupError(t *testing.T) {
|
||||||
func startupError(t *testing.T) error {
|
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
var cfg *config.Config
|
var cfg *config.Config
|
||||||
@@ -233,33 +196,7 @@ func startupError(t *testing.T) error {
|
|||||||
fx.Populate(&cfg),
|
fx.Populate(&cfg),
|
||||||
)
|
)
|
||||||
|
|
||||||
return app.Err()
|
assert.Error(t, app.Err())
|
||||||
}
|
|
||||||
|
|
||||||
// expectStartupError asserts that fx refuses to build the app,
|
|
||||||
// which is what a set-but-invalid environment value must cause.
|
|
||||||
func expectStartupError(t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
assert.Error(t, startupError(t))
|
|
||||||
}
|
|
||||||
|
|
||||||
// expectStartupErrorFor asserts that startup fails, that the error
|
|
||||||
// names the offending variable so an operator can find it, and,
|
|
||||||
// when sentinel is non-nil, that it wraps that sentinel.
|
|
||||||
func expectStartupErrorFor(
|
|
||||||
t *testing.T,
|
|
||||||
key string,
|
|
||||||
sentinel error,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
err := startupError(t)
|
|
||||||
require.ErrorContains(t, err, key)
|
|
||||||
|
|
||||||
if sentinel != nil {
|
|
||||||
require.ErrorIs(t, err, sentinel)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func testRetentionSweepIntervalSuccess(
|
func testRetentionSweepIntervalSuccess(
|
||||||
@@ -313,22 +250,6 @@ func TestSessionIdleTimeout(t *testing.T) {
|
|||||||
value: "not-a-duration",
|
value: "not-a-duration",
|
||||||
expectError: true,
|
expectError: true,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
// Non-positive is "idle expiry disabled" for this
|
|
||||||
// variable, not a configuration error: unlike
|
|
||||||
// RETENTION_SWEEP_INTERVAL it never becomes a ticker
|
|
||||||
// period.
|
|
||||||
name: "zero disables idle expiry",
|
|
||||||
set: true,
|
|
||||||
value: "0s",
|
|
||||||
expected: 0,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "negative disables idle expiry",
|
|
||||||
set: true,
|
|
||||||
value: "-1h",
|
|
||||||
expected: -time.Hour,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
@@ -424,46 +345,12 @@ func TestDefaultDataDir(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestDataDirHelper pins the exported resolver against the value
|
|
||||||
// Config carries. The entry point takes the DATA_DIR lock through the
|
|
||||||
// helper before the fx graph exists, so the two disagreeing would mean
|
|
||||||
// locking one directory and writing to another.
|
|
||||||
func TestDataDirHelper(t *testing.T) {
|
|
||||||
for _, set := range []string{"", "/tmp/webhooker-datadir-helper"} {
|
|
||||||
name := "set"
|
|
||||||
if set == "" {
|
|
||||||
name = "unset"
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
|
||||||
// is incompatible with parallel subtests.
|
|
||||||
if set == "" {
|
|
||||||
require.NoError(t, os.Unsetenv("DATA_DIR"))
|
|
||||||
} else {
|
|
||||||
t.Setenv("DATA_DIR", set)
|
|
||||||
}
|
|
||||||
|
|
||||||
expected := set
|
|
||||||
if expected == "" {
|
|
||||||
expected = config.DefaultDataDir
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, expected, config.DataDir())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestReceiverRateLimit(t *testing.T) {
|
func TestReceiverRateLimit(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
set bool
|
set bool
|
||||||
value string
|
value string
|
||||||
expectError bool
|
expectError bool
|
||||||
// sentinel, when set, must be wrapped by the startup
|
|
||||||
// error; every error case must additionally name the
|
|
||||||
// variable in its message.
|
|
||||||
sentinel error
|
|
||||||
expected int
|
expected int
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
@@ -488,14 +375,12 @@ func TestReceiverRateLimit(t *testing.T) {
|
|||||||
set: true,
|
set: true,
|
||||||
value: "0",
|
value: "0",
|
||||||
expectError: true,
|
expectError: true,
|
||||||
sentinel: config.ErrNonPositiveValue,
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "negative fails startup",
|
name: "negative fails startup",
|
||||||
set: true,
|
set: true,
|
||||||
value: "-5",
|
value: "-5",
|
||||||
expectError: true,
|
expectError: true,
|
||||||
sentinel: config.ErrNonPositiveValue,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -514,9 +399,7 @@ func TestReceiverRateLimit(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if tt.expectError {
|
if tt.expectError {
|
||||||
expectStartupErrorFor(
|
expectStartupError(t)
|
||||||
t, "RECEIVER_RATE_LIMIT", tt.sentinel,
|
|
||||||
)
|
|
||||||
} else {
|
} else {
|
||||||
testReceiverRateLimitSuccess(t, tt.expected)
|
testReceiverRateLimitSuccess(t, tt.expected)
|
||||||
}
|
}
|
||||||
@@ -549,562 +432,3 @@ func testReceiverRateLimitSuccess(
|
|||||||
|
|
||||||
assert.Equal(t, expected, cfg.ReceiverRateLimit)
|
assert.Equal(t, expected, cfg.ReceiverRateLimit)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTrustedProxies(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
set bool
|
|
||||||
value string
|
|
||||||
expectError bool
|
|
||||||
expected []string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
// The default must be "trust nobody": an empty list
|
|
||||||
// means forwarded headers are ignored, never that
|
|
||||||
// every peer may speak for the client.
|
|
||||||
name: caseUnsetUsesDefault,
|
|
||||||
set: false,
|
|
||||||
expected: []string{},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "blank value trusts nothing",
|
|
||||||
set: true,
|
|
||||||
value: " ",
|
|
||||||
expected: []string{},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: caseValidValueParsed,
|
|
||||||
set: true,
|
|
||||||
value: cidrPrivateV4 + ", 192.168.1.7 ,2001:db8::/32",
|
|
||||||
expected: []string{
|
|
||||||
cidrPrivateV4, "192.168.1.7/32", "2001:db8::/32",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "host bits are masked off",
|
|
||||||
set: true,
|
|
||||||
value: "10.1.2.3/8",
|
|
||||||
expected: []string{cidrPrivateV4},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// Peer addresses are unmapped before they are
|
|
||||||
// matched, so an IPv4-mapped prefix kept in that
|
|
||||||
// form could never match anything.
|
|
||||||
name: "IPv4-mapped prefix is unmapped",
|
|
||||||
set: true,
|
|
||||||
value: "::ffff:10.0.0.0/104",
|
|
||||||
expected: []string{cidrPrivateV4},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: caseUnparseableFails,
|
|
||||||
set: true,
|
|
||||||
value: cidrPrivateV4 + ",not-an-address",
|
|
||||||
expectError: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "out-of-range prefix length fails startup",
|
|
||||||
set: true,
|
|
||||||
value: "10.0.0.0/33",
|
|
||||||
expectError: true,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
|
||||||
// is incompatible with parallel subtests.
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
|
||||||
|
|
||||||
if tt.set {
|
|
||||||
t.Setenv("TRUSTED_PROXIES", tt.value)
|
|
||||||
} else {
|
|
||||||
require.NoError(t, os.Unsetenv("TRUSTED_PROXIES"))
|
|
||||||
}
|
|
||||||
|
|
||||||
if tt.expectError {
|
|
||||||
expectStartupErrorFor(
|
|
||||||
t, "TRUSTED_PROXIES", config.ErrInvalidCIDR,
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
testTrustedProxiesSuccess(t, tt.expected)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func testTrustedProxiesSuccess(
|
|
||||||
t *testing.T,
|
|
||||||
expected []string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var cfg *config.Config
|
|
||||||
|
|
||||||
app := fxtest.New(
|
|
||||||
t,
|
|
||||||
fx.Provide(
|
|
||||||
globals.New,
|
|
||||||
logger.New,
|
|
||||||
config.New,
|
|
||||||
),
|
|
||||||
fx.Populate(&cfg),
|
|
||||||
)
|
|
||||||
require.NoError(t, app.Err())
|
|
||||||
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
defer app.RequireStop()
|
|
||||||
|
|
||||||
got := make([]string, 0, len(cfg.TrustedProxies))
|
|
||||||
for _, prefix := range cfg.TrustedProxies {
|
|
||||||
got = append(got, prefix.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, expected, got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestAllowedEgressCIDRs covers ALLOWED_EGRESS_CIDRS, the escape
|
|
||||||
// hatch that lets a self-hosted deployment forward to its own
|
|
||||||
// network. Unset it must stay empty, so the SSRF guard keeps
|
|
||||||
// refusing every private/reserved range; a set-but-unparseable
|
|
||||||
// value must abort startup naming the variable rather than
|
|
||||||
// silently running with a list the operator did not write.
|
|
||||||
func TestAllowedEgressCIDRs(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
set bool
|
|
||||||
value string
|
|
||||||
expected []string
|
|
||||||
expectError bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: caseUnsetUsesDefault,
|
|
||||||
set: false,
|
|
||||||
expected: []string{},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "empty value yields empty list",
|
|
||||||
set: true,
|
|
||||||
value: "",
|
|
||||||
expected: []string{},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: caseValidValueParsed,
|
|
||||||
set: true,
|
|
||||||
value: cidrPrivateV4,
|
|
||||||
expected: []string{cidrPrivateV4},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "multiple blocks with whitespace",
|
|
||||||
set: true,
|
|
||||||
value: " 10.0.0.0/8 , 127.0.0.0/8 ",
|
|
||||||
expected: []string{cidrPrivateV4, "127.0.0.0/8"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "bare address becomes a single host",
|
|
||||||
set: true,
|
|
||||||
value: "172.17.0.5",
|
|
||||||
expected: []string{"172.17.0.5/32"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: caseUnparseableFails,
|
|
||||||
set: true,
|
|
||||||
value: cidrPrivateV4 + ",not-an-address",
|
|
||||||
expectError: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "out-of-range prefix length fails startup",
|
|
||||||
set: true,
|
|
||||||
value: "10.0.0.0/33",
|
|
||||||
expectError: true,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
|
||||||
// is incompatible with parallel subtests.
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
|
||||||
|
|
||||||
if tt.set {
|
|
||||||
t.Setenv("ALLOWED_EGRESS_CIDRS", tt.value)
|
|
||||||
} else {
|
|
||||||
require.NoError(
|
|
||||||
t, os.Unsetenv("ALLOWED_EGRESS_CIDRS"),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if tt.expectError {
|
|
||||||
expectStartupErrorFor(
|
|
||||||
t, "ALLOWED_EGRESS_CIDRS", config.ErrInvalidCIDR,
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
testAllowedEgressCIDRsSuccess(t, tt.expected)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func testAllowedEgressCIDRsSuccess(
|
|
||||||
t *testing.T,
|
|
||||||
expected []string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var cfg *config.Config
|
|
||||||
|
|
||||||
app := fxtest.New(
|
|
||||||
t,
|
|
||||||
fx.Provide(
|
|
||||||
globals.New,
|
|
||||||
logger.New,
|
|
||||||
config.New,
|
|
||||||
),
|
|
||||||
fx.Populate(&cfg),
|
|
||||||
)
|
|
||||||
require.NoError(t, app.Err())
|
|
||||||
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
defer app.RequireStop()
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t, expected, config.PrefixStrings(cfg.AllowedEgressCIDRs),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEgressAllowlistWarning covers the startup log that shows an
|
|
||||||
// operator the hole ALLOWED_EGRESS_CIDRS opened. It must stay
|
|
||||||
// silent on the default (empty) list and, when set, print the
|
|
||||||
// blocks themselves rather than a count.
|
|
||||||
func TestEgressAllowlistWarning(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
allowed string
|
|
||||||
expectWarning bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "empty allowlist is quiet",
|
|
||||||
expectWarning: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "non-empty allowlist warns",
|
|
||||||
allowed: "10.0.0.0/8,127.0.0.0/8",
|
|
||||||
expectWarning: true,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
|
||||||
// is incompatible with parallel subtests.
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", config.EnvironmentDev)
|
|
||||||
|
|
||||||
if tt.allowed == "" {
|
|
||||||
require.NoError(
|
|
||||||
t, os.Unsetenv("ALLOWED_EGRESS_CIDRS"),
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
t.Setenv("ALLOWED_EGRESS_CIDRS", tt.allowed)
|
|
||||||
}
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
log := slog.New(slog.NewJSONHandler(
|
|
||||||
&buf, &slog.HandlerOptions{
|
|
||||||
Level: slog.LevelDebug,
|
|
||||||
},
|
|
||||||
))
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t, config.WarnEgressAllowlistForTest(log),
|
|
||||||
)
|
|
||||||
|
|
||||||
if !tt.expectWarning {
|
|
||||||
assert.Empty(t, buf.String())
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
logged := buf.String()
|
|
||||||
|
|
||||||
assert.Contains(t, logged, `"level":"WARN"`)
|
|
||||||
assert.Contains(t, logged, "ALLOWED_EGRESS_CIDRS")
|
|
||||||
// The blocks themselves, not a count: the operator has
|
|
||||||
// to be able to read back which networks are open.
|
|
||||||
assert.Contains(t, logged, "10.0.0.0/8")
|
|
||||||
assert.Contains(t, logged, "127.0.0.0/8")
|
|
||||||
// What stays shut. Asserted on the clause naming the
|
|
||||||
// wider set rather than on "Link-local" alone, so the
|
|
||||||
// string cannot narrow back to link-local only while
|
|
||||||
// the always-blocked set covers ULA, CGNAT and two
|
|
||||||
// public metadata addresses as well.
|
|
||||||
assert.Contains(t, logged, "metadata endpoints outside it")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSharedRateLimitBucketWarning covers the startup warning that
|
|
||||||
// tells an operator a deployment behind a reverse proxy shares one
|
|
||||||
// rate-limit bucket between every client, which turns the receiver
|
|
||||||
// limits into service-wide ceilings and collapses login failure
|
|
||||||
// counting. It must fire whenever TRUSTED_PROXIES is empty,
|
|
||||||
// in any environment: WEBHOOKER_ENVIRONMENT defaults to dev, so gating
|
|
||||||
// on it would silence the warning for exactly the operator who never
|
|
||||||
// configured the deployment. It stays quiet once proxies are named.
|
|
||||||
func TestSharedRateLimitBucketWarning(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
environment string
|
|
||||||
trustedProxies string
|
|
||||||
expectWarning bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "prod without trusted proxies warns",
|
|
||||||
environment: config.EnvironmentProd,
|
|
||||||
expectWarning: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "prod with trusted proxies is quiet",
|
|
||||||
environment: config.EnvironmentProd,
|
|
||||||
trustedProxies: cidrPrivateV4,
|
|
||||||
expectWarning: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// The default environment. An internet-exposed
|
|
||||||
// deployment whose operator never set
|
|
||||||
// WEBHOOKER_ENVIRONMENT lands here and has exactly
|
|
||||||
// the exposure the warning announces.
|
|
||||||
name: "dev without trusted proxies warns",
|
|
||||||
environment: config.EnvironmentDev,
|
|
||||||
expectWarning: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "dev with trusted proxies is quiet",
|
|
||||||
environment: config.EnvironmentDev,
|
|
||||||
trustedProxies: cidrPrivateV4,
|
|
||||||
expectWarning: false,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
|
||||||
// is incompatible with parallel subtests.
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", tt.environment)
|
|
||||||
|
|
||||||
if tt.trustedProxies == "" {
|
|
||||||
require.NoError(
|
|
||||||
t, os.Unsetenv("TRUSTED_PROXIES"),
|
|
||||||
)
|
|
||||||
} else {
|
|
||||||
t.Setenv("TRUSTED_PROXIES", tt.trustedProxies)
|
|
||||||
}
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
log := slog.New(slog.NewJSONHandler(
|
|
||||||
&buf, &slog.HandlerOptions{
|
|
||||||
Level: slog.LevelDebug,
|
|
||||||
},
|
|
||||||
))
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t,
|
|
||||||
config.WarnSharedRateLimitBucketForTest(log),
|
|
||||||
)
|
|
||||||
|
|
||||||
if !tt.expectWarning {
|
|
||||||
assert.Empty(t, buf.String())
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
logged := buf.String()
|
|
||||||
|
|
||||||
assert.Contains(t, logged, `"level":"WARN"`)
|
|
||||||
assert.Contains(t, logged, "TRUSTED_PROXIES")
|
|
||||||
assert.Contains(t, logged, "share one bucket")
|
|
||||||
assert.Contains(
|
|
||||||
t, logged, "throttle every other client's failed logins",
|
|
||||||
)
|
|
||||||
// The warning must not claim a lockout the login
|
|
||||||
// endpoint no longer permits: credentials are verified
|
|
||||||
// before any budget is spent.
|
|
||||||
assert.Contains(
|
|
||||||
t, logged, "a correct password still gets in",
|
|
||||||
)
|
|
||||||
// The text must stay accurate for a developer with
|
|
||||||
// nothing in front of the process, where an empty
|
|
||||||
// list costs nothing.
|
|
||||||
assert.Contains(
|
|
||||||
t, logged, "nothing proxying to this process",
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// metricsEnv describes what one subtest below puts in the
|
|
||||||
// environment for a single METRICS_ variable. A variable that is
|
|
||||||
// set to the empty string and one that is not set at all are
|
|
||||||
// distinct inputs here, because the reported bug arrived through
|
|
||||||
// the first of them.
|
|
||||||
type metricsEnv struct {
|
|
||||||
set bool
|
|
||||||
value string
|
|
||||||
}
|
|
||||||
|
|
||||||
// unset leaves the variable out of the environment entirely.
|
|
||||||
func unset() metricsEnv {
|
|
||||||
return metricsEnv{set: false, value: ""}
|
|
||||||
}
|
|
||||||
|
|
||||||
// setTo sets the variable, including to the empty string.
|
|
||||||
func setTo(value string) metricsEnv {
|
|
||||||
return metricsEnv{set: true, value: value}
|
|
||||||
}
|
|
||||||
|
|
||||||
// metricsAuthCase is one row of the table in TestMetricsAuthConfig,
|
|
||||||
// named so the table can live in its own function and keep the test
|
|
||||||
// itself short.
|
|
||||||
type metricsAuthCase struct {
|
|
||||||
name string
|
|
||||||
username metricsEnv
|
|
||||||
password metricsEnv
|
|
||||||
expectError bool
|
|
||||||
expectAuth bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// metricsAuthCases enumerates every combination of the two
|
|
||||||
// credentials, counting "set to the empty string" and "not set at
|
|
||||||
// all" as separate inputs on each side.
|
|
||||||
func metricsAuthCases() []metricsAuthCase {
|
|
||||||
return []metricsAuthCase{
|
|
||||||
{
|
|
||||||
name: "both unset leaves metrics unmounted",
|
|
||||||
username: unset(),
|
|
||||||
password: unset(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "both empty leaves metrics unmounted",
|
|
||||||
username: setTo(""),
|
|
||||||
password: setTo(""),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "both set enables metrics auth",
|
|
||||||
username: setTo("metrics"),
|
|
||||||
password: setTo(metricsAuthValue),
|
|
||||||
expectAuth: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "username with unset password fails",
|
|
||||||
username: setTo("metrics"),
|
|
||||||
password: unset(),
|
|
||||||
expectError: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "username with empty password fails",
|
|
||||||
username: setTo("metrics"),
|
|
||||||
password: setTo(""),
|
|
||||||
expectError: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "password with unset username fails",
|
|
||||||
username: unset(),
|
|
||||||
password: setTo(metricsAuthValue),
|
|
||||||
expectError: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "password with empty username fails",
|
|
||||||
username: setTo(""),
|
|
||||||
password: setTo(metricsAuthValue),
|
|
||||||
expectError: true,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestMetricsAuthConfig covers every combination of METRICS_USERNAME
|
|
||||||
// and METRICS_PASSWORD. Either both carry a value, in which case
|
|
||||||
// /metrics is served behind basic auth, or neither does, in which
|
|
||||||
// case the route is never mounted. One without the other is a
|
|
||||||
// startup error rather than a fallback: mounting on the username
|
|
||||||
// alone published /metrics behind a credential map that accepted an
|
|
||||||
// empty password, which is the defect this test exists to pin. See
|
|
||||||
// https://git.eeqj.de/sneak/webhooker/issues/205.
|
|
||||||
func TestMetricsAuthConfig(t *testing.T) {
|
|
||||||
for _, tt := range metricsAuthCases() {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
|
||||||
// is incompatible with parallel subtests.
|
|
||||||
if tt.username.set {
|
|
||||||
t.Setenv("METRICS_USERNAME", tt.username.value)
|
|
||||||
} else {
|
|
||||||
require.NoError(
|
|
||||||
t, os.Unsetenv("METRICS_USERNAME"),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if tt.password.set {
|
|
||||||
t.Setenv("METRICS_PASSWORD", tt.password.value)
|
|
||||||
} else {
|
|
||||||
require.NoError(
|
|
||||||
t, os.Unsetenv("METRICS_PASSWORD"),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if tt.expectError {
|
|
||||||
assertMetricsAuthRejected(t)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
assertMetricsAuthAccepted(t, tt.expectAuth)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertMetricsAuthRejected requires that fx refused to build the
|
|
||||||
// graph, that the failure is ErrIncompleteMetricsAuth, and that the
|
|
||||||
// operator is told both variable names — the point of failing here
|
|
||||||
// rather than degrading is that the message says what to fix.
|
|
||||||
func assertMetricsAuthRejected(t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var cfg *config.Config
|
|
||||||
|
|
||||||
app := fx.New(
|
|
||||||
fx.NopLogger,
|
|
||||||
fx.Provide(globals.New, logger.New, config.New),
|
|
||||||
fx.Populate(&cfg),
|
|
||||||
)
|
|
||||||
|
|
||||||
err := app.Err()
|
|
||||||
require.Error(t, err)
|
|
||||||
require.ErrorIs(t, err, config.ErrIncompleteMetricsAuth)
|
|
||||||
assert.Contains(t, err.Error(), "METRICS_USERNAME")
|
|
||||||
assert.Contains(t, err.Error(), "METRICS_PASSWORD")
|
|
||||||
// The password is a secret and must not reach a startup error.
|
|
||||||
assert.NotContains(t, err.Error(), metricsAuthValue)
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertMetricsAuthAccepted requires that startup succeeded and that
|
|
||||||
// MetricsAuthEnabled — the single value the /metrics mount and the
|
|
||||||
// startup log both read — reports what the environment asked for.
|
|
||||||
func assertMetricsAuthAccepted(t *testing.T, expectAuth bool) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var cfg *config.Config
|
|
||||||
|
|
||||||
app := fxtest.New(
|
|
||||||
t,
|
|
||||||
fx.Provide(globals.New, logger.New, config.New),
|
|
||||||
fx.Populate(&cfg),
|
|
||||||
)
|
|
||||||
require.NoError(t, app.Err())
|
|
||||||
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
defer app.RequireStop()
|
|
||||||
|
|
||||||
assert.Equal(t, expectAuth, cfg.MetricsAuthEnabled())
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,41 +1,9 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import "log/slog"
|
|
||||||
|
|
||||||
// This file exposes the unexported environment parsing helpers to
|
// This file exposes the unexported environment parsing helpers to
|
||||||
// the external config_test package so each helper can be covered by
|
// the external config_test package so each helper can be covered by
|
||||||
// its own table-driven test without weakening the package API.
|
// its own table-driven test without weakening the package API.
|
||||||
|
|
||||||
// WarnSharedRateLimitBucketForTest loads a Config from the current
|
|
||||||
// environment and emits its startup warnings to log. The real logger
|
|
||||||
// writes to stdout, so this lets the warning's firing condition be
|
|
||||||
// asserted against a handler the test controls.
|
|
||||||
func WarnSharedRateLimitBucketForTest(log *slog.Logger) error {
|
|
||||||
c, err := loadFromEnv()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
c.warnSharedRateLimitBucket(log)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// WarnEgressAllowlistForTest loads a Config from the current
|
|
||||||
// environment and emits its egress-allowlist startup warning to
|
|
||||||
// log, so a test can assert both that the warning fires only when
|
|
||||||
// the list is non-empty and that it names the blocks it opened.
|
|
||||||
func WarnEgressAllowlistForTest(log *slog.Logger) error {
|
|
||||||
c, err := loadFromEnv()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
c.warnEgressAllowlist(log)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// EnvBoolForTest exposes envBool.
|
// EnvBoolForTest exposes envBool.
|
||||||
func EnvBoolForTest(key string, defaultValue bool) (bool, error) {
|
func EnvBoolForTest(key string, defaultValue bool) (bool, error) {
|
||||||
return envBool(key, defaultValue)
|
return envBool(key, defaultValue)
|
||||||
|
|||||||
@@ -1,85 +0,0 @@
|
|||||||
package database_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// passwordField is the banner line carrying the plaintext.
|
|
||||||
const passwordField = "password: "
|
|
||||||
|
|
||||||
// bannerPassword returns the password the banner printed.
|
|
||||||
func bannerPassword(t *testing.T, out string) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for line := range strings.SplitSeq(out, "\n") {
|
|
||||||
_, value, found := strings.Cut(line, passwordField)
|
|
||||||
if found {
|
|
||||||
return strings.TrimSpace(value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Fatalf("no %q line in the banner:\n%s", passwordField, out)
|
|
||||||
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFirstBoot_PrintsTheAdminPasswordAsABanner is the bootstrap half
|
|
||||||
// of https://git.eeqj.de/sneak/webhooker/issues/208.
|
|
||||||
//
|
|
||||||
// The password is shown exactly once, and it used to be shown as one
|
|
||||||
// slog record among the roughly 45 fx PROVIDE/RUN/HOOK lines a boot
|
|
||||||
// writes — which is how deployments lost it and, with no reset path,
|
|
||||||
// locked themselves out. It must be emitted as a block an operator can
|
|
||||||
// find by eye, it must carry the plaintext that actually opens the
|
|
||||||
// account, and it must name the command that recovers it.
|
|
||||||
func TestFirstBoot_PrintsTheAdminPasswordAsABanner(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db, lc := setupTestDB(t)
|
|
||||||
|
|
||||||
var out bytes.Buffer
|
|
||||||
|
|
||||||
db.ExportSetBannerOut(&out)
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
require.NoError(t, lc.Start(ctx))
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
|
||||||
|
|
||||||
printed := out.String()
|
|
||||||
|
|
||||||
require.Contains(
|
|
||||||
t, printed, strings.Repeat("=", 20),
|
|
||||||
"the banner must be ruled off, not read as one more log line",
|
|
||||||
)
|
|
||||||
require.Contains(t, printed, "username: admin")
|
|
||||||
assert.Contains(
|
|
||||||
t, printed, "resetpw",
|
|
||||||
"the banner must name the command that recovers the account",
|
|
||||||
)
|
|
||||||
|
|
||||||
password := bannerPassword(t, printed)
|
|
||||||
require.NotEmpty(t, password)
|
|
||||||
|
|
||||||
// The printed plaintext must be the one that opens the account:
|
|
||||||
// a banner showing a different string would be worse than none.
|
|
||||||
var user database.User
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t,
|
|
||||||
db.DB().Where("username = ?", "admin").First(&user).Error,
|
|
||||||
)
|
|
||||||
|
|
||||||
ok, err := database.VerifyPassword(password, user.Password)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.True(
|
|
||||||
t, ok, "the printed password must open the seeded account",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -8,7 +8,6 @@ import (
|
|||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -17,9 +16,7 @@ import (
|
|||||||
"gorm.io/driver/sqlite"
|
"gorm.io/driver/sqlite"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
_ "modernc.org/sqlite" // Pure Go SQLite driver
|
_ "modernc.org/sqlite" // Pure Go SQLite driver
|
||||||
"sneak.berlin/go/webhooker/internal/banner"
|
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -29,20 +26,6 @@ const (
|
|||||||
sessionKeyLen = 32
|
sessionKeyLen = 32
|
||||||
)
|
)
|
||||||
|
|
||||||
// MainDBFileName is the main application database inside DATA_DIR. It
|
|
||||||
// is exported so that an entry point acting on a data directory
|
|
||||||
// outside the fx graph can test for a deployment's existence without
|
|
||||||
// spelling the name a second time.
|
|
||||||
const MainDBFileName = "webhooker.db"
|
|
||||||
|
|
||||||
// BootstrapPasswordNote is what the first-boot banner tells the
|
|
||||||
// operator to do about the password it just printed. It names the
|
|
||||||
// recovery command, because the moment that line scrolls away is
|
|
||||||
// exactly when the operator needs to know one exists.
|
|
||||||
const BootstrapPasswordNote = "Save this password now: it is shown " +
|
|
||||||
"only here, and only once.\nIf it is lost, run `webhooker " +
|
|
||||||
"resetpw admin` on a stopped deployment."
|
|
||||||
|
|
||||||
//nolint:revive // DatabaseParams is a standard fx naming convention.
|
//nolint:revive // DatabaseParams is a standard fx naming convention.
|
||||||
type DatabaseParams struct {
|
type DatabaseParams struct {
|
||||||
fx.In
|
fx.In
|
||||||
@@ -56,39 +39,6 @@ type Database struct {
|
|||||||
db *gorm.DB
|
db *gorm.DB
|
||||||
log *slog.Logger
|
log *slog.Logger
|
||||||
params *DatabaseParams
|
params *DatabaseParams
|
||||||
|
|
||||||
// bannerOut receives the first-boot credentials banner. Nil means
|
|
||||||
// os.Stdout, resolved at write time rather than at construction so
|
|
||||||
// that a caller which redirects the variable still captures it.
|
|
||||||
bannerOut io.Writer
|
|
||||||
}
|
|
||||||
|
|
||||||
// Open connects to the main database in dataDir and migrates it,
|
|
||||||
// without the fx lifecycle and without seeding an admin account.
|
|
||||||
//
|
|
||||||
// It is for entry points that act on an existing deployment's data
|
|
||||||
// directory from outside the server graph — `webhooker resetpw`. Such a
|
|
||||||
// caller must already hold the DATA_DIR lock (see internal/datadir),
|
|
||||||
// and must Close the result.
|
|
||||||
//
|
|
||||||
// It does not create the admin account: seeding belongs to a server
|
|
||||||
// start, and a maintenance command that silently invented an account
|
|
||||||
// would answer "no such user" by creating one.
|
|
||||||
func Open(dataDir string, log *slog.Logger) (*Database, error) {
|
|
||||||
d := &Database{log: log}
|
|
||||||
|
|
||||||
err := d.connectTo(dataDir)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return d, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close closes the underlying connection. It is the exported form of
|
|
||||||
// the fx stop hook, for callers that built the Database with Open.
|
|
||||||
func (d *Database) Close() error {
|
|
||||||
return d.close()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a Database that connects on fx start and disconnects on stop.
|
// New creates a Database that connects on fx start and disconnects on stop.
|
||||||
@@ -171,22 +121,10 @@ func (d *Database) GetOrCreateSessionKey() (string, error) {
|
|||||||
return encoded, nil
|
return encoded, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// connect opens the configured data directory and, this being a
|
|
||||||
// server start, seeds the admin account when the deployment has none.
|
|
||||||
func (d *Database) connect() error {
|
func (d *Database) connect() error {
|
||||||
err := d.connectTo(d.params.Config.DataDir)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return d.ensureAdminUser()
|
|
||||||
}
|
|
||||||
|
|
||||||
// connectTo opens and migrates the main database in dataDir. It seeds
|
|
||||||
// nothing: whether an empty deployment gets an admin account is the
|
|
||||||
// caller's decision.
|
|
||||||
func (d *Database) connectTo(dataDir string) error {
|
|
||||||
// Ensure the data directory exists before opening the database.
|
// Ensure the data directory exists before opening the database.
|
||||||
|
dataDir := d.params.Config.DataDir
|
||||||
|
|
||||||
err := os.MkdirAll(dataDir, dataDirPerm)
|
err := os.MkdirAll(dataDir, dataDirPerm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
@@ -197,7 +135,7 @@ func (d *Database) connectTo(dataDir string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Construct the main application database path inside DATA_DIR.
|
// Construct the main application database path inside DATA_DIR.
|
||||||
dbPath := filepath.Join(dataDir, MainDBFileName)
|
dbPath := filepath.Join(dataDir, "webhooker.db")
|
||||||
dbURL := fmt.Sprintf(
|
dbURL := fmt.Sprintf(
|
||||||
"file:%s?cache=shared&mode=rwc",
|
"file:%s?cache=shared&mode=rwc",
|
||||||
dbPath,
|
dbPath,
|
||||||
@@ -217,10 +155,7 @@ func (d *Database) connectTo(dataDir string) error {
|
|||||||
// Then use it with GORM
|
// Then use it with GORM
|
||||||
db, err := gorm.Open(sqlite.Dialector{
|
db, err := gorm.Open(sqlite.Dialector{
|
||||||
Conn: sqlDB,
|
Conn: sqlDB,
|
||||||
}, &gorm.Config{
|
}, &gorm.Config{})
|
||||||
// Never leave this at GORM's default. See internal/gormlog.
|
|
||||||
Logger: gormlog.New(d.log),
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
d.log.Error(
|
d.log.Error(
|
||||||
"failed to connect to database",
|
"failed to connect to database",
|
||||||
@@ -251,16 +186,10 @@ func (d *Database) migrate() error {
|
|||||||
|
|
||||||
d.log.Info("database migrations completed")
|
d.log.Info("database migrations completed")
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ensureAdminUser creates the bootstrap admin account when the
|
|
||||||
// deployment has no users at all.
|
|
||||||
func (d *Database) ensureAdminUser() error {
|
|
||||||
// Check if admin user exists
|
// Check if admin user exists
|
||||||
var userCount int64
|
var userCount int64
|
||||||
|
|
||||||
err := d.db.Model(&User{}).Count(&userCount).Error
|
err = d.db.Model(&User{}).Count(&userCount).Error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
d.log.Error(
|
d.log.Error(
|
||||||
"failed to count users",
|
"failed to count users",
|
||||||
@@ -320,46 +249,16 @@ func (d *Database) createAdminUser() error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// The plaintext leaves this process here and nowhere else. It is
|
d.log.Info("admin user created",
|
||||||
// deliberately not a log field: as one INFO record among the fx
|
"username", "admin",
|
||||||
// graph's own output it read as one more startup line, which is
|
"password", password,
|
||||||
// how deployments lost it. See internal/banner.
|
"message",
|
||||||
err = banner.Credentials(
|
"SAVE THIS PASSWORD - it will not be shown again!",
|
||||||
d.banner(),
|
|
||||||
"WEBHOOKER FIRST BOOT: an admin account has been created.",
|
|
||||||
adminUser.Username,
|
|
||||||
password,
|
|
||||||
BootstrapPasswordNote,
|
|
||||||
)
|
)
|
||||||
if err != nil {
|
|
||||||
// Fail the start. The account is already committed, so the
|
|
||||||
// next boot seeds nothing and prints nothing: continuing here
|
|
||||||
// would hand the operator a running service whose only
|
|
||||||
// password was never shown. `webhooker resetpw` recovers it.
|
|
||||||
d.log.Error(
|
|
||||||
"failed to print the admin credentials banner",
|
|
||||||
"error", err,
|
|
||||||
)
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
d.log.Info("admin user created", "username", adminUser.Username)
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// banner returns where the credentials banner is written. os.Stdout is
|
|
||||||
// resolved here rather than stored, so that a test which redirects the
|
|
||||||
// variable captures the banner.
|
|
||||||
func (d *Database) banner() io.Writer {
|
|
||||||
if d.bannerOut != nil {
|
|
||||||
return d.bannerOut
|
|
||||||
}
|
|
||||||
|
|
||||||
return os.Stdout
|
|
||||||
}
|
|
||||||
|
|
||||||
func (d *Database) close() error {
|
func (d *Database) close() error {
|
||||||
if d.db != nil {
|
if d.db != nil {
|
||||||
sqlDB, err := d.db.DB()
|
sqlDB, err := d.db.DB()
|
||||||
|
|||||||
@@ -18,11 +18,6 @@ const (
|
|||||||
testVersion = "test"
|
testVersion = "test"
|
||||||
// testContentType is the event content type used in tests.
|
// testContentType is the event content type used in tests.
|
||||||
testContentType = "application/json"
|
testContentType = "application/json"
|
||||||
// testWebhookName is the Webhook.Name used in tests.
|
|
||||||
testWebhookName = "test-webhook"
|
|
||||||
// testForeverLabel is Webhook.RetentionLabel for a retain-forever
|
|
||||||
// webhook.
|
|
||||||
testForeverLabel = "forever"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func setupTestDB(
|
func setupTestDB(
|
||||||
|
|||||||
@@ -1,159 +0,0 @@
|
|||||||
package database
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"log/slog"
|
|
||||||
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"gorm.io/gorm/clause"
|
|
||||||
)
|
|
||||||
|
|
||||||
// omitAssociationsCallback is the name the association guard is
|
|
||||||
// registered under on a per-webhook database's create and update
|
|
||||||
// callback chains.
|
|
||||||
const omitAssociationsCallback = "webhooker:omit_associations"
|
|
||||||
|
|
||||||
// omitAssociations makes every create and update issued against a
|
|
||||||
// per-webhook database skip GORM's automatic association save.
|
|
||||||
//
|
|
||||||
// A per-webhook database holds the event tier only, but Delivery
|
|
||||||
// declares belongs-to Event and Target and the delivery engine fills
|
|
||||||
// both in memory before writing. Without this guard GORM upserts
|
|
||||||
// those parent rows here on the delivery and retry write paths,
|
|
||||||
// copying targets.config, which holds destination URLs and bearer
|
|
||||||
// credentials, into the file most likely to be backed up or handed
|
|
||||||
// to someone else. Registering the guard on the connection covers
|
|
||||||
// every write path, including writes inside a transaction and write
|
|
||||||
// paths added later. Every event-tier row this file holds is written
|
|
||||||
// explicitly, so nothing depends on the automatic save.
|
|
||||||
func omitAssociations(db *gorm.DB) error {
|
|
||||||
omit := func(tx *gorm.DB) {
|
|
||||||
tx.Statement.Omits = append(
|
|
||||||
tx.Statement.Omits, clause.Associations,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
err := db.Callback().Create().
|
|
||||||
Before("gorm:save_before_associations").
|
|
||||||
Register(omitAssociationsCallback, omit)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"registering create association guard: %w", err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = db.Callback().Update().
|
|
||||||
Before("gorm:save_before_associations").
|
|
||||||
Register(omitAssociationsCallback, omit)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"registering update association guard: %w", err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// eventDBSweptVersion is the PRAGMA user_version purgeTargetRows
|
|
||||||
// stamps into a per-webhook database once it has removed any leaked
|
|
||||||
// target rows *and* the VACUUM that removes their bytes has returned.
|
|
||||||
// Nothing else in the tree uses user_version, so 0 means "not swept
|
|
||||||
// by this build".
|
|
||||||
//
|
|
||||||
// The stamp, not the DELETE, is what records that a file is done. A
|
|
||||||
// DELETE commits on its own, so a sweep that is interrupted or whose
|
|
||||||
// VACUUM fails leaves a file whose rows are gone but whose credential
|
|
||||||
// bytes are still in the free pages -- indistinguishable, by row
|
|
||||||
// count, from a file that never leaked. Both leave the stamp unset,
|
|
||||||
// so the next open sweeps again.
|
|
||||||
const eventDBSweptVersion = 1
|
|
||||||
|
|
||||||
// purgeTargetRows deletes target rows that an earlier build's
|
|
||||||
// association upsert wrote into a per-webhook database, and rewrites
|
|
||||||
// the file so their bytes are gone with them. AutoMigrate creates a
|
|
||||||
// targets table in every one of these files because Delivery declares
|
|
||||||
// a belongs-to Target, but nothing in the event tier may put rows in
|
|
||||||
// it. The rows it did put there are junk, not history: they carry an
|
|
||||||
// empty webhook_id, and delivery rows resolve their target against
|
|
||||||
// the main database, so nothing here refers to them.
|
|
||||||
//
|
|
||||||
// The DELETE only unlinks the rows: modernc.org/sqlite leaves
|
|
||||||
// secure_delete at SQLite's default of off, so the credential bytes
|
|
||||||
// stay readable in the file's free pages and a backup of a swept file
|
|
||||||
// would still hand them over. VACUUM rewrites the file without them.
|
|
||||||
//
|
|
||||||
// This runs before every migration and is gated on
|
|
||||||
// eventDBSweptVersion, so a file pays for the rewrite once, on the
|
|
||||||
// first open that finds it unstamped, and every open after that is a
|
|
||||||
// PRAGMA read. A file this build created is stamped before its
|
|
||||||
// targets table exists, so it never vacuums at all. A failure here
|
|
||||||
// fails the open with the stamp left unset, so the sweep is retried
|
|
||||||
// rather than skipped -- a webhook whose file cannot be swept stays
|
|
||||||
// unusable instead of quietly serving from a file that still holds
|
|
||||||
// recoverable credentials.
|
|
||||||
func purgeTargetRows(
|
|
||||||
db *gorm.DB, log *slog.Logger, webhookID string,
|
|
||||||
) error {
|
|
||||||
var version int
|
|
||||||
|
|
||||||
// Row().Scan, not (*gorm.DB).Scan: see internal/gormlog.
|
|
||||||
err := db.Raw("PRAGMA user_version").Row().Scan(&version)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"reading sweep marker of webhook database %s: %w",
|
|
||||||
webhookID, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if version >= eventDBSweptVersion {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var purged int64
|
|
||||||
|
|
||||||
if db.Migrator().HasTable("targets") {
|
|
||||||
res := db.Exec("DELETE FROM targets")
|
|
||||||
if res.Error != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"purging target rows from webhook database %s: %w",
|
|
||||||
webhookID, res.Error,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
purged = res.RowsAffected
|
|
||||||
|
|
||||||
// Unconditional: a zero row count here does not mean there is
|
|
||||||
// nothing to remove, only that no *live* row is left. See
|
|
||||||
// eventDBSweptVersion.
|
|
||||||
err = db.Exec("VACUUM").Error
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"purged %d leaked target rows from webhook database "+
|
|
||||||
"%s but vacuuming it failed, so the deleted "+
|
|
||||||
"target credentials are still recoverable from "+
|
|
||||||
"the file; it stays marked unswept and the next "+
|
|
||||||
"open retries: %w",
|
|
||||||
purged, webhookID, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
err = db.Exec(fmt.Sprintf(
|
|
||||||
"PRAGMA user_version = %d", eventDBSweptVersion,
|
|
||||||
)).Error
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"marking webhook database %s swept: %w", webhookID, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if purged > 0 {
|
|
||||||
log.Warn(
|
|
||||||
"purged leaked target rows from per-webhook database",
|
|
||||||
"webhook_id", webhookID,
|
|
||||||
"rows", purged,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,438 +0,0 @@
|
|||||||
package database_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"database/sql"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
_ "modernc.org/sqlite"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// testDataDirPerm is the mode the test data directory is created
|
|
||||||
// with.
|
|
||||||
const testDataDirPerm = 0o750
|
|
||||||
|
|
||||||
// eventDBDataDir returns a data directory that a WebhookDBManager
|
|
||||||
// can be pointed at.
|
|
||||||
func eventDBDataDir(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
dir := filepath.Join(t.TempDir(), "events")
|
|
||||||
require.NoError(t, os.MkdirAll(dir, testDataDirPerm))
|
|
||||||
|
|
||||||
return dir
|
|
||||||
}
|
|
||||||
|
|
||||||
// openRawEventDB opens the per-webhook database file directly,
|
|
||||||
// without the manager, so a test can put a file on disk in a state
|
|
||||||
// the manager has to cope with, or inspect one afterwards.
|
|
||||||
func openRawEventDB(
|
|
||||||
t *testing.T, dataDir, webhookID string,
|
|
||||||
) *sql.DB {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
path := filepath.Join(
|
|
||||||
dataDir, fmt.Sprintf("events-%s.db", webhookID),
|
|
||||||
)
|
|
||||||
|
|
||||||
sqlDB, err := sql.Open(
|
|
||||||
"sqlite",
|
|
||||||
fmt.Sprintf("file:%s?mode=rwc", path),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
|
||||||
|
|
||||||
return sqlDB
|
|
||||||
}
|
|
||||||
|
|
||||||
// eventDBFileBytes reads a per-webhook database file off disk, so a
|
|
||||||
// test can assert on what the file itself still holds rather than on
|
|
||||||
// what a query returns.
|
|
||||||
func eventDBFileBytes(t *testing.T, dataDir, webhookID string) []byte {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
//nolint:gosec // reads a file the test just created under t.TempDir()
|
|
||||||
raw, err := os.ReadFile(filepath.Join(
|
|
||||||
dataDir, fmt.Sprintf("events-%s.db", webhookID),
|
|
||||||
))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
return raw
|
|
||||||
}
|
|
||||||
|
|
||||||
// eventDBUserVersion returns the PRAGMA user_version of a per-webhook
|
|
||||||
// database file, which is the marker purgeTargetRows stamps once it
|
|
||||||
// has swept and vacuumed.
|
|
||||||
func eventDBUserVersion(t *testing.T, sqlDB *sql.DB) int {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var version int
|
|
||||||
|
|
||||||
require.NoError(t, sqlDB.QueryRowContext(
|
|
||||||
t.Context(), "PRAGMA user_version",
|
|
||||||
).Scan(&version))
|
|
||||||
|
|
||||||
return version
|
|
||||||
}
|
|
||||||
|
|
||||||
// clearEventDBSweptMarker resets the sweep marker to 0, which is what
|
|
||||||
// a file written by a build without the sweep looks like. Tests that
|
|
||||||
// seed a leaked row have to create the file through the manager to
|
|
||||||
// get the real targets table shape, and that stamps it.
|
|
||||||
func clearEventDBSweptMarker(t *testing.T, sqlDB *sql.DB) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
_, err := sqlDB.ExecContext(t.Context(), "PRAGMA user_version = 0")
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// countTargetRows returns the number of rows in the targets table of
|
|
||||||
// a per-webhook database file, or -1 if the table does not exist.
|
|
||||||
func countTargetRows(t *testing.T, sqlDB *sql.DB) int {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var tables int
|
|
||||||
|
|
||||||
require.NoError(t, sqlDB.QueryRowContext(
|
|
||||||
t.Context(),
|
|
||||||
"SELECT count(*) FROM sqlite_master "+
|
|
||||||
"WHERE type = 'table' AND name = 'targets'",
|
|
||||||
).Scan(&tables))
|
|
||||||
|
|
||||||
if tables == 0 {
|
|
||||||
return -1
|
|
||||||
}
|
|
||||||
|
|
||||||
var rows int
|
|
||||||
|
|
||||||
require.NoError(t, sqlDB.QueryRowContext(
|
|
||||||
t.Context(), "SELECT count(*) FROM targets",
|
|
||||||
).Scan(&rows))
|
|
||||||
|
|
||||||
return rows
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestOpenPurgesLeakedTargetRows covers the sweep for event
|
|
||||||
// databases written by a build that let GORM upsert target rows
|
|
||||||
// into them: opening the database clears them, and opening it again
|
|
||||||
// is a no-op.
|
|
||||||
func TestOpenPurgesLeakedTargetRows(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dataDir := eventDBDataDir(t)
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
|
|
||||||
// Create the file the way the application does, so the targets
|
|
||||||
// table has exactly the shape AutoMigrate gives it, then write
|
|
||||||
// a leaked row into it the way the association upsert did.
|
|
||||||
initial := database.NewTestWebhookDBManager(dataDir)
|
|
||||||
|
|
||||||
_, err := initial.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, initial.CloseAll())
|
|
||||||
|
|
||||||
seed := openRawEventDB(t, dataDir, webhookID)
|
|
||||||
|
|
||||||
_, err = seed.ExecContext(
|
|
||||||
t.Context(),
|
|
||||||
"INSERT INTO targets "+
|
|
||||||
"(id, webhook_id, name, type, config) "+
|
|
||||||
"VALUES (?, '', ?, ?, ?)",
|
|
||||||
uuid.New().String(),
|
|
||||||
"leaked-target",
|
|
||||||
"slack",
|
|
||||||
`{"webhookUrl":"https://hooks.example/T000/B000/secret"}`,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, 1, countTargetRows(t, seed))
|
|
||||||
clearEventDBSweptMarker(t, seed)
|
|
||||||
require.NoError(t, seed.Close())
|
|
||||||
|
|
||||||
mgr := database.NewTestWebhookDBManager(dataDir)
|
|
||||||
|
|
||||||
_, err = mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, mgr.CloseAll())
|
|
||||||
|
|
||||||
check := openRawEventDB(t, dataDir, webhookID)
|
|
||||||
assert.Zero(t, countTargetRows(t, check))
|
|
||||||
assert.Equal(
|
|
||||||
t, 1, eventDBUserVersion(t, check),
|
|
||||||
"a completed sweep must mark the file so later opens skip it",
|
|
||||||
)
|
|
||||||
require.NoError(t, check.Close())
|
|
||||||
|
|
||||||
// Idempotent: a second open leaves it at zero and does not
|
|
||||||
// error.
|
|
||||||
again := database.NewTestWebhookDBManager(dataDir)
|
|
||||||
|
|
||||||
_, err = again.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, again.CloseAll())
|
|
||||||
|
|
||||||
recheck := openRawEventDB(t, dataDir, webhookID)
|
|
||||||
assert.Zero(t, countTargetRows(t, recheck))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestOpenPurgeRemovesCredentialBytes covers the sweep at the level
|
|
||||||
// that matters for a backup handed to someone else: the leaked
|
|
||||||
// credential must be gone from the raw bytes of the file, not merely
|
|
||||||
// unreachable by query. A bare DELETE unlinks the row and leaves the
|
|
||||||
// bytes readable in the free pages, so this fails without the VACUUM
|
|
||||||
// in purgeTargetRows.
|
|
||||||
func TestOpenPurgeRemovesCredentialBytes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dataDir := eventDBDataDir(t)
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
credential := "T00000000/B00000000/" + uuid.New().String()
|
|
||||||
|
|
||||||
initial := database.NewTestWebhookDBManager(dataDir)
|
|
||||||
|
|
||||||
_, err := initial.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, initial.CloseAll())
|
|
||||||
|
|
||||||
seed := openRawEventDB(t, dataDir, webhookID)
|
|
||||||
|
|
||||||
_, err = seed.ExecContext(
|
|
||||||
t.Context(),
|
|
||||||
"INSERT INTO targets "+
|
|
||||||
"(id, webhook_id, name, type, config) "+
|
|
||||||
"VALUES (?, '', ?, ?, ?)",
|
|
||||||
uuid.New().String(),
|
|
||||||
"leaked-target",
|
|
||||||
"slack",
|
|
||||||
fmt.Sprintf(
|
|
||||||
`{"webhookUrl":"https://hooks.example/%s"}`, credential,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
clearEventDBSweptMarker(t, seed)
|
|
||||||
require.NoError(t, seed.Close())
|
|
||||||
|
|
||||||
// The seed has to be in the file for its absence later to mean
|
|
||||||
// anything.
|
|
||||||
require.True(
|
|
||||||
t,
|
|
||||||
bytes.Contains(
|
|
||||||
eventDBFileBytes(t, dataDir, webhookID),
|
|
||||||
[]byte(credential),
|
|
||||||
),
|
|
||||||
"seeded credential is not in the file, so this test proves nothing",
|
|
||||||
)
|
|
||||||
|
|
||||||
mgr := database.NewTestWebhookDBManager(dataDir)
|
|
||||||
|
|
||||||
_, err = mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, mgr.CloseAll())
|
|
||||||
|
|
||||||
assert.NotContains(
|
|
||||||
t,
|
|
||||||
string(eventDBFileBytes(t, dataDir, webhookID)),
|
|
||||||
credential,
|
|
||||||
"leaked credential is still recoverable from the raw file",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestOpenRevacuumsAfterIncompleteSweep covers the case a row count
|
|
||||||
// cannot see: the rows are already deleted but the file was never
|
|
||||||
// vacuumed, because an earlier sweep died between the two or its
|
|
||||||
// VACUUM failed. The credential bytes are still recoverable, and the
|
|
||||||
// unset marker is the only thing that says so, so the next open must
|
|
||||||
// vacuum rather than conclude from the empty table that there is
|
|
||||||
// nothing to do.
|
|
||||||
func TestOpenRevacuumsAfterIncompleteSweep(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dataDir := eventDBDataDir(t)
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
credential := "T00000000/B00000000/" + uuid.New().String()
|
|
||||||
|
|
||||||
initial := database.NewTestWebhookDBManager(dataDir)
|
|
||||||
|
|
||||||
_, err := initial.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, initial.CloseAll())
|
|
||||||
|
|
||||||
seed := openRawEventDB(t, dataDir, webhookID)
|
|
||||||
|
|
||||||
_, err = seed.ExecContext(
|
|
||||||
t.Context(),
|
|
||||||
"INSERT INTO targets "+
|
|
||||||
"(id, webhook_id, name, type, config) "+
|
|
||||||
"VALUES (?, '', ?, ?, ?)",
|
|
||||||
uuid.New().String(),
|
|
||||||
"leaked-target",
|
|
||||||
"slack",
|
|
||||||
fmt.Sprintf(
|
|
||||||
`{"webhookUrl":"https://hooks.example/%s"}`, credential,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Exactly the state an interrupted sweep leaves: rows gone,
|
|
||||||
// marker unset, bytes still in the free pages.
|
|
||||||
_, err = seed.ExecContext(t.Context(), "DELETE FROM targets")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Zero(t, countTargetRows(t, seed))
|
|
||||||
clearEventDBSweptMarker(t, seed)
|
|
||||||
require.NoError(t, seed.Close())
|
|
||||||
|
|
||||||
require.True(
|
|
||||||
t,
|
|
||||||
bytes.Contains(
|
|
||||||
eventDBFileBytes(t, dataDir, webhookID),
|
|
||||||
[]byte(credential),
|
|
||||||
),
|
|
||||||
"the deleted row's bytes must still be in the file, or this "+
|
|
||||||
"test proves nothing",
|
|
||||||
)
|
|
||||||
|
|
||||||
mgr := database.NewTestWebhookDBManager(dataDir)
|
|
||||||
|
|
||||||
_, err = mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, mgr.CloseAll())
|
|
||||||
|
|
||||||
assert.NotContains(
|
|
||||||
t,
|
|
||||||
string(eventDBFileBytes(t, dataDir, webhookID)),
|
|
||||||
credential,
|
|
||||||
"an interrupted sweep was not retried, so the credential is "+
|
|
||||||
"still recoverable from the raw file",
|
|
||||||
)
|
|
||||||
|
|
||||||
check := openRawEventDB(t, dataDir, webhookID)
|
|
||||||
assert.Equal(t, 1, eventDBUserVersion(t, check))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestOpenSkipsSweptDatabase covers the other half of the marker: a
|
|
||||||
// file this build created is marked without ever being vacuumed, and
|
|
||||||
// a marked file is not swept again.
|
|
||||||
func TestOpenSkipsSweptDatabase(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dataDir := eventDBDataDir(t)
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
|
|
||||||
mgr := database.NewTestWebhookDBManager(dataDir)
|
|
||||||
|
|
||||||
_, err := mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, mgr.CloseAll())
|
|
||||||
|
|
||||||
marked := openRawEventDB(t, dataDir, webhookID)
|
|
||||||
assert.Equal(t, 1, eventDBUserVersion(t, marked))
|
|
||||||
|
|
||||||
// A marked file is left alone, so a row written into it survives
|
|
||||||
// a reopen. Nothing writes target rows any more; this stands in
|
|
||||||
// for the sweep having run.
|
|
||||||
_, err = marked.ExecContext(
|
|
||||||
t.Context(),
|
|
||||||
"INSERT INTO targets "+
|
|
||||||
"(id, webhook_id, name, type, config) "+
|
|
||||||
"VALUES (?, '', ?, ?, ?)",
|
|
||||||
uuid.New().String(), "sentinel", "slack", `{}`,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, marked.Close())
|
|
||||||
|
|
||||||
again := database.NewTestWebhookDBManager(dataDir)
|
|
||||||
|
|
||||||
_, err = again.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, again.CloseAll())
|
|
||||||
|
|
||||||
check := openRawEventDB(t, dataDir, webhookID)
|
|
||||||
assert.Equal(
|
|
||||||
t, 1, countTargetRows(t, check),
|
|
||||||
"a marked file must not be swept again",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestOpenSucceedsWithoutTargetsTable covers an existing event
|
|
||||||
// database that never grew a targets table. The sweep must not fail
|
|
||||||
// startup on it.
|
|
||||||
func TestOpenSucceedsWithoutTargetsTable(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dataDir := eventDBDataDir(t)
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
|
|
||||||
seed := openRawEventDB(t, dataDir, webhookID)
|
|
||||||
|
|
||||||
_, err := seed.ExecContext(
|
|
||||||
t.Context(),
|
|
||||||
"CREATE TABLE events (id text PRIMARY KEY)",
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, seed.Close())
|
|
||||||
|
|
||||||
mgr := database.NewTestWebhookDBManager(dataDir)
|
|
||||||
|
|
||||||
db, err := mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.NotNil(t, db)
|
|
||||||
require.NoError(t, mgr.CloseAll())
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventDBCreateOmitsAssociations covers the connection-level
|
|
||||||
// guard directly: a Delivery carrying its Event and Target in
|
|
||||||
// memory, written through the manager's handle, must store only the
|
|
||||||
// delivery row.
|
|
||||||
func TestEventDBCreateOmitsAssociations(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dataDir := eventDBDataDir(t)
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
|
|
||||||
mgr := database.NewTestWebhookDBManager(dataDir)
|
|
||||||
|
|
||||||
db, err := mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
target := database.Target{
|
|
||||||
WebhookID: webhookID,
|
|
||||||
Name: "leaky-target",
|
|
||||||
Type: database.TargetTypeSlack,
|
|
||||||
Config: `{"webhookUrl":"https://hooks.example/secret"}`,
|
|
||||||
}
|
|
||||||
target.ID = uuid.New().String()
|
|
||||||
|
|
||||||
event := database.Event{
|
|
||||||
WebhookID: webhookID,
|
|
||||||
EntrypointID: uuid.New().String(),
|
|
||||||
Method: "POST",
|
|
||||||
Headers: `{}`,
|
|
||||||
Body: `{}`,
|
|
||||||
}
|
|
||||||
event.ID = uuid.New().String()
|
|
||||||
|
|
||||||
d := &database.Delivery{
|
|
||||||
EventID: event.ID,
|
|
||||||
TargetID: target.ID,
|
|
||||||
Status: database.DeliveryStatusPending,
|
|
||||||
Event: event,
|
|
||||||
Target: target,
|
|
||||||
}
|
|
||||||
d.ID = uuid.New().String()
|
|
||||||
|
|
||||||
require.NoError(t, db.Create(d).Error)
|
|
||||||
require.NoError(t, db.Model(d).
|
|
||||||
Update("status", database.DeliveryStatusDelivered).
|
|
||||||
Error)
|
|
||||||
require.NoError(t, mgr.CloseAll())
|
|
||||||
|
|
||||||
check := openRawEventDB(t, dataDir, webhookID)
|
|
||||||
assert.Zero(t, countTargetRows(t, check))
|
|
||||||
}
|
|
||||||
@@ -2,7 +2,6 @@ package database
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"io"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
@@ -47,35 +46,11 @@ func (r *RetentionReaper) ExportStart() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ExportStop stops the reaper's background loop for tests.
|
// ExportStop stops the reaper's background loop for tests.
|
||||||
func (r *RetentionReaper) ExportStop(ctx context.Context) error {
|
func (r *RetentionReaper) ExportStop() {
|
||||||
return r.stop(ctx)
|
r.stop()
|
||||||
}
|
|
||||||
|
|
||||||
// ExportWedgeLoop adds a goroutine to the reaper's WaitGroup that
|
|
||||||
// never observes cancellation and returns only when release is
|
|
||||||
// closed. It stands in for a sweep stuck on a locked database.
|
|
||||||
func (r *RetentionReaper) ExportWedgeLoop(
|
|
||||||
release <-chan struct{},
|
|
||||||
) {
|
|
||||||
r.wg.Go(func() {
|
|
||||||
<-release
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportSetInterval overrides the sweep interval for tests.
|
// ExportSetInterval overrides the sweep interval for tests.
|
||||||
func (r *RetentionReaper) ExportSetInterval(d time.Duration) {
|
func (r *RetentionReaper) ExportSetInterval(d time.Duration) {
|
||||||
r.interval = d
|
r.interval = d
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportSetBannerOut redirects the first-boot credentials banner, so a
|
|
||||||
// test can read what the operator would have seen. It must be called
|
|
||||||
// before the fx start hook runs, which is where the account is seeded.
|
|
||||||
func (d *Database) ExportSetBannerOut(w io.Writer) {
|
|
||||||
d.bannerOut = w
|
|
||||||
}
|
|
||||||
|
|
||||||
// DummyPasswordHashForTest exposes the encoded hash that unknown
|
|
||||||
// usernames are verified against.
|
|
||||||
func DummyPasswordHashForTest() string {
|
|
||||||
return dummyPasswordHash()
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,85 +0,0 @@
|
|||||||
package database_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/signature"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestEntrypointSignatureColumnsMigrateToUnconfigured pins the
|
|
||||||
// upgrade path for a deployment that already has entrypoints.
|
|
||||||
//
|
|
||||||
// The signature columns arrive through GORM's AutoMigrate, so every
|
|
||||||
// row written before they existed acquires them with no value. That
|
|
||||||
// has to land on "not configured", because the alternative is an
|
|
||||||
// upgrade that rejects the traffic the operator was already
|
|
||||||
// receiving — a self-inflicted outage on a receiver whose senders
|
|
||||||
// cannot be told to start signing.
|
|
||||||
//
|
|
||||||
// The legacy schema is reproduced by dropping the columns from a
|
|
||||||
// migrated database and writing a row through the old shape, so the
|
|
||||||
// row really predates them rather than merely being blank.
|
|
||||||
func TestEntrypointSignatureColumnsMigrateToUnconfigured(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db, lc := setupTestDB(t)
|
|
||||||
lc.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(lc.RequireStop)
|
|
||||||
|
|
||||||
for _, column := range []string{
|
|
||||||
"signature_scheme", "signature_secret",
|
|
||||||
} {
|
|
||||||
require.NoError(
|
|
||||||
t,
|
|
||||||
db.DB().Exec(
|
|
||||||
"ALTER TABLE entrypoints DROP COLUMN "+column,
|
|
||||||
).Error,
|
|
||||||
"dropping %s to reproduce the pre-upgrade schema",
|
|
||||||
column,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
const legacyID = "legacy-entrypoint"
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t,
|
|
||||||
db.DB().Exec(
|
|
||||||
`INSERT INTO entrypoints
|
|
||||||
(id, created_at, updated_at, webhook_id, path,
|
|
||||||
description, active)
|
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
||||||
legacyID, "2026-01-01 00:00:00", "2026-01-01 00:00:00",
|
|
||||||
"legacy-webhook", "legacy-path", "predates signatures",
|
|
||||||
true,
|
|
||||||
).Error,
|
|
||||||
)
|
|
||||||
|
|
||||||
// The upgrade.
|
|
||||||
require.NoError(t, db.Migrate())
|
|
||||||
|
|
||||||
var ep database.Entrypoint
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t,
|
|
||||||
db.DB().Where("id = ?", legacyID).First(&ep).Error,
|
|
||||||
"the migrated row must still load; a NULL landing in a "+
|
|
||||||
"string column would fail here",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Equal(t, database.SignatureSchemeNone, ep.SignatureScheme)
|
|
||||||
assert.Empty(t, ep.SignatureSecret)
|
|
||||||
assert.False(t, ep.SignatureConfigured())
|
|
||||||
assert.True(t, ep.Active, "the row's other columns survive")
|
|
||||||
|
|
||||||
// The behaviour that actually matters: an unsigned request to
|
|
||||||
// this entrypoint is still accepted.
|
|
||||||
assert.NoError(
|
|
||||||
t,
|
|
||||||
signature.Verify(&ep, http.Header{}, []byte(`{"a":1}`)),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -2,16 +2,12 @@ package database
|
|||||||
|
|
||||||
import "time"
|
import "time"
|
||||||
|
|
||||||
// APIKey represents an API key for a user.
|
// APIKey represents an API key for a user
|
||||||
//
|
|
||||||
// Key is a bearer credential, so it is never marshalled with the
|
|
||||||
// model. A creation handler that has to show it once returns it in its
|
|
||||||
// own response type.
|
|
||||||
type APIKey struct {
|
type APIKey struct {
|
||||||
BaseModel
|
BaseModel
|
||||||
|
|
||||||
UserID string `gorm:"type:uuid;not null" json:"userId"`
|
UserID string `gorm:"type:uuid;not null" json:"userId"`
|
||||||
Key string `gorm:"uniqueIndex;not null" json:"-"`
|
Key string `gorm:"uniqueIndex;not null" json:"key"`
|
||||||
Description string `json:"description"`
|
Description string `json:"description"`
|
||||||
LastUsedAt *time.Time `json:"lastUsedAt,omitempty"`
|
LastUsedAt *time.Time `json:"lastUsedAt,omitempty"`
|
||||||
|
|
||||||
|
|||||||
@@ -11,23 +11,6 @@ const (
|
|||||||
DeliveryStatusRetrying DeliveryStatus = "retrying"
|
DeliveryStatusRetrying DeliveryStatus = "retrying"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Terminal reports whether a delivery in this status has finished, so
|
|
||||||
// the delivery engine will make no further attempt of its own.
|
|
||||||
//
|
|
||||||
// It is what decides which deliveries the event log offers to replay:
|
|
||||||
// a pending or retrying delivery is still the engine's, and replaying
|
|
||||||
// one would race it.
|
|
||||||
func (s DeliveryStatus) Terminal() bool {
|
|
||||||
switch s {
|
|
||||||
case DeliveryStatusDelivered, DeliveryStatusFailed:
|
|
||||||
return true
|
|
||||||
case DeliveryStatusPending, DeliveryStatusRetrying:
|
|
||||||
return false
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Delivery represents a delivery attempt for an event to a target
|
// Delivery represents a delivery attempt for an event to a target
|
||||||
type Delivery struct {
|
type Delivery struct {
|
||||||
BaseModel
|
BaseModel
|
||||||
|
|||||||
@@ -1,22 +1,5 @@
|
|||||||
package database
|
package database
|
||||||
|
|
||||||
// SignatureScheme names the way an entrypoint authenticates inbound
|
|
||||||
// requests. A scheme fixes both the header the signature arrives in
|
|
||||||
// and the algorithm used to check it, so an operator cannot pair one
|
|
||||||
// sender's header with another sender's comparison.
|
|
||||||
type SignatureScheme string
|
|
||||||
|
|
||||||
// Signature scheme values. The empty scheme means the entrypoint
|
|
||||||
// performs no inbound verification: it is the default, and it is the
|
|
||||||
// state every entrypoint created before this column existed migrates
|
|
||||||
// to, so an existing deployment keeps accepting the requests it
|
|
||||||
// accepted before.
|
|
||||||
const (
|
|
||||||
SignatureSchemeNone SignatureScheme = ""
|
|
||||||
SignatureSchemeGitHub SignatureScheme = "github"
|
|
||||||
SignatureSchemeGitLab SignatureScheme = "gitlab"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Entrypoint represents an inbound URL endpoint that feeds into a webhook
|
// Entrypoint represents an inbound URL endpoint that feeds into a webhook
|
||||||
type Entrypoint struct {
|
type Entrypoint struct {
|
||||||
BaseModel
|
BaseModel
|
||||||
@@ -29,43 +12,6 @@ type Entrypoint struct {
|
|||||||
Description string `json:"description"`
|
Description string `json:"description"`
|
||||||
Active bool `gorm:"default:true" json:"active"`
|
Active bool `gorm:"default:true" json:"active"`
|
||||||
|
|
||||||
// SignatureScheme selects how inbound requests to this
|
|
||||||
// entrypoint are authenticated. Empty means unauthenticated,
|
|
||||||
// which is what a UUID-only entrypoint has always been.
|
|
||||||
SignatureScheme SignatureScheme `gorm:"default:''" json:"signatureScheme"`
|
|
||||||
|
|
||||||
// SignatureSecret is the secret shared with the sender.
|
|
||||||
//
|
|
||||||
// It is stored in the clear because HMAC verification needs the
|
|
||||||
// key itself: a hash of it cannot recompute the sender's digest.
|
|
||||||
// It is therefore a live credential, and json:"-" keeps it out of
|
|
||||||
// any handler that marshals the model, the way APIKey.Key and
|
|
||||||
// Target.Config are kept out. handlers.EntrypointView is the
|
|
||||||
// matching barrier for the HTML path.
|
|
||||||
SignatureSecret string `gorm:"default:''" json:"-"`
|
|
||||||
|
|
||||||
// Relations
|
// Relations
|
||||||
Webhook Webhook `json:"webhook,omitzero"`
|
Webhook Webhook `json:"webhook,omitzero"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// SignatureConfigured reports whether this entrypoint verifies
|
|
||||||
// inbound requests. Both halves must be present: a scheme without a
|
|
||||||
// secret, or a secret without a scheme, is a broken configuration
|
|
||||||
// rather than a configured one, and signature.Verify fails those
|
|
||||||
// closed rather than treating them as "off".
|
|
||||||
func (e *Entrypoint) SignatureConfigured() bool {
|
|
||||||
return e.SignatureScheme != SignatureSchemeNone &&
|
|
||||||
e.SignatureSecret != ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// SignatureHalfConfigured reports whether exactly one half of the
|
|
||||||
// scheme/secret pair is present. The receiver refuses such a row on
|
|
||||||
// every request, so the UI must not describe it as unverified. It
|
|
||||||
// reports the state without exposing the secret, which is why it
|
|
||||||
// lives here rather than in the display projection.
|
|
||||||
func (e *Entrypoint) SignatureHalfConfigured() bool {
|
|
||||||
hasScheme := e.SignatureScheme != SignatureSchemeNone
|
|
||||||
hasSecret := e.SignatureSecret != ""
|
|
||||||
|
|
||||||
return hasScheme != hasSecret
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,138 +0,0 @@
|
|||||||
package database_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// keptField is a non-secret value planted alongside each secret, so
|
|
||||||
// the assertions below cannot pass by the model marshalling to nothing.
|
|
||||||
const keptField = "keepme"
|
|
||||||
|
|
||||||
// marshalModel encodes a model the way a future JSON handler would.
|
|
||||||
func marshalModel(t *testing.T, v any) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
encoded, err := json.Marshal(v)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
return string(encoded)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestModelsDoNotMarshalTheirSecrets pins the barrier for the JSON
|
|
||||||
// path. The /api/v1 route group exists and is empty; delivery's
|
|
||||||
// TargetView masks the credential for the HTML path only, so without
|
|
||||||
// these tags the first handler that marshals a model serialises the
|
|
||||||
// secret with it. Each field below is a live credential:
|
|
||||||
//
|
|
||||||
// - Target.Config holds an incoming-webhook URL whose path segments
|
|
||||||
// are the bearer token.
|
|
||||||
// - APIKey.Key is a bearer token outright.
|
|
||||||
// - Setting.Value holds the session encryption key.
|
|
||||||
// - User.Password holds the Argon2 hash, and was already tagged.
|
|
||||||
// - Entrypoint.SignatureSecret is the secret its senders sign with,
|
|
||||||
// stored in the clear because HMAC verification needs the key.
|
|
||||||
func TestModelsDoNotMarshalTheirSecrets(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const marker = "QQMODELMARKERQQ"
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
model any
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "target config",
|
|
||||||
model: database.Target{
|
|
||||||
Name: keptField,
|
|
||||||
Type: database.TargetTypeSlack,
|
|
||||||
Config: `{"webhookUrl":"https://h/s/` + marker + `"}`,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "api key",
|
|
||||||
model: database.APIKey{
|
|
||||||
Description: keptField,
|
|
||||||
Key: marker,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "setting value",
|
|
||||||
model: database.Setting{
|
|
||||||
Key: keptField,
|
|
||||||
Value: marker,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "user password hash",
|
|
||||||
model: database.User{
|
|
||||||
Username: keptField,
|
|
||||||
Password: marker,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "entrypoint signature secret",
|
|
||||||
model: database.Entrypoint{
|
|
||||||
Description: keptField,
|
|
||||||
SignatureScheme: database.SignatureSchemeGitHub,
|
|
||||||
SignatureSecret: marker,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
encoded := marshalModel(t, tc.model)
|
|
||||||
|
|
||||||
assert.NotContains(t, encoded, marker)
|
|
||||||
assert.Contains(t, encoded, keptField)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWebhookMarshalsNoTargetConfig covers the nested case: a webhook
|
|
||||||
// marshalled with its targets preloaded must not carry the credential
|
|
||||||
// through the association either.
|
|
||||||
func TestWebhookMarshalsNoTargetConfig(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const marker = "QQNESTEDMARKERQQ"
|
|
||||||
|
|
||||||
encoded := marshalModel(t, database.Webhook{
|
|
||||||
Name: keptField,
|
|
||||||
Targets: []database.Target{{
|
|
||||||
Name: "slack",
|
|
||||||
Config: `{"webhookUrl":"https://h/s/` + marker + `"}`,
|
|
||||||
}},
|
|
||||||
})
|
|
||||||
|
|
||||||
assert.NotContains(t, encoded, marker)
|
|
||||||
assert.Contains(t, encoded, keptField)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWebhookMarshalsNoEntrypointSecret covers the same nested case
|
|
||||||
// for the entrypoint's inbound signature secret, which reaches a
|
|
||||||
// marshalled webhook through the Entrypoints association.
|
|
||||||
func TestWebhookMarshalsNoEntrypointSecret(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const marker = "QQENTRYPOINTMARKERQQ"
|
|
||||||
|
|
||||||
encoded := marshalModel(t, database.Webhook{
|
|
||||||
Name: keptField,
|
|
||||||
Entrypoints: []database.Entrypoint{{
|
|
||||||
Path: "some-uuid",
|
|
||||||
SignatureScheme: database.SignatureSchemeGitLab,
|
|
||||||
SignatureSecret: marker,
|
|
||||||
}},
|
|
||||||
})
|
|
||||||
|
|
||||||
assert.NotContains(t, encoded, marker)
|
|
||||||
assert.Contains(t, encoded, keptField)
|
|
||||||
}
|
|
||||||
@@ -4,8 +4,5 @@ package database
|
|||||||
// Used for auto-generated values like the session encryption key.
|
// Used for auto-generated values like the session encryption key.
|
||||||
type Setting struct {
|
type Setting struct {
|
||||||
Key string `gorm:"primaryKey" json:"key"`
|
Key string `gorm:"primaryKey" json:"key"`
|
||||||
|
Value string `gorm:"type:text;not null" json:"value"`
|
||||||
// Value holds the session encryption key, so it is never
|
|
||||||
// marshalled with the model.
|
|
||||||
Value string `gorm:"type:text;not null" json:"-"`
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,14 +20,8 @@ type Target struct {
|
|||||||
Type TargetType `gorm:"not null" json:"type"`
|
Type TargetType `gorm:"not null" json:"type"`
|
||||||
Active bool `gorm:"default:true" json:"active"`
|
Active bool `gorm:"default:true" json:"active"`
|
||||||
|
|
||||||
// Configuration fields (JSON stored based on type).
|
// Configuration fields (JSON stored based on type)
|
||||||
//
|
Config string `gorm:"type:text" json:"config"` // JSON configuration
|
||||||
// json:"-" because the blob holds the target's credential — a
|
|
||||||
// Slack incoming-webhook URL, or an http destination whose path
|
|
||||||
// segments are the secret. delivery.TargetView is the masking
|
|
||||||
// barrier for the HTML path; this tag is the barrier for any
|
|
||||||
// handler that marshals the model itself.
|
|
||||||
Config string `gorm:"type:text" json:"-"` // JSON configuration
|
|
||||||
|
|
||||||
// For HTTP targets (max_retries=0 means fire-and-forget,
|
// For HTTP targets (max_retries=0 means fire-and-forget,
|
||||||
// >0 enables retries with backoff)
|
// >0 enables retries with backoff)
|
||||||
|
|||||||
@@ -1,59 +1,6 @@
|
|||||||
package database
|
package database
|
||||||
|
|
||||||
import (
|
|
||||||
"math"
|
|
||||||
"strconv"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gorm.io/gorm"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// DefaultRetentionDays is the event retention period applied to a
|
|
||||||
// webhook created without an explicit retention value. It is the
|
|
||||||
// single source of truth for that policy and must stay in sync
|
|
||||||
// with the `gorm:"default:30"` column default on
|
|
||||||
// Webhook.RetentionDays below; a struct tag cannot reference a
|
|
||||||
// constant, so a test asserts the two agree.
|
|
||||||
DefaultRetentionDays = 30
|
|
||||||
|
|
||||||
// RetentionForeverDays is the sentinel RetentionDays value meaning
|
|
||||||
// "retain events forever". Users express that intent as 0, which
|
|
||||||
// Webhook.BeforeSave rewrites to this value: the column default
|
|
||||||
// substitutes DefaultRetentionDays for a zero value at insert
|
|
||||||
// time, so a zero can never survive a round trip to the database.
|
|
||||||
// Nothing outside this file may hardcode the number.
|
|
||||||
RetentionForeverDays = 365 * 1000
|
|
||||||
|
|
||||||
// MaxFiniteRetentionDays is the largest finite retention period the
|
|
||||||
// reaper's cutoff arithmetic can represent, and therefore the
|
|
||||||
// largest one a caller may request. It is derived from that
|
|
||||||
// arithmetic rather than picked: retentionCutoff computes
|
|
||||||
// retentionDays * hoursPerDay * time.Hour, and a time.Duration is
|
|
||||||
// an int64 nanosecond count, so math.MaxInt64 nanoseconds divided
|
|
||||||
// by an hour and then by a day is the exact ceiling — 106751 days,
|
|
||||||
// a little over 292 years.
|
|
||||||
//
|
|
||||||
// One day more overflows int64, wraps the product negative, and
|
|
||||||
// turns the cutoff into a timestamp in the far future that matches
|
|
||||||
// every row in the webhook's database. That is why this bound is
|
|
||||||
// enforced on input and why retentionCutoff saturates underneath
|
|
||||||
// it. Note that RetentionForeverDays deliberately sits above this
|
|
||||||
// ceiling: such webhooks are skipped before any cutoff is
|
|
||||||
// computed, and never reach the arithmetic at all.
|
|
||||||
MaxFiniteRetentionDays = int(
|
|
||||||
math.MaxInt64 / int64(time.Hour) / hoursPerDay,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
// Webhook represents a webhook processing unit that groups entrypoints and targets
|
// Webhook represents a webhook processing unit that groups entrypoints and targets
|
||||||
//
|
|
||||||
// Every method below takes a pointer receiver. BeforeSave has to,
|
|
||||||
// because it mutates the record and GORM only invokes hooks declared
|
|
||||||
// that way; the display helpers follow suit so the receiver kinds do
|
|
||||||
// not mix. Handlers therefore put a *Webhook into template data:
|
|
||||||
// html/template cannot call a pointer method on a value held in a map,
|
|
||||||
// because a map element is not addressable.
|
|
||||||
type Webhook struct {
|
type Webhook struct {
|
||||||
BaseModel
|
BaseModel
|
||||||
|
|
||||||
@@ -61,9 +8,7 @@ type Webhook struct {
|
|||||||
Name string `gorm:"not null" json:"name"`
|
Name string `gorm:"not null" json:"name"`
|
||||||
Description string `json:"description"`
|
Description string `json:"description"`
|
||||||
|
|
||||||
// RetentionDays is the number of days to retain events. A value of
|
// RetentionDays is the number of days to retain events.
|
||||||
// RetentionForeverDays means retain forever. The column default
|
|
||||||
// must equal DefaultRetentionDays.
|
|
||||||
RetentionDays int `gorm:"default:30" json:"retentionDays"`
|
RetentionDays int `gorm:"default:30" json:"retentionDays"`
|
||||||
|
|
||||||
// Relations
|
// Relations
|
||||||
@@ -71,55 +16,3 @@ type Webhook struct {
|
|||||||
Entrypoints []Entrypoint `json:"entrypoints,omitempty"`
|
Entrypoints []Entrypoint `json:"entrypoints,omitempty"`
|
||||||
Targets []Target `json:"targets,omitempty"`
|
Targets []Target `json:"targets,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// BeforeSave normalises RetentionDays on every insert and update. A
|
|
||||||
// non-positive value is the user's way of asking for "retain forever",
|
|
||||||
// which is stored as the RetentionForeverDays sentinel.
|
|
||||||
//
|
|
||||||
// This has to happen in a hook rather than at the call sites. GORM
|
|
||||||
// substitutes the column default (DefaultRetentionDays) for a zero
|
|
||||||
// value while building the insert statement, which runs after
|
|
||||||
// BeforeSave; rewriting any later than this loses that race and the
|
|
||||||
// row lands at 30 days. Living on the model also means a future call
|
|
||||||
// site — a REST API, a fixture, a migration — cannot bypass it.
|
|
||||||
func (w *Webhook) BeforeSave(_ *gorm.DB) error {
|
|
||||||
if w.RetentionDays <= 0 {
|
|
||||||
w.RetentionDays = RetentionForeverDays
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// retainsForever reports whether a stored RetentionDays value means
|
|
||||||
// "keep events indefinitely". It is the single definition of that
|
|
||||||
// question, shared by Webhook.RetainsForever and by the reaper's
|
|
||||||
// cutoff computation so the two cannot disagree about which webhooks
|
|
||||||
// are exempt from reaping.
|
|
||||||
//
|
|
||||||
// It accepts the RetentionForeverDays sentinel written by BeforeSave
|
|
||||||
// and, defensively, the non-positive values that rows written before
|
|
||||||
// the sentinel existed may still carry.
|
|
||||||
func retainsForever(retentionDays int) bool {
|
|
||||||
return retentionDays <= 0 ||
|
|
||||||
retentionDays >= RetentionForeverDays
|
|
||||||
}
|
|
||||||
|
|
||||||
// RetainsForever reports whether this webhook's events are kept
|
|
||||||
// indefinitely.
|
|
||||||
func (w *Webhook) RetainsForever() bool {
|
|
||||||
return retainsForever(w.RetentionDays)
|
|
||||||
}
|
|
||||||
|
|
||||||
// RetentionLabel returns the webhook's retention policy as display
|
|
||||||
// text, so that no template has to know about the sentinel value.
|
|
||||||
func (w *Webhook) RetentionLabel() string {
|
|
||||||
if w.RetainsForever() {
|
|
||||||
return "forever"
|
|
||||||
}
|
|
||||||
|
|
||||||
if w.RetentionDays == 1 {
|
|
||||||
return "1 day"
|
|
||||||
}
|
|
||||||
|
|
||||||
return strconv.Itoa(w.RetentionDays) + " days"
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,222 +0,0 @@
|
|||||||
package database_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"reflect"
|
|
||||||
"strconv"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"gorm.io/gorm/clause"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// startedTestDB returns a started main database for model-level tests.
|
|
||||||
func startedTestDB(t *testing.T) *gorm.DB {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
db, lc := setupTestDB(t)
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
require.NoError(t, lc.Start(ctx))
|
|
||||||
t.Cleanup(func() { require.NoError(t, lc.Stop(ctx)) })
|
|
||||||
|
|
||||||
return db.DB()
|
|
||||||
}
|
|
||||||
|
|
||||||
// storedRetention reads the retention_days column straight out of the
|
|
||||||
// row, so the assertion is about what was persisted rather than about
|
|
||||||
// whatever the in-memory struct happens to hold.
|
|
||||||
func storedRetention(t *testing.T, db *gorm.DB, id string) int {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var got int
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t,
|
|
||||||
db.Model(&database.Webhook{}).
|
|
||||||
Where("id = ?", id).
|
|
||||||
Pluck("retention_days", &got).Error,
|
|
||||||
)
|
|
||||||
|
|
||||||
return got
|
|
||||||
}
|
|
||||||
|
|
||||||
// newWebhookWithRetention creates a webhook through the ordinary Create
|
|
||||||
// path, so the BeforeSave hook and the GORM column default both apply
|
|
||||||
// exactly as they do in production.
|
|
||||||
func newWebhookWithRetention(
|
|
||||||
t *testing.T,
|
|
||||||
db *gorm.DB,
|
|
||||||
wh *database.Webhook,
|
|
||||||
) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
wh.UserID = uuid.New().String()
|
|
||||||
wh.Name = testWebhookName
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t,
|
|
||||||
db.Omit(clause.Associations).Create(wh).Error,
|
|
||||||
)
|
|
||||||
|
|
||||||
return wh.ID
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWebhookBeforeSave_ZeroBecomesForeverSentinel(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := startedTestDB(t)
|
|
||||||
|
|
||||||
wh := &database.Webhook{RetentionDays: 0}
|
|
||||||
id := newWebhookWithRetention(t, db, wh)
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
database.RetentionForeverDays,
|
|
||||||
storedRetention(t, db, id),
|
|
||||||
"a zero retention must be stored as the sentinel, "+
|
|
||||||
"not replaced by the column default",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWebhookBeforeSave_NegativeBecomesForeverSentinel(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := startedTestDB(t)
|
|
||||||
|
|
||||||
wh := &database.Webhook{RetentionDays: -5}
|
|
||||||
id := newWebhookWithRetention(t, db, wh)
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
database.RetentionForeverDays,
|
|
||||||
storedRetention(t, db, id),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWebhookBeforeSave_PositiveIsPreserved(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := startedTestDB(t)
|
|
||||||
|
|
||||||
wh := &database.Webhook{RetentionDays: 7}
|
|
||||||
id := newWebhookWithRetention(t, db, wh)
|
|
||||||
|
|
||||||
assert.Equal(t, 7, storedRetention(t, db, id))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWebhookBeforeSave_UpdateToZeroBecomesSentinel proves the hook
|
|
||||||
// fires on update as well as insert, via the same Save call the edit
|
|
||||||
// handler makes.
|
|
||||||
func TestWebhookBeforeSave_UpdateToZeroBecomesSentinel(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := startedTestDB(t)
|
|
||||||
|
|
||||||
wh := &database.Webhook{RetentionDays: 30}
|
|
||||||
id := newWebhookWithRetention(t, db, wh)
|
|
||||||
require.Equal(t, 30, storedRetention(t, db, id))
|
|
||||||
|
|
||||||
wh.RetentionDays = 0
|
|
||||||
require.NoError(t, db.Omit(clause.Associations).Save(wh).Error)
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
database.RetentionForeverDays,
|
|
||||||
storedRetention(t, db, id),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWebhookRetentionColumnDefaultMatchesConstant guards the one place
|
|
||||||
// the default lives twice: a struct tag cannot reference a constant, so
|
|
||||||
// this asserts the tag and DefaultRetentionDays agree.
|
|
||||||
func TestWebhookRetentionColumnDefaultMatchesConstant(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
field, ok := reflect.TypeFor[database.Webhook]().
|
|
||||||
FieldByName("RetentionDays")
|
|
||||||
require.True(t, ok, "Webhook.RetentionDays must exist")
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
"default:"+strconv.Itoa(database.DefaultRetentionDays),
|
|
||||||
field.Tag.Get("gorm"),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestMaxFiniteRetentionDaysIsTheOverflowCeiling asserts that the
|
|
||||||
// constant is exactly where the cutoff arithmetic stops working, which
|
|
||||||
// is what makes it a derived bound rather than a round number someone
|
|
||||||
// liked. One day more wraps the int64 nanosecond count negative, and a
|
|
||||||
// negative span is precisely what turned a cutoff into a future
|
|
||||||
// timestamp that matched — and deleted — every row.
|
|
||||||
//
|
|
||||||
// The multiplications are done through variables on purpose: as
|
|
||||||
// constant expressions the overflowing one would not compile.
|
|
||||||
func TestMaxFiniteRetentionDaysIsTheOverflowCeiling(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const hoursPerDay = 24
|
|
||||||
|
|
||||||
atCeiling := database.MaxFiniteRetentionDays
|
|
||||||
overCeiling := database.MaxFiniteRetentionDays + 1
|
|
||||||
|
|
||||||
assert.Positive(
|
|
||||||
t,
|
|
||||||
time.Duration(atCeiling*hoursPerDay)*time.Hour,
|
|
||||||
"the ceiling itself must still be representable",
|
|
||||||
)
|
|
||||||
assert.Negative(
|
|
||||||
t,
|
|
||||||
time.Duration(overCeiling*hoursPerDay)*time.Hour,
|
|
||||||
"one day past the ceiling must overflow",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Less(
|
|
||||||
t,
|
|
||||||
database.MaxFiniteRetentionDays,
|
|
||||||
database.RetentionForeverDays,
|
|
||||||
"the sentinel sits above the ceiling and is only safe "+
|
|
||||||
"because retain-forever webhooks skip the arithmetic",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWebhookRetainsForeverAndLabel(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
days int
|
|
||||||
forever bool
|
|
||||||
label string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
"sentinel",
|
|
||||||
database.RetentionForeverDays, true, testForeverLabel,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"above sentinel",
|
|
||||||
database.RetentionForeverDays + 1, true, testForeverLabel,
|
|
||||||
},
|
|
||||||
{"legacy zero", 0, true, testForeverLabel},
|
|
||||||
{"legacy negative", -1, true, testForeverLabel},
|
|
||||||
{"default", database.DefaultRetentionDays, false, "30 days"},
|
|
||||||
{"one day", 1, false, "1 day"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
wh := database.Webhook{RetentionDays: tc.days}
|
|
||||||
|
|
||||||
assert.Equal(t, tc.forever, wh.RetainsForever())
|
|
||||||
assert.Equal(t, tc.label, wh.RetentionLabel())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -8,7 +8,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"math/big"
|
"math/big"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
|
|
||||||
"golang.org/x/crypto/argon2"
|
"golang.org/x/crypto/argon2"
|
||||||
)
|
)
|
||||||
@@ -30,10 +29,6 @@ const hashParts = 6
|
|||||||
// triggers per-character-class complexity enforcement.
|
// triggers per-character-class complexity enforcement.
|
||||||
const minPasswordComplexityLen = 4
|
const minPasswordComplexityLen = 4
|
||||||
|
|
||||||
// dummyPasswordLen is the length of the throwaway password behind
|
|
||||||
// dummyPasswordHash.
|
|
||||||
const dummyPasswordLen = 32
|
|
||||||
|
|
||||||
// Sentinel errors returned by decodeHash.
|
// Sentinel errors returned by decodeHash.
|
||||||
var (
|
var (
|
||||||
errInvalidHashFormat = errors.New("invalid hash format")
|
errInvalidHashFormat = errors.New("invalid hash format")
|
||||||
@@ -127,38 +122,6 @@ func VerifyPassword(
|
|||||||
return subtle.ConstantTimeCompare(hash, otherHash) == 1, nil
|
return subtle.ConstantTimeCompare(hash, otherHash) == 1, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// dummyPasswordHash is an encoded Argon2id hash of a random
|
|
||||||
// password, computed once on first use. Nothing can match it: the
|
|
||||||
// password it encodes is discarded as soon as it is hashed. It is
|
|
||||||
// process-wide because building it per request would add a second
|
|
||||||
// 64 MB Argon2id pass to every login for an unknown username.
|
|
||||||
//
|
|
||||||
//nolint:gochecknoglobals // computed once, see above
|
|
||||||
var dummyPasswordHash = sync.OnceValue(func() string {
|
|
||||||
password, err := GenerateRandomPassword(dummyPasswordLen)
|
|
||||||
if err != nil {
|
|
||||||
panic(fmt.Sprintf("generating the dummy password: %v", err))
|
|
||||||
}
|
|
||||||
|
|
||||||
hash, err := HashPassword(password)
|
|
||||||
if err != nil {
|
|
||||||
panic(fmt.Sprintf("hashing the dummy password: %v", err))
|
|
||||||
}
|
|
||||||
|
|
||||||
return hash
|
|
||||||
})
|
|
||||||
|
|
||||||
// VerifyDummyPassword performs a credential verification that cannot
|
|
||||||
// succeed, at the same cost as a real one.
|
|
||||||
//
|
|
||||||
// Login must charge an unknown username the same work as a known
|
|
||||||
// one. Returning early for an account that does not exist answers in
|
|
||||||
// microseconds where a real account takes tens of milliseconds, which
|
|
||||||
// is a username oracle any client can read off the response time.
|
|
||||||
func VerifyDummyPassword(password string) {
|
|
||||||
_, _ = VerifyPassword(password, dummyPasswordHash())
|
|
||||||
}
|
|
||||||
|
|
||||||
// decodeHash extracts parameters, salt, and hash from an
|
// decodeHash extracts parameters, salt, and hash from an
|
||||||
// encoded hash string.
|
// encoded hash string.
|
||||||
func decodeHash(
|
func decodeHash(
|
||||||
|
|||||||
@@ -191,41 +191,3 @@ func TestHashPasswordUniqueness(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestVerifyDummyPassword_DoesRealWork covers the anti-enumeration
|
|
||||||
// path. Login charges an unknown username a verification against a
|
|
||||||
// dummy hash so that a nonexistent account is not answered in
|
|
||||||
// microseconds where a real one takes tens of milliseconds. That only
|
|
||||||
// works if the dummy hash is a real, decodable Argon2id hash: a
|
|
||||||
// malformed one would make VerifyPassword fail on the decode and
|
|
||||||
// return before hashing anything.
|
|
||||||
func TestVerifyDummyPassword_DoesRealWork(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Runs the OnceValue that builds the dummy hash, so a panic in
|
|
||||||
// it surfaces here rather than on a live login.
|
|
||||||
database.VerifyDummyPassword("whatever was submitted")
|
|
||||||
|
|
||||||
dummy := database.DummyPasswordHashForTest()
|
|
||||||
|
|
||||||
// A hash the verifier cannot decode would make VerifyPassword
|
|
||||||
// return on the decode error, before hashing anything — the
|
|
||||||
// timing oracle this path exists to close.
|
|
||||||
valid, err := database.VerifyPassword("whatever", dummy)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf(
|
|
||||||
"the dummy hash must decode like a real one: %v", err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if valid {
|
|
||||||
t.Error("nothing may authenticate against the dummy hash")
|
|
||||||
}
|
|
||||||
|
|
||||||
if !strings.HasPrefix(dummy, "$argon2id$") {
|
|
||||||
t.Errorf(
|
|
||||||
"the dummy hash must use the same algorithm as real "+
|
|
||||||
"hashes, got %q", dummy,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ import (
|
|||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/lifecycle"
|
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -63,9 +62,8 @@ func NewRetentionReaper(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// registerHooks wires the reaper's start and stop into the fx
|
// registerHooks wires the reaper's start and stop into the fx
|
||||||
// lifecycle. The start hook's context is deliberately ignored (see
|
// lifecycle. The start hook's context is deliberately ignored: see
|
||||||
// start for why the sweep loop must not inherit it); the stop hook's
|
// start for why the sweep loop must not inherit it.
|
||||||
// context is honoured (see stop).
|
|
||||||
func (r *RetentionReaper) registerHooks(lc fx.Lifecycle) {
|
func (r *RetentionReaper) registerHooks(lc fx.Lifecycle) {
|
||||||
lc.Append(fx.Hook{
|
lc.Append(fx.Hook{
|
||||||
//nolint:contextcheck // Not inheriting the hook context is
|
//nolint:contextcheck // Not inheriting the hook context is
|
||||||
@@ -75,8 +73,10 @@ func (r *RetentionReaper) registerHooks(lc fx.Lifecycle) {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
OnStop: func(ctx context.Context) error {
|
OnStop: func(_ context.Context) error {
|
||||||
return r.stop(ctx)
|
r.stop()
|
||||||
|
|
||||||
|
return nil
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -105,27 +105,15 @@ func (r *RetentionReaper) start() {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// stop cancels the sweep loop's context and waits for it to
|
func (r *RetentionReaper) stop() {
|
||||||
// exit, bounded by the stop hook's context: a sweep wedged on a
|
|
||||||
// locked database must not hang the process past fx's stop
|
|
||||||
// timeout.
|
|
||||||
func (r *RetentionReaper) stop(ctx context.Context) error {
|
|
||||||
r.log.Info("retention reaper stopping")
|
r.log.Info("retention reaper stopping")
|
||||||
|
|
||||||
if r.cancel != nil {
|
if r.cancel != nil {
|
||||||
r.cancel()
|
r.cancel()
|
||||||
}
|
}
|
||||||
|
|
||||||
err := lifecycle.WaitForShutdown(
|
r.wg.Wait()
|
||||||
ctx, r.log, "retention reaper", &r.wg,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
r.log.Info("retention reaper stopped")
|
r.log.Info("retention reaper stopped")
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *RetentionReaper) run(ctx context.Context) {
|
func (r *RetentionReaper) run(ctx context.Context) {
|
||||||
@@ -145,8 +133,7 @@ func (r *RetentionReaper) run(ctx context.Context) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// sweep lists every webhook from the main database and reaps expired
|
// sweep lists every webhook from the main database and reaps expired
|
||||||
// rows from each per-webhook database that has a finite retention
|
// rows from each per-webhook database whose RetentionDays is positive.
|
||||||
// policy. Webhooks set to retain forever are skipped entirely.
|
|
||||||
func (r *RetentionReaper) sweep(ctx context.Context) {
|
func (r *RetentionReaper) sweep(ctx context.Context) {
|
||||||
var webhooks []Webhook
|
var webhooks []Webhook
|
||||||
|
|
||||||
@@ -171,13 +158,8 @@ func (r *RetentionReaper) sweep(ctx context.Context) {
|
|||||||
|
|
||||||
wh := webhooks[i]
|
wh := webhooks[i]
|
||||||
|
|
||||||
// Skip retain-forever webhooks before building any query.
|
// RetentionDays of zero or less means retain forever.
|
||||||
// RetainsForever covers both the RetentionForeverDays
|
if wh.RetentionDays <= 0 {
|
||||||
// sentinel and the non-positive values that predate it: the
|
|
||||||
// sentinel is a positive number, so without this the reaper
|
|
||||||
// would compute a cutoff a thousand years in the past and
|
|
||||||
// issue a DELETE matching nothing on every single sweep.
|
|
||||||
if wh.RetainsForever() {
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -208,10 +190,9 @@ func (r *RetentionReaper) reapWebhook(
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
cutoff, ok := retentionCutoff(time.Now(), retentionDays)
|
cutoff := time.Now().Add(
|
||||||
if !ok {
|
-time.Duration(retentionDays*hoursPerDay) * time.Hour,
|
||||||
return
|
)
|
||||||
}
|
|
||||||
|
|
||||||
deleted, err := reapExpired(db, cutoff)
|
deleted, err := reapExpired(db, cutoff)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -234,37 +215,6 @@ func (r *RetentionReaper) reapWebhook(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// retentionCutoff returns the timestamp before which a webhook's
|
|
||||||
// events have expired, and whether any cutoff applies at all. It
|
|
||||||
// reports false for a retain-forever policy, so no DELETE is issued.
|
|
||||||
//
|
|
||||||
// The day count is clamped to MaxFiniteRetentionDays first. This is
|
|
||||||
// defense in depth rather than decoration: a time.Duration is an int64
|
|
||||||
// nanosecond count, so an unclamped multiplication overflows above
|
|
||||||
// that ceiling and wraps the span negative. Subtracting a negative
|
|
||||||
// span moves the cutoff into the far future, where it matches every
|
|
||||||
// row in the database: the sweep then deletes every event, delivery,
|
|
||||||
// and delivery result, including ones created seconds ago. Rejecting
|
|
||||||
// out-of-range input at the form is the primary guard; saturating here
|
|
||||||
// means an old row, a migration, or a future call site cannot turn a
|
|
||||||
// too-large retention into total data loss.
|
|
||||||
func retentionCutoff(
|
|
||||||
now time.Time,
|
|
||||||
retentionDays int,
|
|
||||||
) (time.Time, bool) {
|
|
||||||
if retainsForever(retentionDays) {
|
|
||||||
return time.Time{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
if retentionDays > MaxFiniteRetentionDays {
|
|
||||||
retentionDays = MaxFiniteRetentionDays
|
|
||||||
}
|
|
||||||
|
|
||||||
return now.Add(
|
|
||||||
-time.Duration(retentionDays*hoursPerDay) * time.Hour,
|
|
||||||
), true
|
|
||||||
}
|
|
||||||
|
|
||||||
// reapExpired hard-deletes, in foreign-key-safe order, the delivery
|
// reapExpired hard-deletes, in foreign-key-safe order, the delivery
|
||||||
// results, deliveries, and events associated with events older than
|
// results, deliveries, and events associated with events older than
|
||||||
// cutoff. Deletes are unscoped so rows are physically removed rather
|
// cutoff. Deletes are unscoped so rows are physically removed rather
|
||||||
|
|||||||
@@ -26,13 +26,6 @@ const (
|
|||||||
// reaperTestRetentionDays is the retention policy the lifecycle
|
// reaperTestRetentionDays is the retention policy the lifecycle
|
||||||
// tests give their webhook.
|
// tests give their webhook.
|
||||||
reaperTestRetentionDays = 30
|
reaperTestRetentionDays = 30
|
||||||
|
|
||||||
// reaperWedgeStopTimeout is the stop timeout the wedged-shutdown
|
|
||||||
// test hands OnStop, standing in for fx's StopTimeout. The test
|
|
||||||
// asserts only that the hook returns at all, and allows it
|
|
||||||
// reaperStopTimeout — forty times this budget — to do so, so no
|
|
||||||
// assertion races the wall clock.
|
|
||||||
reaperWedgeStopTimeout = 250 * time.Millisecond
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// recordingLifecycle is a minimal fx.Lifecycle that records the
|
// recordingLifecycle is a minimal fx.Lifecycle that records the
|
||||||
@@ -214,59 +207,3 @@ func TestRetentionReaper_StopHookStopsLoop(t *testing.T) {
|
|||||||
"a stopped reaper must not sweep anything",
|
"a stopped reaper must not sweep anything",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRetentionReaper_StopHookHonoursStopTimeout is the
|
|
||||||
// regression test for a shutdown that could never complete. fx
|
|
||||||
// hands OnStop a context carrying the application's stop timeout;
|
|
||||||
// an OnStop that discards it and calls wg.Wait() bare hangs the
|
|
||||||
// process forever on a sweep blocked on a locked SQLite database
|
|
||||||
// — precisely when a bounded shutdown matters most.
|
|
||||||
//
|
|
||||||
// The wedged goroutine here never observes cancellation, so the
|
|
||||||
// hook can only return by honouring its context, and it must say
|
|
||||||
// so rather than reporting a clean stop.
|
|
||||||
func TestRetentionReaper_StopHookHonoursStopTimeout(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupRetentionTest(t)
|
|
||||||
|
|
||||||
env.reaper.ExportSetInterval(reaperTestInterval)
|
|
||||||
|
|
||||||
lc := startReaperViaHook(t, env.reaper)
|
|
||||||
|
|
||||||
release := make(chan struct{})
|
|
||||||
|
|
||||||
t.Cleanup(func() { close(release) })
|
|
||||||
|
|
||||||
env.reaper.ExportWedgeLoop(release)
|
|
||||||
|
|
||||||
stopCtx, cancel := context.WithTimeout(
|
|
||||||
context.Background(), reaperWedgeStopTimeout,
|
|
||||||
)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
var stopErr error
|
|
||||||
|
|
||||||
stopped := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
defer close(stopped)
|
|
||||||
|
|
||||||
stopErr = lc.hooks[0].OnStop(stopCtx)
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-stopped:
|
|
||||||
case <-time.After(reaperStopTimeout):
|
|
||||||
t.Fatal(
|
|
||||||
"OnStop did not return: it discarded the stop " +
|
|
||||||
"context and is waiting on a wedged goroutine " +
|
|
||||||
"that will never observe cancellation",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
require.ErrorIs(t, stopErr, context.DeadlineExceeded)
|
|
||||||
require.ErrorContains(t, stopErr, "retention reaper")
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -77,7 +77,7 @@ func createWebhook(
|
|||||||
|
|
||||||
wh := &database.Webhook{
|
wh := &database.Webhook{
|
||||||
UserID: uuid.New().String(),
|
UserID: uuid.New().String(),
|
||||||
Name: testWebhookName,
|
Name: "test-webhook",
|
||||||
RetentionDays: retentionDays,
|
RetentionDays: retentionDays,
|
||||||
}
|
}
|
||||||
require.NoError(
|
require.NoError(
|
||||||
@@ -85,11 +85,10 @@ func createWebhook(
|
|||||||
db.Omit(clause.Associations).Create(wh).Error,
|
db.Omit(clause.Associations).Create(wh).Error,
|
||||||
)
|
)
|
||||||
|
|
||||||
// Webhook.BeforeSave rewrites a non-positive RetentionDays to the
|
// The RetentionDays column carries a GORM default of 30, so a
|
||||||
// retain-forever sentinel, and the column's GORM default would
|
// zero (or negative) value passed to Create is replaced by that
|
||||||
// otherwise substitute 30. Force the requested value with a
|
// default. Force the requested value explicitly so the
|
||||||
// column-level update so tests can plant legacy rows that predate
|
// retain-forever (<= 0) path can be exercised.
|
||||||
// the sentinel and still carry a literal 0 or negative value.
|
|
||||||
require.NoError(
|
require.NoError(
|
||||||
t,
|
t,
|
||||||
db.Model(wh).
|
db.Model(wh).
|
||||||
@@ -99,30 +98,6 @@ func createWebhook(
|
|||||||
return wh.ID
|
return wh.ID
|
||||||
}
|
}
|
||||||
|
|
||||||
// createWebhookNormally inserts a webhook through the ordinary Create
|
|
||||||
// path, with no column-level forcing, so Webhook.BeforeSave applies
|
|
||||||
// exactly as it does in production. Passing 0 therefore yields a row
|
|
||||||
// holding the RetentionForeverDays sentinel.
|
|
||||||
func createWebhookNormally(
|
|
||||||
t *testing.T,
|
|
||||||
db *gorm.DB,
|
|
||||||
retentionDays int,
|
|
||||||
) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
wh := &database.Webhook{
|
|
||||||
UserID: uuid.New().String(),
|
|
||||||
Name: testWebhookName,
|
|
||||||
RetentionDays: retentionDays,
|
|
||||||
}
|
|
||||||
require.NoError(
|
|
||||||
t,
|
|
||||||
db.Omit(clause.Associations).Create(wh).Error,
|
|
||||||
)
|
|
||||||
|
|
||||||
return wh.ID
|
|
||||||
}
|
|
||||||
|
|
||||||
// eventChain is the set of row IDs seeded for a single event.
|
// eventChain is the set of row IDs seeded for a single event.
|
||||||
type eventChain struct {
|
type eventChain struct {
|
||||||
eventID string
|
eventID string
|
||||||
@@ -281,111 +256,12 @@ func TestRetentionReaper_ReapsExpiredKeepsRecent(t *testing.T) {
|
|||||||
assertChainPresent(t, db, recent)
|
assertChainPresent(t, db, recent)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRetentionReaper_SkipsSentinelReapsFiniteInSameSweep covers the
|
|
||||||
// end-to-end retain-forever path: a webhook created the normal way with
|
|
||||||
// a requested retention of 0 lands on the RetentionForeverDays
|
|
||||||
// sentinel, and the reaper leaves its ancient events alone while still
|
|
||||||
// reaping a finite-retention webhook in the very same sweep.
|
|
||||||
func TestRetentionReaper_SkipsSentinelReapsFiniteInSameSweep(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupRetentionTest(t)
|
|
||||||
|
|
||||||
foreverID := createWebhookNormally(t, env.mainDB.DB(), 0)
|
|
||||||
|
|
||||||
var stored database.Webhook
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t,
|
|
||||||
env.mainDB.DB().Where("id = ?", foreverID).
|
|
||||||
First(&stored).Error,
|
|
||||||
)
|
|
||||||
require.Equal(
|
|
||||||
t,
|
|
||||||
database.RetentionForeverDays,
|
|
||||||
stored.RetentionDays,
|
|
||||||
"a requested retention of 0 must persist as the sentinel",
|
|
||||||
)
|
|
||||||
|
|
||||||
finiteID := createWebhookNormally(t, env.mainDB.DB(), 30)
|
|
||||||
|
|
||||||
foreverDB, err := env.mgr.GetDB(foreverID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
finiteDB, err := env.mgr.GetDB(finiteID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
ancient := time.Now().Add(-365 * 24 * time.Hour)
|
|
||||||
kept := seedEventChain(t, foreverDB, foreverID, ancient)
|
|
||||||
doomed := seedEventChain(t, finiteDB, finiteID, ancient)
|
|
||||||
|
|
||||||
env.reaper.ExportSweep(context.Background())
|
|
||||||
|
|
||||||
assertChainPresent(t, foreverDB, kept)
|
|
||||||
assertChainGone(t, finiteDB, doomed)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRetentionReaper_HugeFiniteRetentionRetainsRecentEvents pins the
|
|
||||||
// overflow that made a large finite retention destroy everything.
|
|
||||||
//
|
|
||||||
// The cutoff is a time.Duration, an int64 nanosecond count. A day
|
|
||||||
// count above MaxFiniteRetentionDays multiplied out unclamped wraps
|
|
||||||
// negative, so subtracting it moves the cutoff into the far future,
|
|
||||||
// where "created_at < cutoff" matches every row: an event created a
|
|
||||||
// moment ago, and its delivery and delivery result, were all deleted
|
|
||||||
// on the first sweep. 200000 is inside that band and below the
|
|
||||||
// retain-forever sentinel, so it is treated as a finite policy and
|
|
||||||
// really does reach the arithmetic.
|
|
||||||
//
|
|
||||||
// The row is planted at the column level because such a value can no
|
|
||||||
// longer be submitted through the form; the point of the test is that
|
|
||||||
// a row from an older version, or a future call site, still cannot
|
|
||||||
// trigger the wipe.
|
|
||||||
func TestRetentionReaper_HugeFiniteRetentionRetainsRecentEvents(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupRetentionTest(t)
|
|
||||||
|
|
||||||
const overflowingRetentionDays = 200000
|
|
||||||
|
|
||||||
require.Greater(
|
|
||||||
t,
|
|
||||||
overflowingRetentionDays,
|
|
||||||
database.MaxFiniteRetentionDays,
|
|
||||||
"the test value must exceed what the cutoff can represent",
|
|
||||||
)
|
|
||||||
require.Less(
|
|
||||||
t,
|
|
||||||
overflowingRetentionDays,
|
|
||||||
database.RetentionForeverDays,
|
|
||||||
"the test value must not be rescued by the forever skip",
|
|
||||||
)
|
|
||||||
|
|
||||||
webhookID := createWebhook(
|
|
||||||
t, env.mainDB.DB(), overflowingRetentionDays,
|
|
||||||
)
|
|
||||||
|
|
||||||
db, err := env.mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
fresh := seedEventChain(t, db, webhookID, time.Now())
|
|
||||||
|
|
||||||
env.reaper.ExportSweep(context.Background())
|
|
||||||
|
|
||||||
assertChainPresent(t, db, fresh)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRetentionReaper_RetainsForeverWhenNonPositive(t *testing.T) {
|
func TestRetentionReaper_RetainsForeverWhenNonPositive(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupRetentionTest(t)
|
env := setupRetentionTest(t)
|
||||||
|
|
||||||
// A legacy row written before the sentinel existed still carries a
|
// RetentionDays of zero means retain forever.
|
||||||
// literal 0; the <= 0 guard must keep honouring it.
|
|
||||||
webhookID := createWebhook(t, env.mainDB.DB(), 0)
|
webhookID := createWebhook(t, env.mainDB.DB(), 0)
|
||||||
|
|
||||||
db, err := env.mgr.GetDB(webhookID)
|
db, err := env.mgr.GetDB(webhookID)
|
||||||
|
|||||||
@@ -24,24 +24,11 @@ func NewTestDatabase(db *gorm.DB) *Database {
|
|||||||
// NewTestWebhookDBManager creates a WebhookDBManager backed by the given
|
// NewTestWebhookDBManager creates a WebhookDBManager backed by the given
|
||||||
// data directory. Intended for use in tests without the fx lifecycle.
|
// data directory. Intended for use in tests without the fx lifecycle.
|
||||||
func NewTestWebhookDBManager(dataDir string) *WebhookDBManager {
|
func NewTestWebhookDBManager(dataDir string) *WebhookDBManager {
|
||||||
return NewTestWebhookDBManagerWithLogger(
|
return &WebhookDBManager{
|
||||||
dataDir,
|
dataDir: dataDir,
|
||||||
slog.New(slog.NewTextHandler(
|
log: slog.New(slog.NewTextHandler(
|
||||||
os.Stderr,
|
os.Stderr,
|
||||||
&slog.HandlerOptions{Level: slog.LevelDebug},
|
&slog.HandlerOptions{Level: slog.LevelDebug},
|
||||||
)),
|
)),
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewTestWebhookDBManagerWithLogger is NewTestWebhookDBManager with the
|
|
||||||
// logger supplied by the caller. The per-webhook databases this manager
|
|
||||||
// opens hand that logger to gormlog, so a test that needs to see the SQL
|
|
||||||
// the service emits can capture it.
|
|
||||||
func NewTestWebhookDBManagerWithLogger(
|
|
||||||
dataDir string, log *slog.Logger,
|
|
||||||
) *WebhookDBManager {
|
|
||||||
return &WebhookDBManager{
|
|
||||||
dataDir: dataDir,
|
|
||||||
log: log,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ import (
|
|||||||
"gorm.io/driver/sqlite"
|
"gorm.io/driver/sqlite"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -249,10 +248,7 @@ func (m *WebhookDBManager) openDB(
|
|||||||
|
|
||||||
db, err := gorm.Open(sqlite.Dialector{
|
db, err := gorm.Open(sqlite.Dialector{
|
||||||
Conn: sqlDB,
|
Conn: sqlDB,
|
||||||
}, &gorm.Config{
|
}, &gorm.Config{})
|
||||||
// Never leave this at GORM's default. See internal/gormlog.
|
|
||||||
Logger: gormlog.New(m.log),
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = sqlDB.Close()
|
_ = sqlDB.Close()
|
||||||
|
|
||||||
@@ -262,25 +258,6 @@ func (m *WebhookDBManager) openDB(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Keep main-database rows out of this file. See
|
|
||||||
// event_db_isolation.go.
|
|
||||||
err = omitAssociations(db)
|
|
||||||
if err != nil {
|
|
||||||
_ = sqlDB.Close()
|
|
||||||
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"guarding webhook database %s: %w",
|
|
||||||
webhookID, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = purgeTargetRows(db, m.log, webhookID)
|
|
||||||
if err != nil {
|
|
||||||
_ = sqlDB.Close()
|
|
||||||
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run migrations for event-tier models only
|
// Run migrations for event-tier models only
|
||||||
err = db.AutoMigrate(
|
err = db.AutoMigrate(
|
||||||
&Event{}, &Delivery{}, &DeliveryResult{},
|
&Event{}, &Delivery{}, &DeliveryResult{},
|
||||||
|
|||||||
@@ -339,12 +339,7 @@ func TestWebhookDBManager_MultipleWebhooks(t *testing.T) {
|
|||||||
var events []database.Event
|
var events []database.Event
|
||||||
|
|
||||||
require.NoError(t, db2.Find(&events).Error)
|
require.NoError(t, db2.Find(&events).Error)
|
||||||
|
assert.Len(t, events, 1)
|
||||||
// require, not assert: this is exactly the regression the test
|
|
||||||
// guards, so the empty slice is the expected failure, and a
|
|
||||||
// non-fatal length check would index into it on the next line and
|
|
||||||
// panic the whole package test binary instead of failing here.
|
|
||||||
require.Len(t, events, 1)
|
|
||||||
assert.Equal(t, "PUT", events[0].Method)
|
assert.Equal(t, "PUT", events[0].Method)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,126 +0,0 @@
|
|||||||
// Package datadir guards exclusive access to the directory holding
|
|
||||||
// every SQLite database webhooker writes.
|
|
||||||
//
|
|
||||||
// Two processes sharing a DATA_DIR each open the same per-webhook
|
|
||||||
// event databases and each run delivery recovery over the same rows,
|
|
||||||
// so every pending delivery goes out twice. SQLite's own locking does
|
|
||||||
// not prevent that: both writers are serialised correctly and both
|
|
||||||
// deliver. The only thing that prevents it is refusing to be the
|
|
||||||
// second process.
|
|
||||||
//
|
|
||||||
// The lock lives here rather than in the server's fx graph so that any
|
|
||||||
// entry point which touches DATA_DIR — the server, or a CLI
|
|
||||||
// subcommand that must not operate on a live deployment's data — takes
|
|
||||||
// it the same way.
|
|
||||||
package datadir
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
|
|
||||||
"github.com/gofrs/flock"
|
|
||||||
)
|
|
||||||
|
|
||||||
// LockFileName is the advisory lock file created inside DATA_DIR. Its
|
|
||||||
// contents are never read: the lock is the flock(2) held on the open
|
|
||||||
// descriptor, not the file's existence, so a leftover file from a
|
|
||||||
// process that was killed with SIGKILL blocks nothing.
|
|
||||||
const LockFileName = "webhooker.lock"
|
|
||||||
|
|
||||||
// dirPerm is the mode Acquire creates DATA_DIR with. It matches what
|
|
||||||
// internal/database uses, since whichever runs first creates it.
|
|
||||||
const dirPerm = 0o750
|
|
||||||
|
|
||||||
// ErrLocked reports that another live process holds the data
|
|
||||||
// directory. Callers that need to know whether a deployment is running
|
|
||||||
// — rather than merely failing to start — test for this with
|
|
||||||
// errors.Is.
|
|
||||||
var ErrLocked = errors.New(
|
|
||||||
"data directory is already in use by another instance",
|
|
||||||
)
|
|
||||||
|
|
||||||
// ErrNoDir reports that Acquire was given an empty directory.
|
|
||||||
var ErrNoDir = errors.New("no data directory given")
|
|
||||||
|
|
||||||
// Lock is a held exclusive advisory lock on a data directory. It is
|
|
||||||
// valid only while the process that took it lives: the kernel drops it
|
|
||||||
// when the descriptor closes, whether that is Release, a normal exit,
|
|
||||||
// or a SIGKILL.
|
|
||||||
type Lock struct {
|
|
||||||
dir string
|
|
||||||
file *flock.Flock
|
|
||||||
}
|
|
||||||
|
|
||||||
// Acquire takes the exclusive advisory lock on dir, creating dir if it
|
|
||||||
// does not exist. It never waits: if another process holds the lock it
|
|
||||||
// returns an error wrapping ErrLocked and naming dir.
|
|
||||||
//
|
|
||||||
// The returned Lock must be held for as long as the caller intends to
|
|
||||||
// use dir.
|
|
||||||
func Acquire(dir string) (*Lock, error) {
|
|
||||||
if dir == "" {
|
|
||||||
return nil, ErrNoDir
|
|
||||||
}
|
|
||||||
|
|
||||||
err := os.MkdirAll(dir, dirPerm)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"creating data directory %s: %w", dir, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
path := filepath.Join(dir, LockFileName)
|
|
||||||
fl := flock.New(path)
|
|
||||||
|
|
||||||
held, err := fl.TryLock()
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"locking data directory %s: %w", dir, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !held {
|
|
||||||
// A no-op on flock v0.13.0, which closes its own descriptor on
|
|
||||||
// a failed TryLock; kept so no version can leak one.
|
|
||||||
_ = fl.Close()
|
|
||||||
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"%w: %s (%s). Only one webhooker may use a data "+
|
|
||||||
"directory: two both run delivery recovery over the "+
|
|
||||||
"same rows and both deliver",
|
|
||||||
ErrLocked, dir, path,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &Lock{dir: dir, file: fl}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Dir returns the locked directory.
|
|
||||||
func (l *Lock) Dir() string {
|
|
||||||
return l.dir
|
|
||||||
}
|
|
||||||
|
|
||||||
// Path returns the lock file backing the lock.
|
|
||||||
func (l *Lock) Path() string {
|
|
||||||
return l.file.Path()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Release drops the lock and closes the descriptor. It is safe to call
|
|
||||||
// more than once.
|
|
||||||
//
|
|
||||||
// The lock file is deliberately left on disk. Unlinking it would let
|
|
||||||
// the next process create and lock a fresh inode while a third still
|
|
||||||
// holds the old one, which is the one outcome this package exists to
|
|
||||||
// prevent.
|
|
||||||
func (l *Lock) Release() error {
|
|
||||||
err := l.file.Unlock()
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"releasing lock on data directory %s: %w", l.dir, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,250 +0,0 @@
|
|||||||
package datadir_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/datadir"
|
|
||||||
)
|
|
||||||
|
|
||||||
// holderEnv names the directory the re-executed test binary should
|
|
||||||
// lock and hold. When it is unset the child test does nothing, so an
|
|
||||||
// ordinary run is unaffected.
|
|
||||||
const holderEnv = "WEBHOOKER_DATADIR_LOCK_HOLDER"
|
|
||||||
|
|
||||||
// holderReadyPrefix labels the child's one-line report that it holds
|
|
||||||
// the lock, so the parent can find it among the testing package's own
|
|
||||||
// output on the same descriptor.
|
|
||||||
const holderReadyPrefix = "DATADIR-LOCK-HELD "
|
|
||||||
|
|
||||||
// holderReadyTimeout bounds the wait for the child to take the lock.
|
|
||||||
// It only has to cover process start on a loaded shared host.
|
|
||||||
const holderReadyTimeout = 60 * time.Second
|
|
||||||
|
|
||||||
// holderHold is how long the child keeps the lock if nothing kills it.
|
|
||||||
// A sleep rather than a bare block, so the runtime's deadlock detector
|
|
||||||
// has a pending timer and the child cannot outlive a killed test run
|
|
||||||
// by more than this.
|
|
||||||
const holderHold = 10 * time.Minute
|
|
||||||
|
|
||||||
// TestLockHolder is the child half of the two-process tests below. It
|
|
||||||
// takes the lock on the directory named by holderEnv, reports the lock
|
|
||||||
// file on standard output, and then holds it until it is killed.
|
|
||||||
func TestLockHolder(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := os.Getenv(holderEnv)
|
|
||||||
if dir == "" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
lock, err := datadir.Acquire(dir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Written to the descriptor directly: the parent reads fd 1, not
|
|
||||||
// the testing package's buffered report.
|
|
||||||
_, err = fmt.Fprintf(
|
|
||||||
os.Stdout, "%s%s\n", holderReadyPrefix, lock.Path(),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
time.Sleep(holderHold)
|
|
||||||
}
|
|
||||||
|
|
||||||
// startHolder re-executes this test binary as a separate process that
|
|
||||||
// takes and holds the lock on dir, and returns once that process
|
|
||||||
// actually holds it. The child is killed when the test ends.
|
|
||||||
func startHolder(t *testing.T, dir string) *exec.Cmd {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
//nolint:gosec // Re-executing this test binary, with a fixed arg.
|
|
||||||
cmd := exec.CommandContext(
|
|
||||||
t.Context(), os.Args[0], "-test.run", "^TestLockHolder$",
|
|
||||||
)
|
|
||||||
|
|
||||||
cmd.Env = append(os.Environ(), holderEnv+"="+dir)
|
|
||||||
cmd.Stderr = os.Stderr
|
|
||||||
|
|
||||||
stdout, err := cmd.StdoutPipe()
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, cmd.Start())
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
_ = cmd.Process.Kill()
|
|
||||||
_ = cmd.Wait()
|
|
||||||
})
|
|
||||||
|
|
||||||
ready := make(chan string, 1)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
scanner := bufio.NewScanner(stdout)
|
|
||||||
for scanner.Scan() {
|
|
||||||
after, found := strings.CutPrefix(
|
|
||||||
scanner.Text(), holderReadyPrefix,
|
|
||||||
)
|
|
||||||
if found {
|
|
||||||
ready <- after
|
|
||||||
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
close(ready)
|
|
||||||
|
|
||||||
// Keep draining so the child never blocks on a full pipe.
|
|
||||||
_, _ = io.Copy(io.Discard, stdout)
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case path, ok := <-ready:
|
|
||||||
require.True(
|
|
||||||
t, ok, "holder exited without taking the lock",
|
|
||||||
)
|
|
||||||
require.Equal(t, filepath.Join(dir, datadir.LockFileName), path)
|
|
||||||
case <-time.After(holderReadyTimeout):
|
|
||||||
t.Fatal("timed out waiting for the holder to take the lock")
|
|
||||||
}
|
|
||||||
|
|
||||||
return cmd
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSecondInstanceRefused is the regression test for the duplicate
|
|
||||||
// delivery this package exists to prevent: a real second process
|
|
||||||
// pointed at a data directory a live process already holds must be
|
|
||||||
// refused, with an error that names the directory.
|
|
||||||
func TestSecondInstanceRefused(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
|
|
||||||
startHolder(t, dir)
|
|
||||||
|
|
||||||
lock, err := datadir.Acquire(dir)
|
|
||||||
|
|
||||||
require.Error(t, err, "the second instance took the lock too")
|
|
||||||
require.Nil(t, lock)
|
|
||||||
require.ErrorIs(
|
|
||||||
t, err, datadir.ErrLocked,
|
|
||||||
"the refusal must be distinguishable from any other failure",
|
|
||||||
)
|
|
||||||
assert.Contains(
|
|
||||||
t, err.Error(), dir,
|
|
||||||
"the refusal must name the directory it is about",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRestartAfterHardKill is the other half of the regression: a
|
|
||||||
// process killed with SIGKILL runs no cleanup and leaves its lock file
|
|
||||||
// behind, and the next start must not be blocked by it. This is what a
|
|
||||||
// pidfile would get wrong; the kernel drops a flock when the
|
|
||||||
// descriptor closes, however the process died.
|
|
||||||
func TestRestartAfterHardKill(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
|
|
||||||
holder := startHolder(t, dir)
|
|
||||||
|
|
||||||
require.NoError(t, holder.Process.Kill())
|
|
||||||
|
|
||||||
// Wait for the kill to have actually happened. Re-acquiring while
|
|
||||||
// the corpse still holds a descriptor would be a race, and would
|
|
||||||
// make this test pass or fail on scheduling.
|
|
||||||
_ = holder.Wait()
|
|
||||||
|
|
||||||
require.FileExists(
|
|
||||||
t, filepath.Join(dir, datadir.LockFileName),
|
|
||||||
"the stale lock file is what must not block the restart",
|
|
||||||
)
|
|
||||||
|
|
||||||
lock, err := datadir.Acquire(dir)
|
|
||||||
require.NoError(
|
|
||||||
t, err, "a hard-killed instance must not block the next start",
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, lock.Release())
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSecondFdInSameProcessRefused pins the flock(2) property the
|
|
||||||
// tests in cmd/webhooker rely on: descriptors are locked
|
|
||||||
// independently, so a second acquisition is denied even when it comes
|
|
||||||
// from the process that already holds the lock.
|
|
||||||
func TestSecondFdInSameProcessRefused(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
|
|
||||||
first, err := datadir.Acquire(dir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
defer func() { _ = first.Release() }()
|
|
||||||
|
|
||||||
_, err = datadir.Acquire(dir)
|
|
||||||
require.ErrorIs(t, err, datadir.ErrLocked)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestReleaseAllowsReacquire covers the clean-shutdown path: the lock
|
|
||||||
// is released on exit, so a restart is not blocked by the previous
|
|
||||||
// run.
|
|
||||||
func TestReleaseAllowsReacquire(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
|
|
||||||
first, err := datadir.Acquire(dir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, first.Release())
|
|
||||||
|
|
||||||
second, err := datadir.Acquire(dir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, second.Release())
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestAcquireCreatesDataDir covers a first start against a DATA_DIR
|
|
||||||
// that does not exist yet, which is the normal case for a fresh
|
|
||||||
// deployment: the lock is taken before anything else creates it.
|
|
||||||
func TestAcquireCreatesDataDir(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := filepath.Join(t.TempDir(), "nested", "data")
|
|
||||||
|
|
||||||
lock, err := datadir.Acquire(dir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
defer func() { _ = lock.Release() }()
|
|
||||||
|
|
||||||
assert.Equal(t, dir, lock.Dir())
|
|
||||||
assert.FileExists(t, filepath.Join(dir, datadir.LockFileName))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestAcquireEmptyDir rejects an empty directory rather than locking
|
|
||||||
// the process's working directory.
|
|
||||||
func TestAcquireEmptyDir(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := datadir.Acquire("")
|
|
||||||
require.ErrorIs(t, err, datadir.ErrNoDir)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestAcquireUnusableDir reports an unusable DATA_DIR clearly, naming
|
|
||||||
// it, instead of failing later and deeper.
|
|
||||||
func TestAcquireUnusableDir(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
file := filepath.Join(t.TempDir(), "not-a-directory")
|
|
||||||
require.NoError(t, os.WriteFile(file, nil, 0o600))
|
|
||||||
|
|
||||||
_, err := datadir.Acquire(file)
|
|
||||||
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), file)
|
|
||||||
}
|
|
||||||
@@ -10,7 +10,6 @@ import (
|
|||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/lifecycle"
|
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -68,9 +67,10 @@ func NewArchiveSweeper(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// registerHooks wires the sweeper's start and stop into the fx
|
// registerHooks wires the sweeper's start and stop into the fx
|
||||||
// lifecycle. The start hook's context is deliberately ignored
|
// lifecycle. Both hook contexts are deliberately ignored: see
|
||||||
// (see start for why the background loop must not inherit it);
|
// start for why the background loop must not inherit the start
|
||||||
// the stop hook's context is honoured (see stop).
|
// hook's context, and stop for why shutdown blocks on the loop
|
||||||
|
// rather than on the stop hook's deadline.
|
||||||
func (s *ArchiveSweeper) registerHooks(lc fx.Lifecycle) {
|
func (s *ArchiveSweeper) registerHooks(lc fx.Lifecycle) {
|
||||||
lc.Append(fx.Hook{
|
lc.Append(fx.Hook{
|
||||||
//nolint:contextcheck // Not passing the hook context is
|
//nolint:contextcheck // Not passing the hook context is
|
||||||
@@ -80,8 +80,10 @@ func (s *ArchiveSweeper) registerHooks(lc fx.Lifecycle) {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
OnStop: func(ctx context.Context) error {
|
OnStop: func(_ context.Context) error {
|
||||||
return s.stop(ctx)
|
s.stop()
|
||||||
|
|
||||||
|
return nil
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -111,27 +113,15 @@ func (s *ArchiveSweeper) start() {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// stop cancels the sweep loop's context and waits for it to
|
func (s *ArchiveSweeper) stop() {
|
||||||
// exit, bounded by the stop hook's context: a prune wedged on a
|
|
||||||
// locked archive must not hang the process past fx's stop
|
|
||||||
// timeout.
|
|
||||||
func (s *ArchiveSweeper) stop(ctx context.Context) error {
|
|
||||||
s.log.Info("archive sweeper stopping")
|
s.log.Info("archive sweeper stopping")
|
||||||
|
|
||||||
if s.cancel != nil {
|
if s.cancel != nil {
|
||||||
s.cancel()
|
s.cancel()
|
||||||
}
|
}
|
||||||
|
|
||||||
err := lifecycle.WaitForShutdown(
|
s.wg.Wait()
|
||||||
ctx, s.log, "archive sweeper", &s.wg,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
s.log.Info("archive sweeper stopped")
|
s.log.Info("archive sweeper stopped")
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *ArchiveSweeper) run(ctx context.Context) {
|
func (s *ArchiveSweeper) run(ctx context.Context) {
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import (
|
|||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
|
"go.uber.org/fx"
|
||||||
"gorm.io/driver/sqlite"
|
"gorm.io/driver/sqlite"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"gorm.io/gorm/clause"
|
"gorm.io/gorm/clause"
|
||||||
@@ -225,6 +226,17 @@ func countArchivedRows(path string) (int64, error) {
|
|||||||
return count, nil
|
return count, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// captureLifecycle is a minimal fx.Lifecycle that records the
|
||||||
|
// hooks a component registers, so a test can invoke the real
|
||||||
|
// OnStart/OnStop functions with a context of its choosing.
|
||||||
|
type captureLifecycle struct {
|
||||||
|
hooks []fx.Hook
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *captureLifecycle) Append(h fx.Hook) {
|
||||||
|
l.hooks = append(l.hooks, h)
|
||||||
|
}
|
||||||
|
|
||||||
// TestArchiveSweeper_LoopOutlivesStartHookContext is the
|
// TestArchiveSweeper_LoopOutlivesStartHookContext is the
|
||||||
// regression test for a sweeper that never swept. fx calls
|
// regression test for a sweeper that never swept. fx calls
|
||||||
// OnStart with a context carrying the application's start
|
// OnStart with a context carrying the application's start
|
||||||
@@ -258,7 +270,7 @@ func TestArchiveSweeper_LoopOutlivesStartHookContext(
|
|||||||
|
|
||||||
// Drive the genuine fx hooks the application registers,
|
// Drive the genuine fx hooks the application registers,
|
||||||
// rather than a test-only entry point.
|
// rather than a test-only entry point.
|
||||||
lc := &recordingLifecycle{}
|
lc := &captureLifecycle{}
|
||||||
env.sweeper.ExportRegisterHooks(lc)
|
env.sweeper.ExportRegisterHooks(lc)
|
||||||
require.Len(t, lc.hooks, 1)
|
require.Len(t, lc.hooks, 1)
|
||||||
|
|
||||||
@@ -912,36 +924,7 @@ func TestArchiveSweeper_StopsCleanly(t *testing.T) {
|
|||||||
env.sweeper.ExportSetInterval(time.Millisecond)
|
env.sweeper.ExportSetInterval(time.Millisecond)
|
||||||
env.sweeper.ExportStart()
|
env.sweeper.ExportStart()
|
||||||
|
|
||||||
// stop blocks on the loop's WaitGroup, so returning without
|
// stop blocks on the loop's WaitGroup, so returning at all
|
||||||
// error proves the loop observed the cancellation and exited
|
// proves the loop observed the cancellation and exited.
|
||||||
// well inside the stop context.
|
env.sweeper.ExportStop()
|
||||||
require.NoError(
|
|
||||||
t, env.sweeper.ExportStop(context.Background()),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveSweeper_StopHookHonoursStopTimeout is the sweeper's
|
|
||||||
// half of the same shutdown defect the engine and the retention
|
|
||||||
// reaper carried: an OnStop that discards its context and waits
|
|
||||||
// on the WaitGroup bare hangs the process forever on a prune
|
|
||||||
// wedged inside a locked archive.
|
|
||||||
func TestArchiveSweeper_StopHookHonoursStopTimeout(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSweeperTest(t)
|
|
||||||
|
|
||||||
lc := &recordingLifecycle{}
|
|
||||||
env.sweeper.ExportRegisterHooks(lc)
|
|
||||||
require.Len(t, lc.hooks, 1)
|
|
||||||
require.NoError(t, lc.hooks[0].OnStart(context.Background()))
|
|
||||||
|
|
||||||
release := make(chan struct{})
|
|
||||||
|
|
||||||
t.Cleanup(func() { close(release) })
|
|
||||||
|
|
||||||
env.sweeper.ExportWedgeLoop(release)
|
|
||||||
|
|
||||||
requireStopHookExpires(t, lc.hooks[0], "archive sweeper")
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,26 +19,25 @@ func newSSRFTestEngine() *delivery.Engine {
|
|||||||
|
|
||||||
client := &http.Client{
|
client := &http.Client{
|
||||||
Timeout: 30 * time.Second,
|
Timeout: 30 * time.Second,
|
||||||
Transport: delivery.NewTestGuard().
|
Transport: delivery.NewSSRFSafeTransport(),
|
||||||
NewSSRFSafeTransport(),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return delivery.NewTestEngine(log, client, 1)
|
return delivery.NewTestEngine(log, client, 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestClientForRequest_TimeoutKeepsSSRFGuard asserts that a
|
// TestClientForConfig_TimeoutKeepsSSRFGuard asserts that a
|
||||||
// client returned by clientForRequest for a config with a
|
// client returned by clientForConfig for a config with a
|
||||||
// per-target timeout still refuses connections to
|
// per-target timeout still refuses connections to
|
||||||
// private/reserved addresses (the timeout must not drop the
|
// private/reserved addresses (the timeout must not drop the
|
||||||
// SSRF-safe transport).
|
// SSRF-safe transport).
|
||||||
func TestClientForRequest_TimeoutKeepsSSRFGuard(t *testing.T) {
|
func TestClientForConfig_TimeoutKeepsSSRFGuard(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
engine := newSSRFTestEngine()
|
engine := newSSRFTestEngine()
|
||||||
|
|
||||||
blocked := []string{
|
blocked := []string{
|
||||||
loopbackHookURL,
|
"http://127.0.0.1/hook",
|
||||||
metadataURL,
|
"http://169.254.169.254/latest/meta-data/",
|
||||||
"http://[fe80::1]/hook",
|
"http://[fe80::1]/hook",
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -51,7 +50,7 @@ func TestClientForRequest_TimeoutKeepsSSRFGuard(t *testing.T) {
|
|||||||
Timeout: 5,
|
Timeout: 5,
|
||||||
}
|
}
|
||||||
|
|
||||||
client := engine.ExportClientForRequest(cfg, nil)
|
client := engine.ExportClientForConfig(cfg)
|
||||||
|
|
||||||
require.NotSame(t, engine.ExportClient(), client,
|
require.NotSame(t, engine.ExportClient(), client,
|
||||||
"a per-target timeout must yield a "+
|
"a per-target timeout must yield a "+
|
||||||
@@ -92,11 +91,10 @@ func TestClientForRequest_TimeoutKeepsSSRFGuard(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestClientForRequest_NoTimeoutUnchanged asserts that a
|
// TestClientForConfig_NoTimeoutUnchanged asserts that with
|
||||||
// request with neither a per-target timeout nor an origin-scoped
|
// no per-target timeout the shared SSRF-safe client is
|
||||||
// header gets the shared SSRF-safe client unchanged: there is then
|
// returned unchanged.
|
||||||
// nothing for a redirect policy to strip.
|
func TestClientForConfig_NoTimeoutUnchanged(t *testing.T) {
|
||||||
func TestClientForRequest_NoTimeoutUnchanged(t *testing.T) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
engine := newSSRFTestEngine()
|
engine := newSSRFTestEngine()
|
||||||
@@ -105,46 +103,10 @@ func TestClientForRequest_NoTimeoutUnchanged(t *testing.T) {
|
|||||||
URL: "https://example.com/hook",
|
URL: "https://example.com/hook",
|
||||||
}
|
}
|
||||||
|
|
||||||
client := engine.ExportClientForRequest(cfg, nil)
|
client := engine.ExportClientForConfig(cfg)
|
||||||
|
|
||||||
assert.Same(t, engine.ExportClient(), client,
|
assert.Same(t, engine.ExportClient(), client,
|
||||||
"without a per-target timeout the shared client "+
|
"without a per-target timeout the shared client "+
|
||||||
"must be returned unchanged",
|
"must be returned unchanged",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestClientForRequest_HeadersKeepSSRFGuard asserts that the
|
|
||||||
// redirect policy an origin-scoped header installs is added to a
|
|
||||||
// client that still carries the SSRF-safe transport. The guard is
|
|
||||||
// a dial hook, so keeping it is what makes each redirect hop pass
|
|
||||||
// the private-IP check too.
|
|
||||||
func TestClientForRequest_HeadersKeepSSRFGuard(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
engine := newSSRFTestEngine()
|
|
||||||
|
|
||||||
cfg := &delivery.HTTPTargetConfig{
|
|
||||||
URL: "https://example.com/with-headers",
|
|
||||||
Headers: map[string]string{
|
|
||||||
"X-Api-Key": "configured",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
client := engine.ExportClientForRequest(
|
|
||||||
cfg, []string{"X-Api-Key"},
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NotNil(t, client.CheckRedirect,
|
|
||||||
"an origin-scoped header must install a redirect policy",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Same(t,
|
|
||||||
engine.ExportClient().Transport, client.Transport,
|
|
||||||
"the SSRF-safe transport must be reused, not dropped",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
engine.ExportClient().Timeout, client.Timeout,
|
|
||||||
"the shared client's timeout must be inherited",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -13,9 +13,7 @@ import (
|
|||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/lifecycle"
|
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
"sneak.berlin/go/webhooker/internal/metrics"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -121,7 +119,6 @@ type EngineParams struct {
|
|||||||
DB *database.Database
|
DB *database.Database
|
||||||
DBManager *database.WebhookDBManager
|
DBManager *database.WebhookDBManager
|
||||||
Logger *logger.Logger
|
Logger *logger.Logger
|
||||||
SSRFGuard *Guard
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Engine processes queued deliveries in the background
|
// Engine processes queued deliveries in the background
|
||||||
@@ -141,12 +138,6 @@ type Engine struct {
|
|||||||
retryCh chan Task
|
retryCh chan Task
|
||||||
workers int
|
workers int
|
||||||
|
|
||||||
// mtr is the delivery metric set. Production wires the
|
|
||||||
// process-wide one; a test can substitute a set registered on
|
|
||||||
// a private registry so its assertions are not disturbed by
|
|
||||||
// deliveries other tests are making at the same time.
|
|
||||||
mtr *metrics.Set
|
|
||||||
|
|
||||||
// targets maps each target type to its implementation.
|
// targets maps each target type to its implementation.
|
||||||
targets map[database.TargetType]Target
|
targets map[database.TargetType]Target
|
||||||
|
|
||||||
@@ -172,12 +163,11 @@ func New(
|
|||||||
deliveryCh: make(chan Task, deliveryChannelSize),
|
deliveryCh: make(chan Task, deliveryChannelSize),
|
||||||
retryCh: make(chan Task, retryChannelSize),
|
retryCh: make(chan Task, retryChannelSize),
|
||||||
workers: defaultWorkers,
|
workers: defaultWorkers,
|
||||||
mtr: metrics.Default(),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
e.initTargets(&http.Client{
|
e.initTargets(&http.Client{
|
||||||
Timeout: httpClientTimeout,
|
Timeout: httpClientTimeout,
|
||||||
Transport: params.SSRFGuard.NewSSRFSafeTransport(),
|
Transport: NewSSRFSafeTransport(),
|
||||||
})
|
})
|
||||||
|
|
||||||
e.registerHooks(lc)
|
e.registerHooks(lc)
|
||||||
@@ -244,9 +234,8 @@ func (e *Engine) ScheduleRetry(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// registerHooks wires the engine's start and stop into the fx
|
// registerHooks wires the engine's start and stop into the fx
|
||||||
// lifecycle. The start hook's context is deliberately ignored
|
// lifecycle. The start hook's context is deliberately ignored:
|
||||||
// (see start for why the worker pool must not inherit it); the
|
// see start for why the worker pool must not inherit it.
|
||||||
// stop hook's context is honoured (see stop).
|
|
||||||
func (e *Engine) registerHooks(lc fx.Lifecycle) {
|
func (e *Engine) registerHooks(lc fx.Lifecycle) {
|
||||||
lc.Append(fx.Hook{
|
lc.Append(fx.Hook{
|
||||||
//nolint:contextcheck // Not inheriting the hook context
|
//nolint:contextcheck // Not inheriting the hook context
|
||||||
@@ -256,8 +245,10 @@ func (e *Engine) registerHooks(lc fx.Lifecycle) {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
OnStop: func(ctx context.Context) error {
|
OnStop: func(_ context.Context) error {
|
||||||
return e.stop(ctx)
|
e.stop()
|
||||||
|
|
||||||
|
return nil
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -292,36 +283,17 @@ func (e *Engine) start() {
|
|||||||
|
|
||||||
go e.retrySweep(ctx)
|
go e.retrySweep(ctx)
|
||||||
|
|
||||||
e.wg.Add(1)
|
|
||||||
|
|
||||||
go e.queueDepthSampler(ctx)
|
|
||||||
|
|
||||||
e.log.Info(
|
e.log.Info(
|
||||||
"delivery engine started",
|
"delivery engine started",
|
||||||
"workers", e.workers,
|
"workers", e.workers,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// stop cancels the worker pool's context and waits for the pool
|
func (e *Engine) stop() {
|
||||||
// to drain, bounded by the stop hook's context: a wedged worker
|
|
||||||
// must not hang the process past fx's stop timeout.
|
|
||||||
func (e *Engine) stop(ctx context.Context) error {
|
|
||||||
e.log.Info("delivery engine stopping")
|
e.log.Info("delivery engine stopping")
|
||||||
|
|
||||||
if e.cancel != nil {
|
|
||||||
e.cancel()
|
e.cancel()
|
||||||
}
|
e.wg.Wait()
|
||||||
|
|
||||||
err := lifecycle.WaitForShutdown(
|
|
||||||
ctx, e.log, "delivery engine", &e.wg,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
e.log.Info("delivery engine stopped")
|
e.log.Info("delivery engine stopped")
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (e *Engine) worker(ctx context.Context) {
|
func (e *Engine) worker(ctx context.Context) {
|
||||||
@@ -813,9 +785,9 @@ func (e *Engine) sweepSingleRetry(
|
|||||||
// status retrying themselves. Re-dispatching under the new type
|
// status retrying themselves. Re-dispatching under the new type
|
||||||
// would be a delivery the operator never asked for, and leaving
|
// would be a delivery the operator never asked for, and leaving
|
||||||
// the row retrying strands it forever, so the delivery is
|
// the row retrying strands it forever, so the delivery is
|
||||||
// failed with a recorded reason. The event stays stored, but
|
// failed with a recorded reason and can be redelivered
|
||||||
// nothing redelivers it today. Logged at warn, not error: this
|
// manually. Logged at warn, not error: this is operator-caused
|
||||||
// is operator-caused state, not a system fault.
|
// state, not a system fault.
|
||||||
func (e *Engine) failUnretryableRetry(
|
func (e *Engine) failUnretryableRetry(
|
||||||
webhookDB *gorm.DB,
|
webhookDB *gorm.DB,
|
||||||
webhookID string,
|
webhookID string,
|
||||||
@@ -850,15 +822,8 @@ func (e *Engine) failUnretryableRetry(
|
|||||||
0,
|
0,
|
||||||
)
|
)
|
||||||
|
|
||||||
// The type is passed rather than assigned onto d: the delivery
|
|
||||||
// is loaded here without its target relation, and populating
|
|
||||||
// d.Target would make GORM's SaveBeforeAssociations upsert the
|
|
||||||
// whole target row — plaintext config, which for a slack target
|
|
||||||
// is the credential — into the per-webhook event database. See
|
|
||||||
// https://git.eeqj.de/sneak/webhooker/issues/206.
|
|
||||||
e.updateDeliveryStatus(
|
e.updateDeliveryStatus(
|
||||||
webhookDB, d, target.Type,
|
webhookDB, d, database.DeliveryStatusFailed,
|
||||||
database.DeliveryStatusFailed,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -879,8 +844,7 @@ func (e *Engine) processDelivery(
|
|||||||
)
|
)
|
||||||
|
|
||||||
e.updateDeliveryStatus(
|
e.updateDeliveryStatus(
|
||||||
webhookDB, d, d.Target.Type,
|
webhookDB, d, database.DeliveryStatusFailed,
|
||||||
database.DeliveryStatusFailed,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -889,24 +853,6 @@ func (e *Engine) processDelivery(
|
|||||||
target.Deliver(ctx, webhookDB, d, task, e)
|
target.Deliver(ctx, webhookDB, d, task, e)
|
||||||
}
|
}
|
||||||
|
|
||||||
// observeAttempt counts one delivery attempt that was actually
|
|
||||||
// dispatched to a target, and records how long it took.
|
|
||||||
//
|
|
||||||
// It is called from the dispatch paths rather than from around
|
|
||||||
// Target.Deliver, because Deliver is also entered for deliveries
|
|
||||||
// that never reach the wire: a delivery an open circuit breaker
|
|
||||||
// refuses sends nothing, records no DeliveryResult, and is
|
|
||||||
// rescheduled. Counting those would climb the attempts counter with
|
|
||||||
// no traffic behind it and fill the duration histogram with
|
|
||||||
// microsecond samples, which would make the delivery-duration
|
|
||||||
// quantiles improve during exactly the outage they exist to reveal.
|
|
||||||
func (e *Engine) observeAttempt(
|
|
||||||
t database.TargetType, dur time.Duration,
|
|
||||||
) {
|
|
||||||
e.mtr.DeliveryAttempted(t)
|
|
||||||
e.mtr.ObserveDeliveryDuration(t, dur)
|
|
||||||
}
|
|
||||||
|
|
||||||
// recordResult persists a DeliveryResult row describing a
|
// recordResult persists a DeliveryResult row describing a
|
||||||
// single attempt. It is a cross-target helper the targets
|
// single attempt. It is a cross-target helper the targets
|
||||||
// call.
|
// call.
|
||||||
@@ -940,22 +886,10 @@ func (e *Engine) recordResult(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// updateDeliveryStatus persists a new status for a delivery.
|
// updateDeliveryStatus persists a new status for a delivery.
|
||||||
// It is a cross-target helper the targets call, and therefore the
|
// It is a cross-target helper the targets call.
|
||||||
// single point where a delivery's outcome — delivered, terminally
|
|
||||||
// failed, or put back into retry — is counted.
|
|
||||||
//
|
|
||||||
// The target type is a parameter rather than read off d.Target
|
|
||||||
// because one caller — failUnretryableRetry — deliberately holds a
|
|
||||||
// delivery loaded without its target relation, and must keep it that
|
|
||||||
// way: a populated d.Target makes GORM upsert the target row, config
|
|
||||||
// and all, into the per-webhook database.
|
|
||||||
//
|
|
||||||
// The counter moves only after the row is written, so a transition
|
|
||||||
// the database rejected is not claimed as an outcome that happened.
|
|
||||||
func (e *Engine) updateDeliveryStatus(
|
func (e *Engine) updateDeliveryStatus(
|
||||||
webhookDB *gorm.DB,
|
webhookDB *gorm.DB,
|
||||||
d *database.Delivery,
|
d *database.Delivery,
|
||||||
targetType database.TargetType,
|
|
||||||
status database.DeliveryStatus,
|
status database.DeliveryStatus,
|
||||||
) {
|
) {
|
||||||
err := webhookDB.Model(d).
|
err := webhookDB.Model(d).
|
||||||
@@ -967,11 +901,7 @@ func (e *Engine) updateDeliveryStatus(
|
|||||||
"status", status,
|
"status", status,
|
||||||
"error", err,
|
"error", err,
|
||||||
)
|
)
|
||||||
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
e.mtr.DeliveryStatusChanged(targetType, status)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func truncate(s string, maxLen int) string {
|
func truncate(s string, maxLen int) string {
|
||||||
|
|||||||
@@ -501,7 +501,7 @@ func TestWorkerLifecycle_StartStop(t *testing.T) {
|
|||||||
|
|
||||||
iWaitForDelivered(t, s.WebhookDB, d.ID)
|
iWaitForDelivered(t, s.WebhookDB, d.ID)
|
||||||
|
|
||||||
require.NoError(t, s.Engine.ExportStop(context.Background()))
|
s.Engine.ExportStop()
|
||||||
}
|
}
|
||||||
|
|
||||||
// iWaitForDelivered polls until the delivery reaches the
|
// iWaitForDelivered polls until the delivery reaches the
|
||||||
@@ -567,7 +567,7 @@ func TestWorkerLifecycle_ProcessesRetryChannel(
|
|||||||
|
|
||||||
iWaitForDelivered(t, s.WebhookDB, d.ID)
|
iWaitForDelivered(t, s.WebhookDB, d.ID)
|
||||||
|
|
||||||
require.NoError(t, s.Engine.ExportStop(context.Background()))
|
s.Engine.ExportStop()
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- processDelivery: unknown target type ---
|
// --- processDelivery: unknown target type ---
|
||||||
@@ -886,82 +886,6 @@ func TestSweepSingleRetry_TypeNoLongerRetries(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestFailUnretryableRetry_WritesNoTargetRow proves the
|
|
||||||
// orphaned-retry terminal path leaves no target row — and so no
|
|
||||||
// plaintext target config — in the per-webhook event database.
|
|
||||||
//
|
|
||||||
// That path loads the delivery without its Target relation on
|
|
||||||
// purpose. Populating d.Target makes GORM's SaveBeforeAssociations
|
|
||||||
// upsert the whole target row on the status UPDATE, which for a slack
|
|
||||||
// target writes the incoming-webhook credential into events-*.db.
|
|
||||||
// See https://git.eeqj.de/sneak/webhooker/issues/206.
|
|
||||||
func TestFailUnretryableRetry_WritesNoTargetRow(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
|
|
||||||
iCreateWebhook(
|
|
||||||
t, s.MainDB, s.WebhookID, "no-target-row",
|
|
||||||
)
|
|
||||||
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
|
|
||||||
// A Slack incoming-webhook URL: the target config IS the
|
|
||||||
// credential, which is what makes a leaked target row a
|
|
||||||
// disclosure rather than a curiosity.
|
|
||||||
hookURL := "https://hooks.slack.com/services/T00/B00/x"
|
|
||||||
|
|
||||||
iCreateTarget(t, s.MainDB, targetID,
|
|
||||||
s.WebhookID, "credential-bearing",
|
|
||||||
database.TargetTypeLog, iHTTPConfig(hookURL), 5,
|
|
||||||
)
|
|
||||||
|
|
||||||
event := iSeedEvent(
|
|
||||||
t, s.WebhookDB, s.WebhookID, `{"orphaned":"retry"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
d := iSeedDelivery(
|
|
||||||
t, s.WebhookDB, event.ID, targetID,
|
|
||||||
database.DeliveryStatusRetrying,
|
|
||||||
)
|
|
||||||
|
|
||||||
iSeedFailedResult(t, s.WebhookDB, d.ID)
|
|
||||||
|
|
||||||
s.Engine.ExportSweepWebhookRetries(
|
|
||||||
context.Background(), s.WebhookID,
|
|
||||||
)
|
|
||||||
|
|
||||||
iAssertStatus(t, s.WebhookDB, d.ID,
|
|
||||||
database.DeliveryStatusFailed,
|
|
||||||
)
|
|
||||||
|
|
||||||
// The table exists in the per-webhook database because GORM
|
|
||||||
// migrates the Delivery relation's model alongside it. It must
|
|
||||||
// stay empty.
|
|
||||||
var targetRows int64
|
|
||||||
|
|
||||||
require.NoError(t, s.WebhookDB.
|
|
||||||
Table("targets").
|
|
||||||
Count(&targetRows).Error)
|
|
||||||
|
|
||||||
assert.Zero(t, targetRows,
|
|
||||||
"orphaned-retry terminal failure wrote a target row "+
|
|
||||||
"into the per-webhook event database",
|
|
||||||
)
|
|
||||||
|
|
||||||
var configs []string
|
|
||||||
|
|
||||||
require.NoError(t, s.WebhookDB.
|
|
||||||
Table("targets").
|
|
||||||
Pluck("config", &configs).Error)
|
|
||||||
|
|
||||||
assert.NotContains(
|
|
||||||
t, strings.Join(configs, " "), hookURL,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRecoverSingleRetry_UnknownTargetType(
|
func TestRecoverSingleRetry_UnknownTargetType(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
@@ -1201,83 +1125,6 @@ func TestDeliverHTTP_TargetTimeout(t *testing.T) {
|
|||||||
iAssertResultFailed(t, db, del.ID)
|
iAssertResultFailed(t, db, del.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestDeliverHTTP_CutsStoredResponseAtMaxBodyLog pins the size
|
|
||||||
// this engine stores for an oversized response, because the
|
|
||||||
// event log's redaction is written against it: the row holds
|
|
||||||
// exactly maxBodyLog bytes and records nothing about how much
|
|
||||||
// more the remote sent, so a credential echoed across that
|
|
||||||
// boundary reaches the database already severed and no reader
|
|
||||||
// of the row can tell the cut happened.
|
|
||||||
func TestDeliverHTTP_CutsStoredResponseAtMaxBodyLog(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Padded so the cut falls five bytes before the end of the
|
|
||||||
// echoed webhook URL.
|
|
||||||
const (
|
|
||||||
severedTail = 5
|
|
||||||
overshoot = 100000
|
|
||||||
)
|
|
||||||
|
|
||||||
sent := strings.Repeat(
|
|
||||||
"A",
|
|
||||||
delivery.ExportMaxBodyLog-len(slackWebhookURL)+
|
|
||||||
severedTail,
|
|
||||||
) + slackWebhookURL + strings.Repeat("Z", overshoot)
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusBadGateway)
|
|
||||||
_, _ = io.WriteString(w, sent)
|
|
||||||
},
|
|
||||||
))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
cfgJSON := iHTTPConfig(ts.URL)
|
|
||||||
|
|
||||||
event := iSeedEvent(
|
|
||||||
t, s.WebhookDB, s.WebhookID, `{"cut":"test"}`,
|
|
||||||
)
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
|
|
||||||
del := iSeedDelivery(
|
|
||||||
t, s.WebhookDB, event.ID, targetID,
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
|
|
||||||
bodyStr := event.Body
|
|
||||||
task := iTask(
|
|
||||||
del, event, s.WebhookID, targetID,
|
|
||||||
"cut-target", cfgJSON, 0, 1, &bodyStr,
|
|
||||||
)
|
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(context.TODO(), &task)
|
|
||||||
|
|
||||||
results := iResults(t, s.WebhookDB, del.ID)
|
|
||||||
require.Len(t, results, 1)
|
|
||||||
|
|
||||||
stored := results[0].ResponseBody
|
|
||||||
|
|
||||||
assert.Len(
|
|
||||||
t, stored, delivery.ExportMaxBodyLog,
|
|
||||||
"an oversized response is stored at exactly the cap",
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t, sent[:delivery.ExportMaxBodyLog], stored,
|
|
||||||
)
|
|
||||||
assert.NotContains(
|
|
||||||
t, stored, slackWebhookURL,
|
|
||||||
"the echoed URL is severed by the cut",
|
|
||||||
)
|
|
||||||
assert.Contains(
|
|
||||||
t, stored, "T00000000",
|
|
||||||
"the severed prefix still carries the credential",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// iSeedEventAndDelivery creates event + delivery
|
// iSeedEventAndDelivery creates event + delivery
|
||||||
// for standalone tests.
|
// for standalone tests.
|
||||||
func iSeedEventAndDelivery(
|
func iSeedEventAndDelivery(
|
||||||
|
|||||||
@@ -27,13 +27,6 @@ const (
|
|||||||
// and a ready deliveryCh are chosen between at random and a
|
// and a ready deliveryCh are chosen between at random and a
|
||||||
// doomed pool still delivers.
|
// doomed pool still delivers.
|
||||||
hookSettleDelay = 250 * time.Millisecond
|
hookSettleDelay = 250 * time.Millisecond
|
||||||
|
|
||||||
// wedgeStopTimeout is the stop timeout a wedged-shutdown test
|
|
||||||
// hands OnStop, standing in for fx's StopTimeout. The test
|
|
||||||
// asserts only that the hook returns at all, and allows it
|
|
||||||
// hookStopTimeout — forty times this budget — to do so, so no
|
|
||||||
// assertion here races the wall clock.
|
|
||||||
wedgeStopTimeout = 250 * time.Millisecond
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// recordingLifecycle is a minimal fx.Lifecycle that records the
|
// recordingLifecycle is a minimal fx.Lifecycle that records the
|
||||||
@@ -47,44 +40,6 @@ func (l *recordingLifecycle) Append(h fx.Hook) {
|
|||||||
l.hooks = append(l.hooks, h)
|
l.hooks = append(l.hooks, h)
|
||||||
}
|
}
|
||||||
|
|
||||||
// requireStopHookExpires drives hook.OnStop with a stop context
|
|
||||||
// that expires while a wedged goroutine is still running, and
|
|
||||||
// requires the hook to return the deadline error naming
|
|
||||||
// component instead of blocking on the WaitGroup forever.
|
|
||||||
func requireStopHookExpires(
|
|
||||||
t *testing.T, hook fx.Hook, component string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
stopCtx, cancel := context.WithTimeout(
|
|
||||||
context.Background(), wedgeStopTimeout,
|
|
||||||
)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
var stopErr error
|
|
||||||
|
|
||||||
stopped := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
defer close(stopped)
|
|
||||||
|
|
||||||
stopErr = hook.OnStop(stopCtx)
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-stopped:
|
|
||||||
case <-time.After(hookStopTimeout):
|
|
||||||
t.Fatal(
|
|
||||||
"OnStop did not return: it discarded the stop " +
|
|
||||||
"context and is waiting on a wedged goroutine " +
|
|
||||||
"that will never observe cancellation",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
require.ErrorIs(t, stopErr, context.DeadlineExceeded)
|
|
||||||
require.ErrorContains(t, stopErr, component)
|
|
||||||
}
|
|
||||||
|
|
||||||
// startEngineViaHook drives the genuine fx hooks the application
|
// startEngineViaHook drives the genuine fx hooks the application
|
||||||
// registers for the engine, handing OnStart a context that is
|
// registers for the engine, handing OnStart a context that is
|
||||||
// already done, and returns only once a pool that inherited that
|
// already done, and returns only once a pool that inherited that
|
||||||
@@ -242,30 +197,3 @@ func TestEngine_StopHookStopsWorkers(t *testing.T) {
|
|||||||
"a stopped engine must not deliver anything",
|
"a stopped engine must not deliver anything",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestEngine_StopHookHonoursStopTimeout is the regression test
|
|
||||||
// for a shutdown that could never complete. fx hands OnStop a
|
|
||||||
// context carrying the application's stop timeout; an OnStop
|
|
||||||
// that discards it and calls wg.Wait() bare hangs the process
|
|
||||||
// forever on a single worker stuck inside a delivery target that
|
|
||||||
// never returns — precisely when a bounded shutdown matters
|
|
||||||
// most.
|
|
||||||
//
|
|
||||||
// The wedged goroutine here never observes cancellation, so the
|
|
||||||
// hook can only return by honouring its context, and it must say
|
|
||||||
// so rather than reporting a clean stop.
|
|
||||||
func TestEngine_StopHookHonoursStopTimeout(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
|
|
||||||
lc := startEngineViaHook(t, s.Engine)
|
|
||||||
|
|
||||||
release := make(chan struct{})
|
|
||||||
|
|
||||||
t.Cleanup(func() { close(release) })
|
|
||||||
|
|
||||||
s.Engine.ExportWedgeWorker(release)
|
|
||||||
|
|
||||||
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,157 +0,0 @@
|
|||||||
package delivery_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
_ "modernc.org/sqlite"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// assertNoTargetRows opens the per-webhook database file directly,
|
|
||||||
// outside GORM, and fails if its targets table holds any rows.
|
|
||||||
// Target config is the credential for slack and http targets, and
|
|
||||||
// event databases are the files that get backed up and handed
|
|
||||||
// around.
|
|
||||||
func assertNoTargetRows(t *testing.T, dbPath string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
sqlDB, err := sql.Open(
|
|
||||||
"sqlite", fmt.Sprintf("file:%s?mode=ro", dbPath),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
defer func() { _ = sqlDB.Close() }()
|
|
||||||
|
|
||||||
var tables int
|
|
||||||
|
|
||||||
require.NoError(t, sqlDB.QueryRowContext(
|
|
||||||
t.Context(),
|
|
||||||
"SELECT count(*) FROM sqlite_master "+
|
|
||||||
"WHERE type = 'table' AND name = 'targets'",
|
|
||||||
).Scan(&tables))
|
|
||||||
|
|
||||||
if tables == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
var rows int
|
|
||||||
|
|
||||||
require.NoError(t, sqlDB.QueryRowContext(
|
|
||||||
t.Context(), "SELECT count(*) FROM targets",
|
|
||||||
).Scan(&rows))
|
|
||||||
|
|
||||||
assert.Zero(
|
|
||||||
t, rows,
|
|
||||||
"per-webhook event database must hold no target rows",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventDBHoldsNoTargetRows drives a delivery and then a retry
|
|
||||||
// through the real engine write paths and asserts neither leaves a
|
|
||||||
// target row behind in events-*.db.
|
|
||||||
func TestEventDBHoldsNoTargetRows(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
},
|
|
||||||
))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
cfg := iHTTPConfig(ts.URL)
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
dbPath := s.DBMgr.DBPath(s.WebhookID)
|
|
||||||
|
|
||||||
event := iSeedEvent(
|
|
||||||
t, s.WebhookDB, s.WebhookID, `{"leak":"none"}`,
|
|
||||||
)
|
|
||||||
body := event.Body
|
|
||||||
|
|
||||||
// A new delivery.
|
|
||||||
d := iSeedDelivery(
|
|
||||||
t, s.WebhookDB, event.ID, targetID,
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
task := iTask(
|
|
||||||
d, event, s.WebhookID, targetID,
|
|
||||||
"leaky-target", cfg, 5, 1, &body,
|
|
||||||
)
|
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(context.TODO(), &task)
|
|
||||||
|
|
||||||
iAssertStatus(
|
|
||||||
t, s.WebhookDB, d.ID,
|
|
||||||
database.DeliveryStatusDelivered,
|
|
||||||
)
|
|
||||||
assertNoTargetRows(t, dbPath)
|
|
||||||
|
|
||||||
// A retry.
|
|
||||||
rd := iSeedDelivery(
|
|
||||||
t, s.WebhookDB, event.ID, targetID,
|
|
||||||
database.DeliveryStatusRetrying,
|
|
||||||
)
|
|
||||||
rTask := iTask(
|
|
||||||
rd, event, s.WebhookID, targetID,
|
|
||||||
"leaky-target", cfg, 5, 2, &body,
|
|
||||||
)
|
|
||||||
|
|
||||||
s.Engine.ExportProcessRetryTask(context.TODO(), &rTask)
|
|
||||||
|
|
||||||
iAssertStatus(
|
|
||||||
t, s.WebhookDB, rd.ID,
|
|
||||||
database.DeliveryStatusDelivered,
|
|
||||||
)
|
|
||||||
assertNoTargetRows(t, dbPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventDBHoldsNoTargetRowsOnFailedDelivery covers the failure
|
|
||||||
// write path, which updates the delivery to failed and records a
|
|
||||||
// result, rather than the success path above.
|
|
||||||
func TestEventDBHoldsNoTargetRowsOnFailedDelivery(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
},
|
|
||||||
))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
cfg := iHTTPConfig(ts.URL)
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
|
|
||||||
event := iSeedEvent(
|
|
||||||
t, s.WebhookDB, s.WebhookID, `{"leak":"none"}`,
|
|
||||||
)
|
|
||||||
body := event.Body
|
|
||||||
|
|
||||||
d := iSeedDelivery(
|
|
||||||
t, s.WebhookDB, event.ID, targetID,
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
task := iTask(
|
|
||||||
d, event, s.WebhookID, targetID,
|
|
||||||
"leaky-target", cfg, 0, 1, &body,
|
|
||||||
)
|
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(context.TODO(), &task)
|
|
||||||
|
|
||||||
iAssertStatus(
|
|
||||||
t, s.WebhookDB, d.ID,
|
|
||||||
database.DeliveryStatusFailed,
|
|
||||||
)
|
|
||||||
assertNoTargetRows(t, s.DBMgr.DBPath(s.WebhookID))
|
|
||||||
}
|
|
||||||
@@ -5,14 +5,11 @@ import (
|
|||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
|
||||||
"net/url"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/metrics"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrExportArchiveWriterEvicted exposes the sentinel returned by
|
// ErrExportArchiveWriterEvicted exposes the sentinel returned by
|
||||||
@@ -27,12 +24,6 @@ const (
|
|||||||
ExportRetryChannelSize = retryChannelSize
|
ExportRetryChannelSize = retryChannelSize
|
||||||
ExportDefaultFailureThreshold = defaultFailureThreshold
|
ExportDefaultFailureThreshold = defaultFailureThreshold
|
||||||
ExportDefaultCooldown = defaultCooldown
|
ExportDefaultCooldown = defaultCooldown
|
||||||
|
|
||||||
// ExportMaxBodyLog is the cap the engine applies to a
|
|
||||||
// recorded response body. The event log's handling of a cut
|
|
||||||
// response is written against this number, so a test has to
|
|
||||||
// be able to name it.
|
|
||||||
ExportMaxBodyLog = maxBodyLog
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// ExportIsBlockedIP exposes isBlockedIP for testing.
|
// ExportIsBlockedIP exposes isBlockedIP for testing.
|
||||||
@@ -40,26 +31,6 @@ func ExportIsBlockedIP(ip net.IP) bool {
|
|||||||
return isBlockedIP(ip)
|
return isBlockedIP(ip)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewTestGuard builds an SSRF Guard from an explicit egress
|
|
||||||
// allowlist, without going through config. Passing no prefixes
|
|
||||||
// yields the default guard, which blocks every private/reserved
|
|
||||||
// range.
|
|
||||||
func NewTestGuard(allowed ...netip.Prefix) *Guard {
|
|
||||||
return &Guard{allowed: allowed}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportCheckIP exposes the guard's single decision point, so a
|
|
||||||
// test can assert the policy both the validator and the dialer
|
|
||||||
// inherit without needing a live destination.
|
|
||||||
func (g *Guard) ExportCheckIP(ip net.IP) error {
|
|
||||||
return g.checkIP(ip)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportAlwaysBlockedNetworks exposes alwaysBlockedNetworks.
|
|
||||||
func ExportAlwaysBlockedNetworks() []*net.IPNet {
|
|
||||||
return alwaysBlockedNetworks
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportBlockedNetworks exposes blockedNetworks.
|
// ExportBlockedNetworks exposes blockedNetworks.
|
||||||
func ExportBlockedNetworks() []*net.IPNet {
|
func ExportBlockedNetworks() []*net.IPNet {
|
||||||
return blockedNetworks
|
return blockedNetworks
|
||||||
@@ -70,17 +41,6 @@ func ExportIsForwardableHeader(name string) bool {
|
|||||||
return isForwardableHeader(name)
|
return isForwardableHeader(name)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportApplyRequestHeaders exposes applyRequestHeaders, so a test
|
|
||||||
// can inspect the header set an outbound delivery actually carries
|
|
||||||
// and the origin-scoped names it reports for the redirect policy.
|
|
||||||
func ExportApplyRequestHeaders(
|
|
||||||
req *http.Request,
|
|
||||||
event *database.Event,
|
|
||||||
cfg *HTTPTargetConfig,
|
|
||||||
) []string {
|
|
||||||
return applyRequestHeaders(req, event, cfg)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportTruncate exposes truncate for testing.
|
// ExportTruncate exposes truncate for testing.
|
||||||
func ExportTruncate(s string, maxLen int) string {
|
func ExportTruncate(s string, maxLen int) string {
|
||||||
return truncate(s, maxLen)
|
return truncate(s, maxLen)
|
||||||
@@ -188,27 +148,12 @@ func (e *Engine) ExportDoHTTPRequest(
|
|||||||
return e.httpTarget.doHTTPRequest(ctx, cfg, event)
|
return e.httpTarget.doHTTPRequest(ctx, cfg, event)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportClientForRequest exposes the http target's
|
// ExportClientForConfig exposes the http target's
|
||||||
// clientForRequest.
|
// clientForConfig.
|
||||||
func (e *Engine) ExportClientForRequest(
|
func (e *Engine) ExportClientForConfig(
|
||||||
cfg *HTTPTargetConfig,
|
cfg *HTTPTargetConfig,
|
||||||
originScoped []string,
|
|
||||||
) *http.Client {
|
) *http.Client {
|
||||||
return e.httpTarget.clientForRequest(cfg, originScoped)
|
return e.httpTarget.clientForConfig(cfg)
|
||||||
}
|
|
||||||
|
|
||||||
// ErrExportTooManyRedirects exposes the sentinel the redirect
|
|
||||||
// policy returns once a chain exceeds the hop cap. It carries the
|
|
||||||
// Err prefix rather than this file's usual Export one because it
|
|
||||||
// is a sentinel error.
|
|
||||||
var ErrExportTooManyRedirects = errTooManyRedirects
|
|
||||||
|
|
||||||
// ExportMaxDeliveryRedirects exposes the redirect hop cap.
|
|
||||||
const ExportMaxDeliveryRedirects = maxDeliveryRedirects
|
|
||||||
|
|
||||||
// ExportSameDeliveryOrigin exposes sameDeliveryOrigin.
|
|
||||||
func ExportSameDeliveryOrigin(origin, dest *url.URL) bool {
|
|
||||||
return sameDeliveryOrigin(origin, dest)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportClient returns the http target's shared HTTP client.
|
// ExportClient returns the http target's shared HTTP client.
|
||||||
@@ -271,19 +216,8 @@ func (e *Engine) ExportRegisterHooks(lc fx.Lifecycle) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ExportStop exposes stop for testing.
|
// ExportStop exposes stop for testing.
|
||||||
func (e *Engine) ExportStop(ctx context.Context) error {
|
func (e *Engine) ExportStop() {
|
||||||
return e.stop(ctx)
|
e.stop()
|
||||||
}
|
|
||||||
|
|
||||||
// ExportWedgeWorker adds a goroutine to the engine's WaitGroup
|
|
||||||
// that never observes cancellation and returns only when release
|
|
||||||
// is closed. It stands in for a worker stuck inside a delivery
|
|
||||||
// target that never returns, which is the only way stop can be
|
|
||||||
// made to outlast its context.
|
|
||||||
func (e *Engine) ExportWedgeWorker(release <-chan struct{}) {
|
|
||||||
e.wg.Go(func() {
|
|
||||||
<-release
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportDeliveryCh returns the delivery channel.
|
// ExportDeliveryCh returns the delivery channel.
|
||||||
@@ -308,7 +242,6 @@ func NewTestEngine(
|
|||||||
deliveryCh: make(chan Task, deliveryChannelSize),
|
deliveryCh: make(chan Task, deliveryChannelSize),
|
||||||
retryCh: make(chan Task, retryChannelSize),
|
retryCh: make(chan Task, retryChannelSize),
|
||||||
workers: workers,
|
workers: workers,
|
||||||
mtr: metrics.Default(),
|
|
||||||
}
|
}
|
||||||
e.initTargets(client)
|
e.initTargets(client)
|
||||||
|
|
||||||
@@ -323,7 +256,6 @@ func NewTestEngineSmallRetry(
|
|||||||
e := &Engine{
|
e := &Engine{
|
||||||
log: log,
|
log: log,
|
||||||
retryCh: make(chan Task, 1),
|
retryCh: make(chan Task, 1),
|
||||||
mtr: metrics.Default(),
|
|
||||||
}
|
}
|
||||||
e.initTargets(nil)
|
e.initTargets(nil)
|
||||||
|
|
||||||
@@ -346,25 +278,12 @@ func NewTestEngineWithDB(
|
|||||||
deliveryCh: make(chan Task, deliveryChannelSize),
|
deliveryCh: make(chan Task, deliveryChannelSize),
|
||||||
retryCh: make(chan Task, retryChannelSize),
|
retryCh: make(chan Task, retryChannelSize),
|
||||||
workers: workers,
|
workers: workers,
|
||||||
mtr: metrics.Default(),
|
|
||||||
}
|
}
|
||||||
e.initTargets(client)
|
e.initTargets(client)
|
||||||
|
|
||||||
return e
|
return e
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportSetMetrics substitutes the engine's metric set, so a test can
|
|
||||||
// assert on collectors registered on a private registry instead of
|
|
||||||
// the process-wide ones every other test is also moving.
|
|
||||||
func (e *Engine) ExportSetMetrics(mtr *metrics.Set) {
|
|
||||||
e.mtr = mtr
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportSampleQueueDepths runs one queue depth sample synchronously.
|
|
||||||
func (e *Engine) ExportSampleQueueDepths(ctx context.Context) {
|
|
||||||
e.sampleQueueDepths(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewTestCircuitBreaker creates a CircuitBreaker with
|
// NewTestCircuitBreaker creates a CircuitBreaker with
|
||||||
// custom settings for testing.
|
// custom settings for testing.
|
||||||
func NewTestCircuitBreaker(
|
func NewTestCircuitBreaker(
|
||||||
@@ -599,19 +518,8 @@ func (s *ArchiveSweeper) ExportRegisterHooks(lc fx.Lifecycle) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ExportStop stops the sweeper's background loop for tests.
|
// ExportStop stops the sweeper's background loop for tests.
|
||||||
func (s *ArchiveSweeper) ExportStop(ctx context.Context) error {
|
func (s *ArchiveSweeper) ExportStop() {
|
||||||
return s.stop(ctx)
|
s.stop()
|
||||||
}
|
|
||||||
|
|
||||||
// ExportWedgeLoop adds a goroutine to the sweeper's WaitGroup
|
|
||||||
// that never observes cancellation and returns only when release
|
|
||||||
// is closed. It stands in for a prune stuck on a locked archive.
|
|
||||||
func (s *ArchiveSweeper) ExportWedgeLoop(
|
|
||||||
release <-chan struct{},
|
|
||||||
) {
|
|
||||||
s.wg.Go(func() {
|
|
||||||
<-release
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportSetInterval overrides the sweep interval for tests.
|
// ExportSetInterval overrides the sweep interval for tests.
|
||||||
|
|||||||
@@ -1,545 +0,0 @@
|
|||||||
package delivery_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
dto "github.com/prometheus/client_model/go"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
"sneak.berlin/go/webhooker/internal/metrics"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Metric names as exposed on /metrics.
|
|
||||||
const (
|
|
||||||
mAttempts = "webhooker_delivery_attempts_total"
|
|
||||||
mSucceeded = "webhooker_deliveries_succeeded_total"
|
|
||||||
mFailed = "webhooker_deliveries_failed_total"
|
|
||||||
mRetries = "webhooker_delivery_retries_total"
|
|
||||||
mDuration = "webhooker_delivery_duration_seconds"
|
|
||||||
mPending = "webhooker_deliveries_pending"
|
|
||||||
mRetrying = "webhooker_deliveries_retrying"
|
|
||||||
mBreakers = "webhooker_circuit_breakers_open"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
mTypeHTTP = "http"
|
|
||||||
mTypeLog = "log"
|
|
||||||
mTypeUnknown = "unknown"
|
|
||||||
)
|
|
||||||
|
|
||||||
// mIsolate gives the setup's engine a metric set registered on a
|
|
||||||
// private registry. The process-wide collectors are moved by every
|
|
||||||
// other delivery test running in parallel, so exact assertions are
|
|
||||||
// only possible against a registry this test owns.
|
|
||||||
func mIsolate(
|
|
||||||
t *testing.T, s iSetup,
|
|
||||||
) *prometheus.Registry {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
reg := prometheus.NewRegistry()
|
|
||||||
s.Engine.ExportSetMetrics(metrics.New(reg))
|
|
||||||
|
|
||||||
return reg
|
|
||||||
}
|
|
||||||
|
|
||||||
// mFind returns the series of the named metric carrying the given
|
|
||||||
// target_type label.
|
|
||||||
func mFind(
|
|
||||||
t *testing.T,
|
|
||||||
reg *prometheus.Registry,
|
|
||||||
name, targetType string,
|
|
||||||
) *dto.Metric {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
families, err := reg.Gather()
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
for _, fam := range families {
|
|
||||||
if fam.GetName() != name {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, m := range fam.GetMetric() {
|
|
||||||
for _, label := range m.GetLabel() {
|
|
||||||
if label.GetName() == "target_type" &&
|
|
||||||
label.GetValue() == targetType {
|
|
||||||
return m
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Fatalf(
|
|
||||||
"metric %s{target_type=%q} not found",
|
|
||||||
name, targetType,
|
|
||||||
)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func mCounter(
|
|
||||||
t *testing.T,
|
|
||||||
reg *prometheus.Registry,
|
|
||||||
name, targetType string,
|
|
||||||
) float64 {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return mFind(t, reg, name, targetType).
|
|
||||||
GetCounter().GetValue()
|
|
||||||
}
|
|
||||||
|
|
||||||
func mGauge(
|
|
||||||
t *testing.T,
|
|
||||||
reg *prometheus.Registry,
|
|
||||||
name, targetType string,
|
|
||||||
) float64 {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return mFind(t, reg, name, targetType).
|
|
||||||
GetGauge().GetValue()
|
|
||||||
}
|
|
||||||
|
|
||||||
// mHTTPDurations returns how many samples the delivery duration
|
|
||||||
// histogram holds for the http target type, which is the type every
|
|
||||||
// test here times.
|
|
||||||
func mHTTPDurations(
|
|
||||||
t *testing.T, reg *prometheus.Registry,
|
|
||||||
) uint64 {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return mFind(t, reg, mDuration, mTypeHTTP).
|
|
||||||
GetHistogram().GetSampleCount()
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDeliveryMetrics_SuccessAndRetryExhaustion drives one delivery
|
|
||||||
// that succeeds and one that fails every attempt until its retries
|
|
||||||
// are exhausted, and asserts every delivery counter across both.
|
|
||||||
func TestDeliveryMetrics_SuccessAndRetryExhaustion(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
reg := mIsolate(t, s)
|
|
||||||
|
|
||||||
mDeliverOK(t, s)
|
|
||||||
|
|
||||||
assert.InDelta(t, 1.0,
|
|
||||||
mCounter(t, reg, mAttempts, mTypeHTTP), 0)
|
|
||||||
assert.InDelta(t, 1.0,
|
|
||||||
mCounter(t, reg, mSucceeded, mTypeHTTP), 0)
|
|
||||||
assert.InDelta(t, 0.0,
|
|
||||||
mCounter(t, reg, mFailed, mTypeHTTP), 0)
|
|
||||||
assert.InDelta(t, 0.0,
|
|
||||||
mCounter(t, reg, mRetries, mTypeHTTP), 0)
|
|
||||||
assert.Equal(t, uint64(1),
|
|
||||||
mHTTPDurations(t, reg))
|
|
||||||
|
|
||||||
mExhaustRetries(t, s)
|
|
||||||
|
|
||||||
// Two further attempts: the first is retried, the second is
|
|
||||||
// the last one allowed and fails the delivery terminally.
|
|
||||||
assert.InDelta(t, 3.0,
|
|
||||||
mCounter(t, reg, mAttempts, mTypeHTTP), 0)
|
|
||||||
assert.InDelta(t, 1.0,
|
|
||||||
mCounter(t, reg, mSucceeded, mTypeHTTP), 0)
|
|
||||||
assert.InDelta(t, 1.0,
|
|
||||||
mCounter(t, reg, mRetries, mTypeHTTP), 0)
|
|
||||||
assert.InDelta(t, 1.0,
|
|
||||||
mCounter(t, reg, mFailed, mTypeHTTP), 0)
|
|
||||||
assert.Equal(t, uint64(3),
|
|
||||||
mHTTPDurations(t, reg))
|
|
||||||
|
|
||||||
// Two consecutive failures are below the trip threshold.
|
|
||||||
assert.InDelta(t, 0.0,
|
|
||||||
mGauge(t, reg, mBreakers, mTypeHTTP), 0)
|
|
||||||
|
|
||||||
// The label is the target type and nothing finer: two http
|
|
||||||
// targets shared one series, and no other type's moved.
|
|
||||||
assert.InDelta(t, 0.0,
|
|
||||||
mCounter(t, reg, mAttempts, mTypeLog), 0)
|
|
||||||
assert.InDelta(t, 0.0,
|
|
||||||
mCounter(t, reg, mFailed, mTypeLog), 0)
|
|
||||||
}
|
|
||||||
|
|
||||||
// mDeliverOK delivers one event to a target that answers 200.
|
|
||||||
func mDeliverOK(t *testing.T, s iSetup) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
},
|
|
||||||
))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
event := iSeedEvent(
|
|
||||||
t, s.WebhookDB, s.WebhookID, `{"ok":true}`,
|
|
||||||
)
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
|
|
||||||
d := iSeedDelivery(
|
|
||||||
t, s.WebhookDB, event.ID, targetID,
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
|
|
||||||
body := event.Body
|
|
||||||
task := iTask(
|
|
||||||
d, event, s.WebhookID, targetID,
|
|
||||||
"metrics-ok", iHTTPConfig(ts.URL), 3, 1, &body,
|
|
||||||
)
|
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(context.TODO(), &task)
|
|
||||||
|
|
||||||
iAssertStatus(t, s.WebhookDB, d.ID,
|
|
||||||
database.DeliveryStatusDelivered,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// mExhaustRetries delivers to a target that answers 500 with a
|
|
||||||
// two-attempt budget, driving both attempts so the delivery ends
|
|
||||||
// terminally failed.
|
|
||||||
func mExhaustRetries(t *testing.T, s iSetup) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
},
|
|
||||||
))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
event := iSeedEvent(
|
|
||||||
t, s.WebhookDB, s.WebhookID, `{"ok":false}`,
|
|
||||||
)
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
|
|
||||||
d := iSeedDelivery(
|
|
||||||
t, s.WebhookDB, event.ID, targetID,
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
|
|
||||||
body := event.Body
|
|
||||||
cfg := iHTTPConfig(ts.URL)
|
|
||||||
|
|
||||||
first := iTask(
|
|
||||||
d, event, s.WebhookID, targetID,
|
|
||||||
"metrics-fail", cfg, 2, 1, &body,
|
|
||||||
)
|
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(context.TODO(), &first)
|
|
||||||
|
|
||||||
iAssertStatus(t, s.WebhookDB, d.ID,
|
|
||||||
database.DeliveryStatusRetrying,
|
|
||||||
)
|
|
||||||
|
|
||||||
// The engine's own scheduler would re-enqueue this after the
|
|
||||||
// backoff; driving the second attempt directly keeps the test
|
|
||||||
// deterministic and off the wall clock.
|
|
||||||
second := iTask(
|
|
||||||
d, event, s.WebhookID, targetID,
|
|
||||||
"metrics-fail", cfg, 2, 2, &body,
|
|
||||||
)
|
|
||||||
|
|
||||||
s.Engine.ExportProcessRetryTask(
|
|
||||||
context.TODO(), &second,
|
|
||||||
)
|
|
||||||
|
|
||||||
iAssertStatus(t, s.WebhookDB, d.ID,
|
|
||||||
database.DeliveryStatusFailed,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDeliveryMetrics_CircuitBreakerGauge proves the open-breaker
|
|
||||||
// gauge follows a breaker that trips.
|
|
||||||
func TestDeliveryMetrics_CircuitBreakerGauge(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
reg := mIsolate(t, s)
|
|
||||||
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
},
|
|
||||||
))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
event := iSeedEvent(
|
|
||||||
t, s.WebhookDB, s.WebhookID, `{"trip":true}`,
|
|
||||||
)
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
|
|
||||||
d := iSeedDelivery(
|
|
||||||
t, s.WebhookDB, event.ID, targetID,
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
|
|
||||||
body := event.Body
|
|
||||||
cfg := iHTTPConfig(ts.URL)
|
|
||||||
|
|
||||||
// A retry budget above the failure threshold, so the breaker
|
|
||||||
// rather than the budget is what stops the delivery.
|
|
||||||
maxRetries := delivery.ExportDefaultFailureThreshold + 5
|
|
||||||
|
|
||||||
first := iTask(
|
|
||||||
d, event, s.WebhookID, targetID,
|
|
||||||
"metrics-trip", cfg, maxRetries, 1, &body,
|
|
||||||
)
|
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(context.TODO(), &first)
|
|
||||||
|
|
||||||
assert.InDelta(t, 0.0,
|
|
||||||
mGauge(t, reg, mBreakers, mTypeHTTP), 0)
|
|
||||||
|
|
||||||
for attempt := 2; attempt <= delivery.
|
|
||||||
ExportDefaultFailureThreshold; attempt++ {
|
|
||||||
task := iTask(
|
|
||||||
d, event, s.WebhookID, targetID,
|
|
||||||
"metrics-trip", cfg, maxRetries, attempt, &body,
|
|
||||||
)
|
|
||||||
|
|
||||||
s.Engine.ExportProcessRetryTask(
|
|
||||||
context.TODO(), &task,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.InDelta(t, 1.0,
|
|
||||||
mGauge(t, reg, mBreakers, mTypeHTTP), 0)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDeliveryMetrics_BreakerBlockedIsNotAnAttempt proves a delivery
|
|
||||||
// an open circuit breaker refuses is neither counted as an attempt
|
|
||||||
// nor observed in the duration histogram.
|
|
||||||
//
|
|
||||||
// It sends nothing and records no result row, so counting it would
|
|
||||||
// climb the attempts counter with no traffic behind it and pull the
|
|
||||||
// duration quantiles down with near-zero samples for as long as the
|
|
||||||
// breaker stayed open — the metric moving the wrong way during the
|
|
||||||
// outage it exists to reveal.
|
|
||||||
func TestDeliveryMetrics_BreakerBlockedIsNotAnAttempt(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
reg := mIsolate(t, s)
|
|
||||||
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusInternalServerError)
|
|
||||||
},
|
|
||||||
))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
event := iSeedEvent(
|
|
||||||
t, s.WebhookDB, s.WebhookID, `{"blocked":true}`,
|
|
||||||
)
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
|
|
||||||
d := iSeedDelivery(
|
|
||||||
t, s.WebhookDB, event.ID, targetID,
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
|
|
||||||
body := event.Body
|
|
||||||
cfg := iHTTPConfig(ts.URL)
|
|
||||||
maxRetries := delivery.ExportDefaultFailureThreshold + 5
|
|
||||||
|
|
||||||
first := iTask(
|
|
||||||
d, event, s.WebhookID, targetID,
|
|
||||||
"metrics-blocked", cfg, maxRetries, 1, &body,
|
|
||||||
)
|
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(context.TODO(), &first)
|
|
||||||
|
|
||||||
for attempt := 2; attempt <= delivery.
|
|
||||||
ExportDefaultFailureThreshold; attempt++ {
|
|
||||||
task := iTask(
|
|
||||||
d, event, s.WebhookID, targetID,
|
|
||||||
"metrics-blocked", cfg, maxRetries, attempt, &body,
|
|
||||||
)
|
|
||||||
|
|
||||||
s.Engine.ExportProcessRetryTask(context.TODO(), &task)
|
|
||||||
}
|
|
||||||
|
|
||||||
require.InDelta(t, 1.0,
|
|
||||||
mGauge(t, reg, mBreakers, mTypeHTTP), 0,
|
|
||||||
"breaker should be open before the blocked attempt")
|
|
||||||
|
|
||||||
threshold := float64(
|
|
||||||
delivery.ExportDefaultFailureThreshold,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.InDelta(t, threshold,
|
|
||||||
mCounter(t, reg, mAttempts, mTypeHTTP), 0)
|
|
||||||
assert.Equal(t, uint64(threshold),
|
|
||||||
mHTTPDurations(t, reg))
|
|
||||||
|
|
||||||
retriesBefore := mCounter(t, reg, mRetries, mTypeHTTP)
|
|
||||||
|
|
||||||
blocked := iTask(
|
|
||||||
d, event, s.WebhookID, targetID,
|
|
||||||
"metrics-blocked", cfg, maxRetries,
|
|
||||||
delivery.ExportDefaultFailureThreshold+1, &body,
|
|
||||||
)
|
|
||||||
|
|
||||||
s.Engine.ExportProcessRetryTask(context.TODO(), &blocked)
|
|
||||||
|
|
||||||
// The breaker refused it: rescheduled, so the retry counter
|
|
||||||
// moved, but nothing was attempted or timed.
|
|
||||||
assert.InDelta(t, retriesBefore+1,
|
|
||||||
mCounter(t, reg, mRetries, mTypeHTTP), 0)
|
|
||||||
assert.InDelta(t, threshold,
|
|
||||||
mCounter(t, reg, mAttempts, mTypeHTTP), 0)
|
|
||||||
assert.Equal(t, uint64(threshold),
|
|
||||||
mHTTPDurations(t, reg))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDeliveryMetrics_OrphanedRetryFailureLabelled proves the
|
|
||||||
// terminal failure of a delivery whose target no longer retries is
|
|
||||||
// counted against the target's real type, not against unknown. The
|
|
||||||
// type is threaded in as an argument because populating d.Target on
|
|
||||||
// that path would write the target row into the per-webhook database
|
|
||||||
// (https://git.eeqj.de/sneak/webhooker/issues/206).
|
|
||||||
func TestDeliveryMetrics_OrphanedRetryFailureLabelled(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
reg := mIsolate(t, s)
|
|
||||||
|
|
||||||
iCreateWebhook(
|
|
||||||
t, s.MainDB, s.WebhookID, "orphaned-label",
|
|
||||||
)
|
|
||||||
|
|
||||||
deliveryID := iSeedRetryingWithType(
|
|
||||||
t, s, database.TargetTypeLog,
|
|
||||||
)
|
|
||||||
|
|
||||||
s.Engine.ExportSweepWebhookRetries(
|
|
||||||
context.Background(), s.WebhookID,
|
|
||||||
)
|
|
||||||
|
|
||||||
iAssertStatus(t, s.WebhookDB, deliveryID,
|
|
||||||
database.DeliveryStatusFailed,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.InDelta(t, 1.0,
|
|
||||||
mCounter(t, reg, mFailed, mTypeLog), 0)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDeliveryMetrics_QueueDepthGauges proves the sampler publishes
|
|
||||||
// the queued deliveries it finds in the per-webhook databases, and
|
|
||||||
// that a drained queue reads zero rather than keeping its last
|
|
||||||
// value.
|
|
||||||
func TestDeliveryMetrics_QueueDepthGauges(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
reg := mIsolate(t, s)
|
|
||||||
|
|
||||||
iCreateWebhook(
|
|
||||||
t, s.MainDB, s.WebhookID, "queue-depth",
|
|
||||||
)
|
|
||||||
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
|
|
||||||
iCreateTarget(t, s.MainDB, targetID, s.WebhookID,
|
|
||||||
"queue-depth-target", database.TargetTypeHTTP,
|
|
||||||
iHTTPConfig("https://example.com/hook"), 3,
|
|
||||||
)
|
|
||||||
|
|
||||||
event := iSeedEvent(
|
|
||||||
t, s.WebhookDB, s.WebhookID, `{"queued":true}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
pending := iSeedDelivery(
|
|
||||||
t, s.WebhookDB, event.ID, targetID,
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
|
|
||||||
iSeedDelivery(
|
|
||||||
t, s.WebhookDB, event.ID, targetID,
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
|
|
||||||
retrying := iSeedDelivery(
|
|
||||||
t, s.WebhookDB, event.ID, targetID,
|
|
||||||
database.DeliveryStatusRetrying,
|
|
||||||
)
|
|
||||||
|
|
||||||
s.Engine.ExportSampleQueueDepths(context.Background())
|
|
||||||
|
|
||||||
assert.InDelta(t, 2.0,
|
|
||||||
mGauge(t, reg, mPending, mTypeHTTP), 0)
|
|
||||||
assert.InDelta(t, 1.0,
|
|
||||||
mGauge(t, reg, mRetrying, mTypeHTTP), 0)
|
|
||||||
assert.InDelta(t, 0.0,
|
|
||||||
mGauge(t, reg, mPending, mTypeLog), 0)
|
|
||||||
|
|
||||||
require.NoError(t, s.WebhookDB.
|
|
||||||
Model(&database.Delivery{}).
|
|
||||||
Where("id IN ?", []string{pending.ID, retrying.ID}).
|
|
||||||
Update(
|
|
||||||
"status", database.DeliveryStatusDelivered,
|
|
||||||
).Error)
|
|
||||||
|
|
||||||
s.Engine.ExportSampleQueueDepths(context.Background())
|
|
||||||
|
|
||||||
assert.InDelta(t, 1.0,
|
|
||||||
mGauge(t, reg, mPending, mTypeHTTP), 0)
|
|
||||||
assert.InDelta(t, 0.0,
|
|
||||||
mGauge(t, reg, mRetrying, mTypeHTTP), 0)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDeliveryMetrics_QueueDepthDeletedTarget proves a backlog queued
|
|
||||||
// against a target that has since been deleted stays visible, in the
|
|
||||||
// unknown series, instead of being dropped. That backlog is the one
|
|
||||||
// nobody is watching, so losing it would defeat the queue-depth
|
|
||||||
// alerting this metric exists for.
|
|
||||||
func TestDeliveryMetrics_QueueDepthDeletedTarget(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
reg := mIsolate(t, s)
|
|
||||||
|
|
||||||
iCreateWebhook(
|
|
||||||
t, s.MainDB, s.WebhookID, "deleted-target",
|
|
||||||
)
|
|
||||||
|
|
||||||
// No target row is created: this is a delivery whose target was
|
|
||||||
// deleted out from under it.
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
|
|
||||||
event := iSeedEvent(
|
|
||||||
t, s.WebhookDB, s.WebhookID, `{"orphan":true}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
iSeedDelivery(
|
|
||||||
t, s.WebhookDB, event.ID, targetID,
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
|
|
||||||
iSeedDelivery(
|
|
||||||
t, s.WebhookDB, event.ID, targetID,
|
|
||||||
database.DeliveryStatusRetrying,
|
|
||||||
)
|
|
||||||
|
|
||||||
s.Engine.ExportSampleQueueDepths(context.Background())
|
|
||||||
|
|
||||||
assert.InDelta(t, 1.0,
|
|
||||||
mGauge(t, reg, mPending, mTypeUnknown), 0)
|
|
||||||
assert.InDelta(t, 1.0,
|
|
||||||
mGauge(t, reg, mRetrying, mTypeUnknown), 0)
|
|
||||||
assert.InDelta(t, 0.0,
|
|
||||||
mGauge(t, reg, mPending, mTypeHTTP), 0)
|
|
||||||
}
|
|
||||||
@@ -1,196 +0,0 @@
|
|||||||
package delivery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// queueDepthSampleInterval is how often the pending and retrying
|
|
||||||
// queue depths are counted and published as gauges.
|
|
||||||
const queueDepthSampleInterval = 30 * time.Second
|
|
||||||
|
|
||||||
// queueDepthSampler publishes the pending and retrying queue depths
|
|
||||||
// on a timer for as long as the engine runs.
|
|
||||||
//
|
|
||||||
// The depths are counted out of the databases rather than tracked as
|
|
||||||
// deltas alongside the status transitions. A delta counter would have
|
|
||||||
// to be seeded correctly at startup from rows written by a previous
|
|
||||||
// process, and would drift permanently on any transition that failed
|
|
||||||
// to persist. Counting is the measurement that cannot go wrong, and
|
|
||||||
// it is the same whole-database walk the retry sweep already makes.
|
|
||||||
func (e *Engine) queueDepthSampler(ctx context.Context) {
|
|
||||||
defer e.wg.Done()
|
|
||||||
|
|
||||||
ticker := time.NewTicker(queueDepthSampleInterval)
|
|
||||||
defer ticker.Stop()
|
|
||||||
|
|
||||||
e.sampleQueueDepths(ctx)
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-ticker.C:
|
|
||||||
e.sampleQueueDepths(ctx)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sampleQueueDepths counts every queued delivery across all
|
|
||||||
// per-webhook databases and publishes the result.
|
|
||||||
func (e *Engine) sampleQueueDepths(ctx context.Context) {
|
|
||||||
if e.database == nil || e.dbManager == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
types, err := e.targetTypesByID()
|
|
||||||
if err != nil {
|
|
||||||
e.log.Error(
|
|
||||||
"queue depth sample: failed to load target types",
|
|
||||||
"error", err,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
var webhookIDs []string
|
|
||||||
|
|
||||||
err = e.database.DB().
|
|
||||||
Model(&database.Webhook{}).
|
|
||||||
Pluck("id", &webhookIDs).Error
|
|
||||||
if err != nil {
|
|
||||||
e.log.Error(
|
|
||||||
"queue depth sample: failed to query webhook IDs",
|
|
||||||
"error", err,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
pending := make(map[database.TargetType]int)
|
|
||||||
retrying := make(map[database.TargetType]int)
|
|
||||||
|
|
||||||
for _, webhookID := range webhookIDs {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
|
|
||||||
if !e.dbManager.DBExists(webhookID) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
e.sampleWebhookQueueDepths(
|
|
||||||
webhookID, types, pending, retrying,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
e.mtr.SetQueueDepths(pending, retrying)
|
|
||||||
}
|
|
||||||
|
|
||||||
// targetTypesByID maps every configured target id to its type. The
|
|
||||||
// deliveries live in the per-webhook databases but carry only a
|
|
||||||
// target id, so the type label has to come from the main database.
|
|
||||||
//
|
|
||||||
// Find rather than Scan: see sampleWebhookQueueDepths.
|
|
||||||
func (e *Engine) targetTypesByID() (
|
|
||||||
map[string]database.TargetType, error,
|
|
||||||
) {
|
|
||||||
var rows []struct {
|
|
||||||
ID string
|
|
||||||
Type database.TargetType
|
|
||||||
}
|
|
||||||
|
|
||||||
err := e.database.DB().
|
|
||||||
Model(&database.Target{}).
|
|
||||||
Select("id", "type").
|
|
||||||
Find(&rows).Error
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("loading targets: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
types := make(map[string]database.TargetType, len(rows))
|
|
||||||
|
|
||||||
for _, row := range rows {
|
|
||||||
types[row.ID] = row.Type
|
|
||||||
}
|
|
||||||
|
|
||||||
return types, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// sampleWebhookQueueDepths adds one webhook's queued deliveries into
|
|
||||||
// the running totals.
|
|
||||||
//
|
|
||||||
// A delivery whose target has since been deleted is not in the type
|
|
||||||
// map and so counts under the empty target type. Set.SetQueueDepths
|
|
||||||
// folds that into the unknown series rather than dropping it: a
|
|
||||||
// backlog stuck behind a deleted target is a backlog that still needs
|
|
||||||
// to be alertable.
|
|
||||||
//
|
|
||||||
// The aggregate is read with Find, not Scan. (*gorm.DB).Scan swaps
|
|
||||||
// GORM's own trace recorder in for the logging adapter, and that
|
|
||||||
// recorder does not implement gorm.ParamsFilter, so the statement
|
|
||||||
// reaches the log with its bound values interpolated — here, the
|
|
||||||
// status list. Find goes through the normal query callback, which is
|
|
||||||
// filtered. See internal/gormlog and its scan_guard_test.go.
|
|
||||||
func (e *Engine) sampleWebhookQueueDepths(
|
|
||||||
webhookID string,
|
|
||||||
types map[string]database.TargetType,
|
|
||||||
pending, retrying map[database.TargetType]int,
|
|
||||||
) {
|
|
||||||
webhookDB, err := e.dbManager.GetDB(webhookID)
|
|
||||||
if err != nil {
|
|
||||||
e.log.Error(
|
|
||||||
"queue depth sample: failed to get webhook database",
|
|
||||||
"webhook_id", webhookID,
|
|
||||||
"error", err,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
var rows []struct {
|
|
||||||
TargetID string
|
|
||||||
Status database.DeliveryStatus
|
|
||||||
Depth int
|
|
||||||
}
|
|
||||||
|
|
||||||
err = webhookDB.
|
|
||||||
Model(&database.Delivery{}).
|
|
||||||
Select("target_id", "status", "count(*) as depth").
|
|
||||||
Where("status IN ?", []database.DeliveryStatus{
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
database.DeliveryStatusRetrying,
|
|
||||||
}).
|
|
||||||
Group("target_id, status").
|
|
||||||
Find(&rows).Error
|
|
||||||
if err != nil {
|
|
||||||
e.log.Error(
|
|
||||||
"queue depth sample: "+
|
|
||||||
"failed to count queued deliveries",
|
|
||||||
"webhook_id", webhookID,
|
|
||||||
"error", err,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, row := range rows {
|
|
||||||
targetType := types[row.TargetID]
|
|
||||||
|
|
||||||
switch row.Status {
|
|
||||||
case database.DeliveryStatusPending:
|
|
||||||
pending[targetType] += row.Depth
|
|
||||||
case database.DeliveryStatusRetrying:
|
|
||||||
retrying[targetType] += row.Depth
|
|
||||||
case database.DeliveryStatusDelivered,
|
|
||||||
database.DeliveryStatusFailed:
|
|
||||||
// Excluded by the query above: a delivery that has
|
|
||||||
// reached a terminal state is not queued.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,179 +0,0 @@
|
|||||||
package delivery_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"fmt"
|
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/driver/sqlite"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// qdAggregateMarker identifies the queue-depth aggregate in the
|
|
||||||
// captured SQL. It is the one statement in this test that binds
|
|
||||||
// anything, and the raw count() expression appears in no other.
|
|
||||||
const qdAggregateMarker = "count(*)"
|
|
||||||
|
|
||||||
// qdSyncBuf collects log output from whichever goroutine GORM writes
|
|
||||||
// on.
|
|
||||||
type qdSyncBuf struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
b bytes.Buffer
|
|
||||||
}
|
|
||||||
|
|
||||||
func (q *qdSyncBuf) Write(p []byte) (int, error) {
|
|
||||||
q.mu.Lock()
|
|
||||||
defer q.mu.Unlock()
|
|
||||||
|
|
||||||
return q.b.Write(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (q *qdSyncBuf) String() string {
|
|
||||||
q.mu.Lock()
|
|
||||||
defer q.mu.Unlock()
|
|
||||||
|
|
||||||
return q.b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// qdMainDB opens a main database whose GORM logger is the service's
|
|
||||||
// adapter, writing through log.
|
|
||||||
func qdMainDB(t *testing.T, log *slog.Logger) *gorm.DB {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
dsn := fmt.Sprintf(
|
|
||||||
"file:%s?cache=shared&mode=rwc",
|
|
||||||
filepath.Join(t.TempDir(), "main-gormlog.db"),
|
|
||||||
)
|
|
||||||
|
|
||||||
sqlDB, err := sql.Open("sqlite", dsn)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
|
||||||
|
|
||||||
db, err := gorm.Open(
|
|
||||||
sqlite.Dialector{Conn: sqlDB},
|
|
||||||
&gorm.Config{Logger: gormlog.New(log)},
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
require.NoError(t, db.AutoMigrate(
|
|
||||||
&database.Webhook{},
|
|
||||||
&database.Target{},
|
|
||||||
))
|
|
||||||
|
|
||||||
return db
|
|
||||||
}
|
|
||||||
|
|
||||||
// qdLinesContaining returns every captured line carrying marker.
|
|
||||||
func qdLinesContaining(out, marker string) []string {
|
|
||||||
var found []string
|
|
||||||
|
|
||||||
for line := range strings.SplitSeq(out, "\n") {
|
|
||||||
if strings.Contains(line, marker) {
|
|
||||||
found = append(found, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return found
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestQueueDepthSample_LogsNoBoundValue holds the queue-depth sampler
|
|
||||||
// to the values-off property internal/gormlog exists to provide.
|
|
||||||
//
|
|
||||||
// The aggregate binds the delivery status list. Read with
|
|
||||||
// (*gorm.DB).Scan it was logged with those values interpolated, because
|
|
||||||
// Scan records the statement through GORM's own traceRecorder, which
|
|
||||||
// does not implement gorm.ParamsFilter. Read with Find it goes through
|
|
||||||
// the normal query callback and the adapter's filter applies. Restore
|
|
||||||
// the Scan call in queue_depth.go and this fails on the status literals
|
|
||||||
// below; scan_guard_test.go catches the same regression statically.
|
|
||||||
func TestQueueDepthSample_LogsNoBoundValue(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
buf := &qdSyncBuf{}
|
|
||||||
log := slog.New(slog.NewTextHandler(
|
|
||||||
buf, &slog.HandlerOptions{Level: slog.LevelDebug},
|
|
||||||
))
|
|
||||||
|
|
||||||
mainDB := qdMainDB(t, log)
|
|
||||||
dbMgr := database.NewTestWebhookDBManagerWithLogger(
|
|
||||||
t.TempDir(), log,
|
|
||||||
)
|
|
||||||
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
webhookDB := iSeedWebhookDB(t, dbMgr, webhookID)
|
|
||||||
|
|
||||||
iCreateWebhook(t, mainDB, webhookID, "queue-depth-gormlog")
|
|
||||||
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
|
|
||||||
iCreateTarget(t, mainDB, targetID, webhookID,
|
|
||||||
"queue-depth-gormlog-target", database.TargetTypeHTTP,
|
|
||||||
iHTTPConfig("https://example.com/hook"), 3,
|
|
||||||
)
|
|
||||||
|
|
||||||
event := iSeedEvent(
|
|
||||||
t, webhookDB, webhookID, `{"queued":true}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
iSeedDelivery(
|
|
||||||
t, webhookDB, event.ID, targetID,
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
iSeedDelivery(
|
|
||||||
t, webhookDB, event.ID, targetID,
|
|
||||||
database.DeliveryStatusRetrying,
|
|
||||||
)
|
|
||||||
|
|
||||||
engine := delivery.NewTestEngineWithDB(
|
|
||||||
database.NewTestDatabase(mainDB),
|
|
||||||
dbMgr,
|
|
||||||
log,
|
|
||||||
&http.Client{Timeout: 5 * time.Second},
|
|
||||||
2,
|
|
||||||
)
|
|
||||||
|
|
||||||
engine.ExportSampleQueueDepths(context.Background())
|
|
||||||
|
|
||||||
out := buf.String()
|
|
||||||
|
|
||||||
lines := qdLinesContaining(out, qdAggregateMarker)
|
|
||||||
require.NotEmpty(
|
|
||||||
t, lines,
|
|
||||||
"the queue-depth aggregate was never logged, so the "+
|
|
||||||
"assertions below are vacuous",
|
|
||||||
)
|
|
||||||
|
|
||||||
for _, line := range lines {
|
|
||||||
assert.Contains(
|
|
||||||
t, line, "?",
|
|
||||||
"the aggregate was logged without its placeholders: %s",
|
|
||||||
line,
|
|
||||||
)
|
|
||||||
|
|
||||||
for _, status := range []database.DeliveryStatus{
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
database.DeliveryStatusRetrying,
|
|
||||||
} {
|
|
||||||
assert.NotContains(
|
|
||||||
t, line, string(status),
|
|
||||||
"a bound status value was interpolated into the "+
|
|
||||||
"logged statement: %s", line,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,107 +0,0 @@
|
|||||||
package delivery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// maxDeliveryRedirects caps a redirect chain. Installing a
|
|
||||||
// CheckRedirect replaces net/http's default policy including its
|
|
||||||
// own limit, so the limit is restated rather than dropped.
|
|
||||||
const maxDeliveryRedirects = 10
|
|
||||||
|
|
||||||
// schemeHTTPS names the scheme the origin comparison treats
|
|
||||||
// specially: a step down from it is never the same origin.
|
|
||||||
const schemeHTTPS = "https"
|
|
||||||
|
|
||||||
var errTooManyRedirects = errors.New("too many redirects")
|
|
||||||
|
|
||||||
// offOriginHeaderPolicy returns a CheckRedirect that drops every
|
|
||||||
// origin-scoped header once a redirect leaves the origin the
|
|
||||||
// operator configured. names is the set applyRequestHeaders
|
|
||||||
// reports: the operator's configured headers and the inbound event
|
|
||||||
// headers this delivery forwarded, under one rule rather than two.
|
|
||||||
//
|
|
||||||
// net/http withholds Authorization and Cookie across a host change
|
|
||||||
// and forwards everything else. A target header is routinely a
|
|
||||||
// credential under another name — X-Api-Key, PRIVATE-TOKEN,
|
|
||||||
// X-Auth-Token — and a forwarded inbound header is routinely a
|
|
||||||
// sender's signature — X-Hub-Signature — so an open redirect at an
|
|
||||||
// otherwise trusted destination would hand either to a host the
|
|
||||||
// operator never named. Redirects are still followed: refusing them
|
|
||||||
// would break every destination that legitimately redirects and
|
|
||||||
// would record the 3xx as the delivery's result.
|
|
||||||
//
|
|
||||||
// The strip is per hop, not permanent: net/http re-copies the
|
|
||||||
// initial request's headers for every hop, so a chain that returns
|
|
||||||
// to the configured origin carries them again, exactly as net/http
|
|
||||||
// treats Authorization.
|
|
||||||
//
|
|
||||||
// Each hop is dialled through the same SSRF-safe transport, whose
|
|
||||||
// guard runs per connection, so a redirect aimed at a private or
|
|
||||||
// reserved address is still refused at connect time.
|
|
||||||
func offOriginHeaderPolicy(
|
|
||||||
names []string,
|
|
||||||
) func(*http.Request, []*http.Request) error {
|
|
||||||
return func(req *http.Request, via []*http.Request) error {
|
|
||||||
if len(via) >= maxDeliveryRedirects {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"%w: stopped after %d",
|
|
||||||
errTooManyRedirects, maxDeliveryRedirects,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if sameDeliveryOrigin(via[0].URL, req.URL) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, name := range names {
|
|
||||||
req.Header.Del(name)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sameDeliveryOrigin reports whether dest is close enough to the
|
|
||||||
// configured target URL to keep carrying its origin-scoped headers.
|
|
||||||
//
|
|
||||||
// This is stricter than the rule net/http applies to Authorization:
|
|
||||||
// the port is part of the comparison (a different port is a
|
|
||||||
// different service), and a subdomain of the configured host is not
|
|
||||||
// the same origin. An https origin stepping down to http is never
|
|
||||||
// the same origin whatever the hosts are, because that puts the
|
|
||||||
// header on the wire in clear.
|
|
||||||
func sameDeliveryOrigin(origin, dest *url.URL) bool {
|
|
||||||
if origin.Scheme == schemeHTTPS && dest.Scheme != schemeHTTPS {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
return originHostPort(origin) == originHostPort(dest)
|
|
||||||
}
|
|
||||||
|
|
||||||
// originHostPort renders a URL's host for comparison, lowercased
|
|
||||||
// and with the scheme's default port normalised away so that
|
|
||||||
// "https://h" and "https://h:443" are one origin.
|
|
||||||
//
|
|
||||||
// The port is joined with net.JoinHostPort rather than a bare
|
|
||||||
// colon: Hostname() unwraps an IPv6 literal's brackets, so
|
|
||||||
// "[2001:db8::1]:8080" and "[2001:db8::1:8080]" — a different
|
|
||||||
// address on a different port — would otherwise render the same
|
|
||||||
// string and pass as one origin.
|
|
||||||
func originHostPort(u *url.URL) string {
|
|
||||||
host := strings.ToLower(u.Hostname())
|
|
||||||
|
|
||||||
port := u.Port()
|
|
||||||
if port == "" ||
|
|
||||||
(u.Scheme == "http" && port == "80") ||
|
|
||||||
(u.Scheme == schemeHTTPS && port == "443") {
|
|
||||||
return host
|
|
||||||
}
|
|
||||||
|
|
||||||
return net.JoinHostPort(host, port)
|
|
||||||
}
|
|
||||||
@@ -1,383 +0,0 @@
|
|||||||
package delivery_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/url"
|
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The headers these tests drive stand in for the two classes the
|
|
||||||
// off-origin rule covers: an operator-configured credential and an
|
|
||||||
// inbound header the delivery path forwards. net/http withholds
|
|
||||||
// Authorization and Cookie across a host change, and nothing else.
|
|
||||||
const (
|
|
||||||
probeHeaderName = "X-Api-Key"
|
|
||||||
probeHeaderValue = "QQNEVERONTHEWIREQQ"
|
|
||||||
inboundHeaderName = "X-Hub-Signature"
|
|
||||||
inboundHeaderValue = "sha1=QQINBOUNDQQ"
|
|
||||||
)
|
|
||||||
|
|
||||||
// redirectProbe records what the last hop of a redirect chain
|
|
||||||
// actually received.
|
|
||||||
type redirectProbe struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
seen http.Header
|
|
||||||
hits int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *redirectProbe) serve(
|
|
||||||
w http.ResponseWriter, r *http.Request,
|
|
||||||
) {
|
|
||||||
p.mu.Lock()
|
|
||||||
p.seen = r.Header.Clone()
|
|
||||||
p.hits++
|
|
||||||
p.mu.Unlock()
|
|
||||||
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *redirectProbe) result() (http.Header, int) {
|
|
||||||
p.mu.Lock()
|
|
||||||
defer p.mu.Unlock()
|
|
||||||
|
|
||||||
return p.seen, p.hits
|
|
||||||
}
|
|
||||||
|
|
||||||
// deliverWithProbeHeaders runs one real delivery of a new task
|
|
||||||
// through the engine to targetURL, carrying both probe headers —
|
|
||||||
// probeHeaderName configured on the target, inboundHeaderName
|
|
||||||
// forwarded from the event — and returns the delivery status the
|
|
||||||
// engine recorded.
|
|
||||||
func deliverWithProbeHeaders(
|
|
||||||
t *testing.T, targetURL string,
|
|
||||||
) database.DeliveryStatus {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
|
|
||||||
event := iSeedEvent(
|
|
||||||
t, s.WebhookDB, s.WebhookID, `{"hello":"world"}`,
|
|
||||||
)
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
|
|
||||||
inbound, err := json.Marshal(map[string][]string{
|
|
||||||
inboundHeaderName: {inboundHeaderValue},
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
event.Headers = string(inbound)
|
|
||||||
|
|
||||||
d := iSeedDelivery(
|
|
||||||
t, s.WebhookDB, event.ID, targetID,
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
|
|
||||||
cfg, err := json.Marshal(delivery.HTTPTargetConfig{
|
|
||||||
URL: targetURL,
|
|
||||||
Headers: map[string]string{
|
|
||||||
probeHeaderName: probeHeaderValue,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
body := event.Body
|
|
||||||
task := iTask(
|
|
||||||
d, event, s.WebhookID, targetID,
|
|
||||||
"redirect-target", string(cfg), 0, 1, &body,
|
|
||||||
)
|
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(context.TODO(), &task)
|
|
||||||
|
|
||||||
var updated database.Delivery
|
|
||||||
|
|
||||||
require.NoError(t, s.WebhookDB.First(
|
|
||||||
&updated, "id = ?", d.ID,
|
|
||||||
).Error)
|
|
||||||
|
|
||||||
return updated.Status
|
|
||||||
}
|
|
||||||
|
|
||||||
// A 302 to an origin the operator never configured must not carry
|
|
||||||
// the credential they configured for the one they did, nor the
|
|
||||||
// inbound header this delivery forwarded — one rule for both
|
|
||||||
// classes. The chain is still followed, so the delivery is recorded
|
|
||||||
// from the final hop.
|
|
||||||
func TestDelivery_CrossOriginRedirectDropsOriginScopedHeaders(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var probe redirectProbe
|
|
||||||
|
|
||||||
final := httptest.NewServer(
|
|
||||||
http.HandlerFunc(probe.serve),
|
|
||||||
)
|
|
||||||
defer final.Close()
|
|
||||||
|
|
||||||
// httptest listens on loopback, so reach the second server
|
|
||||||
// under loopback's other name: the hop then differs in
|
|
||||||
// hostname as well as port and is cross-host by any reading.
|
|
||||||
finalURL, err := url.Parse(final.URL)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
finalURL.Host = "localhost:" + finalURL.Port()
|
|
||||||
finalURL.Path = "/moved"
|
|
||||||
|
|
||||||
origin := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
http.Redirect(
|
|
||||||
w, r, finalURL.String(),
|
|
||||||
http.StatusFound,
|
|
||||||
)
|
|
||||||
},
|
|
||||||
))
|
|
||||||
defer origin.Close()
|
|
||||||
|
|
||||||
status := deliverWithProbeHeaders(t, origin.URL)
|
|
||||||
|
|
||||||
seen, hits := probe.result()
|
|
||||||
|
|
||||||
assert.Equal(t, 1, hits,
|
|
||||||
"the redirect must still be followed",
|
|
||||||
)
|
|
||||||
assert.Empty(t, seen.Get(probeHeaderName),
|
|
||||||
"a configured credential header must not reach an "+
|
|
||||||
"origin the operator did not configure",
|
|
||||||
)
|
|
||||||
assert.Empty(t, seen.Get(inboundHeaderName),
|
|
||||||
"a forwarded inbound header must not reach an origin "+
|
|
||||||
"the operator did not configure",
|
|
||||||
)
|
|
||||||
assert.Equal(t,
|
|
||||||
database.DeliveryStatusDelivered, status,
|
|
||||||
"the final hop's 200 is the delivery's result",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stripping must not fire within the configured origin, or every
|
|
||||||
// destination that redirects its own path would lose its
|
|
||||||
// credential and start answering 401 — and would lose the inbound
|
|
||||||
// signature the receiver verifies.
|
|
||||||
func TestDelivery_SameOriginRedirectKeepsOriginScopedHeaders(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var probe redirectProbe
|
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if r.URL.Path == "/moved" {
|
|
||||||
probe.serve(w, r)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
http.Redirect(
|
|
||||||
w, r, "/moved", http.StatusFound,
|
|
||||||
)
|
|
||||||
},
|
|
||||||
))
|
|
||||||
defer srv.Close()
|
|
||||||
|
|
||||||
status := deliverWithProbeHeaders(t, srv.URL+"/hook")
|
|
||||||
|
|
||||||
seen, hits := probe.result()
|
|
||||||
|
|
||||||
assert.Equal(t, 1, hits)
|
|
||||||
assert.Equal(t, probeHeaderValue, seen.Get(probeHeaderName),
|
|
||||||
"a redirect within the configured origin must keep "+
|
|
||||||
"the configured header",
|
|
||||||
)
|
|
||||||
assert.Equal(t,
|
|
||||||
inboundHeaderValue, seen.Get(inboundHeaderName),
|
|
||||||
"a redirect within the configured origin must keep "+
|
|
||||||
"the forwarded inbound header",
|
|
||||||
)
|
|
||||||
assert.Equal(t,
|
|
||||||
database.DeliveryStatusDelivered, status,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The origin comparison is deliberately stricter than the one
|
|
||||||
// net/http applies to Authorization: the port counts and a
|
|
||||||
// subdomain does not inherit. Only the default-port spellings of
|
|
||||||
// one origin are the same origin.
|
|
||||||
func TestSameDeliveryOrigin(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// The configured target URL every case redirects away from.
|
|
||||||
// Destination paths differ only so that no literal repeats.
|
|
||||||
const configured = "https://h/a"
|
|
||||||
|
|
||||||
cases := map[string]struct {
|
|
||||||
origin string
|
|
||||||
dest string
|
|
||||||
want bool
|
|
||||||
}{
|
|
||||||
"other path": {configured, "https://h/b", true},
|
|
||||||
"default port spelled": {configured, "https://h:443/c", true},
|
|
||||||
"host in another case": {configured, "https://H/d", true},
|
|
||||||
"http default port": {"http://h:80/a", "http://h/e", true},
|
|
||||||
"upgrade to https": {"http://h/a", "https://h/f", true},
|
|
||||||
"downgrade to http": {configured, "http://h/g", false},
|
|
||||||
"another host": {configured, "https://i/h", false},
|
|
||||||
"a subdomain": {configured, "https://x.h/i", false},
|
|
||||||
"the parent domain": {"https://x.h/a", "https://h/j", false},
|
|
||||||
"another port": {configured, "https://h:8443/k", false},
|
|
||||||
|
|
||||||
// Hostname() unwraps an IPv6 literal's brackets, so a
|
|
||||||
// bracketed host whose last group is the origin's port
|
|
||||||
// renders identically to the origin unless the port is
|
|
||||||
// re-joined with brackets. Each dest below differs from
|
|
||||||
// its origin in address AND in port.
|
|
||||||
"ipv6 port as final group": {
|
|
||||||
"https://[2001:db8::1]:8080/a",
|
|
||||||
"https://[2001:db8::1:8080]/l",
|
|
||||||
false,
|
|
||||||
},
|
|
||||||
"ipv6 loopback port as final group": {
|
|
||||||
"https://[::1]:8080/a",
|
|
||||||
"https://[::1:8080]/m",
|
|
||||||
false,
|
|
||||||
},
|
|
||||||
"ipv6 same origin": {
|
|
||||||
"https://[2001:db8::1]:8080/a",
|
|
||||||
"https://[2001:DB8::1]:8080/n",
|
|
||||||
true,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, tc := range cases {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
origin, err := url.Parse(tc.origin)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
dest, err := url.Parse(tc.dest)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
assert.Equal(t, tc.want,
|
|
||||||
delivery.ExportSameDeliveryOrigin(
|
|
||||||
origin, dest,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Installing a CheckRedirect discards net/http's own redirect
|
|
||||||
// limit, so the cap this policy restates is the only thing between
|
|
||||||
// a self-redirecting destination and an unbounded chain. A
|
|
||||||
// destination that always redirects must be cut off after exactly
|
|
||||||
// maxDeliveryRedirects requests, with the sentinel surfacing to the
|
|
||||||
// caller rather than a generic net/http error.
|
|
||||||
func TestRedirectPolicy_StopsAtHopCap(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var hits atomic.Int64
|
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
hits.Add(1)
|
|
||||||
http.Redirect(
|
|
||||||
w, r, "/loop", http.StatusFound,
|
|
||||||
)
|
|
||||||
},
|
|
||||||
))
|
|
||||||
defer srv.Close()
|
|
||||||
|
|
||||||
engine := delivery.NewTestEngine(
|
|
||||||
slog.New(slog.DiscardHandler),
|
|
||||||
&http.Client{Timeout: 10 * time.Second},
|
|
||||||
1,
|
|
||||||
)
|
|
||||||
|
|
||||||
client := engine.ExportClientForRequest(
|
|
||||||
&delivery.HTTPTargetConfig{URL: srv.URL},
|
|
||||||
[]string{probeHeaderName},
|
|
||||||
)
|
|
||||||
require.NotNil(t, client.CheckRedirect)
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(
|
|
||||||
context.Background(), http.MethodGet, srv.URL, http.NoBody,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
resp, doErr := client.Do(req)
|
|
||||||
if resp != nil {
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
require.Error(t, doErr,
|
|
||||||
"an endless redirect chain must not be followed forever",
|
|
||||||
)
|
|
||||||
require.ErrorIs(t, doErr, delivery.ErrExportTooManyRedirects)
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
int64(delivery.ExportMaxDeliveryRedirects), hits.Load(),
|
|
||||||
"the chain must stop after exactly %d hops",
|
|
||||||
delivery.ExportMaxDeliveryRedirects,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The set the redirect policy strips is whatever the delivery path
|
|
||||||
// actually put on the wire, so a header added to the forward set is
|
|
||||||
// covered without a second edit. A header the event never carried
|
|
||||||
// is not in the set, and the delivery path's own two are deliberately
|
|
||||||
// excluded: Content-Type describes the body, which a 307 carries
|
|
||||||
// across hosts, and the inbound User-Agent every real sender supplies
|
|
||||||
// is overwritten before the request goes out.
|
|
||||||
func TestApplyRequestHeaders_ReportsOriginScopedNames(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
inbound, err := json.Marshal(map[string][]string{
|
|
||||||
inboundHeaderName: {inboundHeaderValue},
|
|
||||||
"Content-Type": {testContentType},
|
|
||||||
"User-Agent": {"curl/8.7.1"},
|
|
||||||
"Host": {"inbound.example.com"},
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(
|
|
||||||
context.Background(),
|
|
||||||
http.MethodPost,
|
|
||||||
"https://target.example.com/hook",
|
|
||||||
http.NoBody,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
names := delivery.ExportApplyRequestHeaders(
|
|
||||||
req,
|
|
||||||
&database.Event{
|
|
||||||
Headers: string(inbound),
|
|
||||||
ContentType: testContentType,
|
|
||||||
},
|
|
||||||
&delivery.HTTPTargetConfig{
|
|
||||||
Headers: map[string]string{
|
|
||||||
probeHeaderName: probeHeaderValue,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
[]string{probeHeaderName, inboundHeaderName}, names,
|
|
||||||
"both header classes are reported, and only those: "+
|
|
||||||
"Host is never forwarded, Content-Type and "+
|
|
||||||
"User-Agent are the delivery path's own",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -6,11 +6,8 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
|
||||||
"net/url"
|
"net/url"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -28,83 +25,20 @@ var (
|
|||||||
errBlockedIP = errors.New(
|
errBlockedIP = errors.New(
|
||||||
"blocked private/reserved IP range",
|
"blocked private/reserved IP range",
|
||||||
)
|
)
|
||||||
errBlockedMetadata = errors.New(
|
|
||||||
"blocked link-local or cloud instance metadata " +
|
|
||||||
"address: ALLOWED_EGRESS_CIDRS cannot open it",
|
|
||||||
)
|
|
||||||
errInvalidScheme = errors.New(
|
errInvalidScheme = errors.New(
|
||||||
"only http and https are allowed",
|
"only http and https are allowed",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
// blockedNetworks contains all private/reserved IP ranges
|
// blockedNetworks contains all private/reserved IP ranges
|
||||||
// that should be blocked to prevent SSRF attacks. An operator
|
// that should be blocked to prevent SSRF attacks.
|
||||||
// can permit specific blocks out of this set with
|
|
||||||
// ALLOWED_EGRESS_CIDRS; see Guard.
|
|
||||||
//
|
//
|
||||||
//nolint:gochecknoglobals // package-level network list is appropriate here
|
//nolint:gochecknoglobals // package-level network list is appropriate here
|
||||||
var blockedNetworks []*net.IPNet
|
var blockedNetworks []*net.IPNet
|
||||||
|
|
||||||
// alwaysBlockedNetworks are the ranges no configuration can
|
|
||||||
// open: the link-local blocks and the cloud instance metadata
|
|
||||||
// endpoints that live outside them. Reaching one is credential
|
|
||||||
// or user-data theft rather than delivery to an internal
|
|
||||||
// service, so a supplied CIDR that covers such an address still
|
|
||||||
// leaves it blocked.
|
|
||||||
//
|
|
||||||
// Inclusion criterion — an address belongs here only if BOTH
|
|
||||||
// hold, and every entry below satisfies both:
|
|
||||||
//
|
|
||||||
// 1. It is a fixed address assigned by the provider, or a
|
|
||||||
// range reserved by IANA — never one the operator chose.
|
|
||||||
// That is what makes a host route free: it cannot collide
|
|
||||||
// with anything the operator runs.
|
|
||||||
// 2. Reaching it discloses credentials, or user data or
|
|
||||||
// bootstrap material — something granting onward access, or
|
|
||||||
// not cheaply rotated.
|
|
||||||
//
|
|
||||||
// Both halves are load-bearing, so use them to refuse a
|
|
||||||
// candidate and say why. An endpoint disclosing only the
|
|
||||||
// operator's own inventory (instance id, region, disks, NICs)
|
|
||||||
// fails (2): letting a delivery target reach the operator's own
|
|
||||||
// infrastructure is the feature ALLOWED_EGRESS_CIDRS exists to
|
|
||||||
// provide. But (2) is not "IAM credentials only" either —
|
|
||||||
// fd00:42::42 serves /user_data and /conf rather than tokens,
|
|
||||||
// and user data routinely carries bootstrap secrets. An address
|
|
||||||
// stays out if it fails (1) however well it clears (2): a host
|
|
||||||
// route inside a block operators really assign from, such as
|
|
||||||
// 10.0.0.0/8, can collide with a real internal service and
|
|
||||||
// forfeits the justification in (1).
|
|
||||||
//
|
|
||||||
// A publicly routable unicast address does not belong here even
|
|
||||||
// when it clears both halves. Nothing in this list can be
|
|
||||||
// reopened, so putting a public address here leaves the operator
|
|
||||||
// no escape hatch at all — the condition ALLOWED_EGRESS_CIDRS
|
|
||||||
// exists to remove. Default-block it in blockedNetworks instead,
|
|
||||||
// which an allowlist can override.
|
|
||||||
//
|
|
||||||
// This is a criterion, not an enumeration of every metadata
|
|
||||||
// address in existence.
|
|
||||||
//
|
|
||||||
// Every entry is either already in blockedNetworks — this list is
|
|
||||||
// what makes it unconditional — or an alternate encoding of
|
|
||||||
// 169.254.169.254 that Contains does not match against
|
|
||||||
// 169.254.0.0/16. Every entry outside the link-local blocks is a
|
|
||||||
// /32 or /128 host route, so blocking it costs an operator
|
|
||||||
// nothing else on the surrounding network.
|
|
||||||
//
|
|
||||||
// Derive membership from the address, never from the vendor's
|
|
||||||
// prose. Several providers call these endpoints "link-local" or
|
|
||||||
// even "localhost" in their own documentation while the address
|
|
||||||
// is a ULA outside fe80::/10, so a set derived from the docs
|
|
||||||
// comes out wrong.
|
|
||||||
//
|
|
||||||
//nolint:gochecknoglobals // package-level network list is appropriate here
|
|
||||||
var alwaysBlockedNetworks []*net.IPNet
|
|
||||||
|
|
||||||
//nolint:gochecknoinits // init is the idiomatic way to parse CIDRs once at startup
|
//nolint:gochecknoinits // init is the idiomatic way to parse CIDRs once at startup
|
||||||
func init() {
|
func init() {
|
||||||
blockedNetworks = mustParseCIDRs([]string{
|
cidrs := []string{
|
||||||
"127.0.0.0/8",
|
"127.0.0.0/8",
|
||||||
"10.0.0.0/8",
|
"10.0.0.0/8",
|
||||||
"172.16.0.0/12",
|
"172.16.0.0/12",
|
||||||
@@ -122,73 +56,8 @@ func init() {
|
|||||||
"::1/128",
|
"::1/128",
|
||||||
"fc00::/7",
|
"fc00::/7",
|
||||||
"fe80::/10",
|
"fe80::/10",
|
||||||
})
|
|
||||||
|
|
||||||
// Every entry is named. The set must not grow or shrink
|
|
||||||
// without a matching change to
|
|
||||||
// TestAlwaysBlockedNetworks_PinnedSet.
|
|
||||||
//
|
|
||||||
// The IPv4-mapped form ::ffff:169.254.169.254 needs no
|
|
||||||
// entry: net.IPNet.Contains normalises it via To4() before
|
|
||||||
// comparing, so 169.254.0.0/16 already matches it. To4()
|
|
||||||
// does not normalise the IPv4-compatible or NAT64 forms,
|
|
||||||
// which is why those are listed separately.
|
|
||||||
alwaysBlockedNetworks = mustParseCIDRs([]string{
|
|
||||||
// IPv4 link-local, carrying the 169.254.169.254
|
|
||||||
// metadata service used by AWS, Azure, DigitalOcean,
|
|
||||||
// Hetzner, OpenStack and others. Not Alibaba, which uses
|
|
||||||
// 100.100.100.200 below exclusively.
|
|
||||||
"169.254.0.0/16",
|
|
||||||
// IPv6 link-local, its IPv6 counterpart.
|
|
||||||
"fe80::/10",
|
|
||||||
|
|
||||||
// IPv6 metadata endpoints in ULA space. Each is a host
|
|
||||||
// route, and fd00::/8 is an ordinary block for an
|
|
||||||
// operator to allowlist, so without these entries that
|
|
||||||
// one allowlist line hands out cloud credentials on
|
|
||||||
// every provider below.
|
|
||||||
//
|
|
||||||
// AWS IPv6 IMDS.
|
|
||||||
"fd00:ec2::254/128",
|
|
||||||
// AWS EKS Pod Identity Agent, which issues pod identity
|
|
||||||
// credentials. A second AWS endpoint, distinct from
|
|
||||||
// IMDS above. AWS's own docs call it "localhost".
|
|
||||||
"fd00:ec2::23/128",
|
|
||||||
// GCP metadata server for IPv6-only instances.
|
|
||||||
"fd20:ce::254/128",
|
|
||||||
// Oracle OCI IMDS, serving /opc/v2 instance principals.
|
|
||||||
"fd00:c1::a9fe:a9fe/128",
|
|
||||||
// Scaleway metadata, serving /user_data and /conf.
|
|
||||||
"fd00:42::42/128",
|
|
||||||
// Linode/Akamai metadata. Akamai's docs call it
|
|
||||||
// "link-local"; it is not.
|
|
||||||
"fd00:a9fe:a9fe::1/128",
|
|
||||||
|
|
||||||
// IPv4 metadata endpoints outside link-local.
|
|
||||||
//
|
|
||||||
// Alibaba Cloud metadata. It sits in CGNAT
|
|
||||||
// 100.64.0.0/10, which Tailscale also uses, so an
|
|
||||||
// operator allowlisting a Tailscale peer's range would
|
|
||||||
// otherwise reopen it.
|
|
||||||
"100.100.100.200/32",
|
|
||||||
// Oracle Cloud Classic metadata. Inside the blocked
|
|
||||||
// 192.0.0.0/24, so this entry is what stops an
|
|
||||||
// allowlist from opening it.
|
|
||||||
"192.0.0.192/32",
|
|
||||||
|
|
||||||
// 169.254.169.254 as an IPv4-compatible IPv6 address.
|
|
||||||
"::a9fe:a9fe/128",
|
|
||||||
// 169.254.169.254 behind the NAT64 well-known prefix.
|
|
||||||
"64:ff9b::a9fe:a9fe/128",
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// mustParseCIDRs parses a list of CIDR literals, panicking on a
|
|
||||||
// bad one. The inputs are compile-time constants, so a failure
|
|
||||||
// is a programming error rather than a runtime condition.
|
|
||||||
func mustParseCIDRs(cidrs []string) []*net.IPNet {
|
|
||||||
networks := make([]*net.IPNet, 0, len(cidrs))
|
|
||||||
|
|
||||||
for _, cidr := range cidrs {
|
for _, cidr := range cidrs {
|
||||||
_, network, err := net.ParseCIDR(cidr)
|
_, network, err := net.ParseCIDR(cidr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -198,15 +67,16 @@ func mustParseCIDRs(cidrs []string) []*net.IPNet {
|
|||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
networks = append(networks, network)
|
blockedNetworks = append(
|
||||||
|
blockedNetworks, network,
|
||||||
|
)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return networks
|
// isBlockedIP checks whether an IP address falls within
|
||||||
}
|
// any blocked private/reserved network range.
|
||||||
|
func isBlockedIP(ip net.IP) bool {
|
||||||
// matchesAny reports whether ip falls inside any of networks.
|
for _, network := range blockedNetworks {
|
||||||
func matchesAny(networks []*net.IPNet, ip net.IP) bool {
|
|
||||||
for _, network := range networks {
|
|
||||||
if network.Contains(ip) {
|
if network.Contains(ip) {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
@@ -215,50 +85,14 @@ func matchesAny(networks []*net.IPNet, ip net.IP) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// isBlockedIP checks whether an IP address falls within
|
|
||||||
// any blocked private/reserved network range, before any
|
|
||||||
// operator allowlist is considered.
|
|
||||||
func isBlockedIP(ip net.IP) bool {
|
|
||||||
return matchesAny(blockedNetworks, ip)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Guard makes every SSRF decision in the process.
|
|
||||||
//
|
|
||||||
// It holds the operator's ALLOWED_EGRESS_CIDRS allowlist and
|
|
||||||
// applies it in exactly one place, checkIP, which both the
|
|
||||||
// target-creation validator (ValidateTargetURL) and the delivery
|
|
||||||
// dialer call. Routing both through the same function is the
|
|
||||||
// point: when the two paths decided separately they drifted and
|
|
||||||
// disagreed, which is what made a target creatable but
|
|
||||||
// undeliverable.
|
|
||||||
//
|
|
||||||
// The guard is always on. The allowlist only ever adds specific
|
|
||||||
// networks to what the default blocklist refuses, and no
|
|
||||||
// configuration turns the guard off wholesale.
|
|
||||||
type Guard struct {
|
|
||||||
// allowed is the operator's ALLOWED_EGRESS_CIDRS. Empty
|
|
||||||
// (the default) means the default blocklist stands as-is.
|
|
||||||
allowed []netip.Prefix
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewGuard builds the process-wide SSRF guard from configuration.
|
|
||||||
func NewGuard(cfg *config.Config) *Guard {
|
|
||||||
return &Guard{allowed: cfg.AllowedEgressCIDRs}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ValidateTargetURL checks that an HTTP delivery target
|
// ValidateTargetURL checks that an HTTP delivery target
|
||||||
// URL is safe from SSRF attacks.
|
// URL is safe from SSRF attacks.
|
||||||
func (g *Guard) ValidateTargetURL(
|
func ValidateTargetURL(
|
||||||
ctx context.Context, targetURL string,
|
ctx context.Context, targetURL string,
|
||||||
) error {
|
) error {
|
||||||
parsed, err := url.Parse(targetURL)
|
parsed, err := url.Parse(targetURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// url.Parse embeds the whole URL in its error, and
|
return fmt.Errorf("invalid URL: %w", err)
|
||||||
// this one is logged and shown; mask it. Every other
|
|
||||||
// branch below reports only the hostname.
|
|
||||||
return fmt.Errorf(
|
|
||||||
"invalid URL: %w", maskURLError(err),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
err = validateScheme(parsed.Scheme)
|
err = validateScheme(parsed.Scheme)
|
||||||
@@ -272,79 +106,36 @@ func (g *Guard) ValidateTargetURL(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if ip := net.ParseIP(host); ip != nil {
|
if ip := net.ParseIP(host); ip != nil {
|
||||||
return g.checkIP(ip)
|
return checkBlockedIP(ip)
|
||||||
}
|
}
|
||||||
|
|
||||||
return g.validateHostname(ctx, host)
|
return validateHostname(ctx, host)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewSSRFSafeTransport creates an http.Transport with a
|
func validateScheme(scheme string) error {
|
||||||
// custom DialContext that refuses connections to any address
|
if scheme != "http" && scheme != "https" {
|
||||||
// this guard blocks. It resolves and checks at dial time, so a
|
|
||||||
// name that passed validation but now answers with a blocked
|
|
||||||
// address (DNS rebinding) is still refused.
|
|
||||||
func (g *Guard) NewSSRFSafeTransport() *http.Transport {
|
|
||||||
return &http.Transport{
|
|
||||||
DialContext: g.ssrfDialContext,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// allows reports whether ip falls inside the operator's
|
|
||||||
// configured egress allowlist.
|
|
||||||
func (g *Guard) allows(ip net.IP) bool {
|
|
||||||
if len(g.allowed) == 0 {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
addr, ok := netip.AddrFromSlice(ip)
|
|
||||||
if !ok {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// Config unmaps every parsed prefix, so an IPv4-mapped
|
|
||||||
// address has to be unmapped too or it would never match.
|
|
||||||
addr = addr.Unmap()
|
|
||||||
|
|
||||||
for _, prefix := range g.allowed {
|
|
||||||
if prefix.Contains(addr) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkIP is the single point at which SSRF policy is decided.
|
|
||||||
//
|
|
||||||
// The order is the policy:
|
|
||||||
//
|
|
||||||
// 1. alwaysBlockedNetworks is refused before the allowlist is
|
|
||||||
// consulted, so no configured CIDR reaches link-local or a
|
|
||||||
// cloud instance metadata endpoint.
|
|
||||||
// 2. The allowlist is consulted next, so a listed private
|
|
||||||
// network becomes reachable.
|
|
||||||
// 3. Everything else keeps the default blocklist's answer.
|
|
||||||
func (g *Guard) checkIP(ip net.IP) error {
|
|
||||||
if matchesAny(alwaysBlockedNetworks, ip) {
|
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"target IP %s: %w", ip, errBlockedMetadata,
|
"unsupported URL scheme %q: %w",
|
||||||
|
scheme, errInvalidScheme,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if g.allows(ip) {
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func checkBlockedIP(ip net.IP) error {
|
||||||
if isBlockedIP(ip) {
|
if isBlockedIP(ip) {
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"target IP %s: %w", ip, errBlockedIP,
|
"target IP %s is in a blocked "+
|
||||||
|
"private/reserved range: %w",
|
||||||
|
ip, errBlockedIP,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (g *Guard) validateHostname(
|
func validateHostname(
|
||||||
ctx context.Context, host string,
|
ctx context.Context, host string,
|
||||||
) error {
|
) error {
|
||||||
dnsCtx, cancel := context.WithTimeout(
|
dnsCtx, cancel := context.WithTimeout(
|
||||||
@@ -369,11 +160,11 @@ func (g *Guard) validateHostname(
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, ipAddr := range ips {
|
for _, ipAddr := range ips {
|
||||||
err = g.checkIP(ipAddr.IP)
|
if isBlockedIP(ipAddr.IP) {
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"hostname %q resolves to a blocked address: %w",
|
"hostname %q resolves to blocked "+
|
||||||
host, err,
|
"IP %s: %w",
|
||||||
|
host, ipAddr.IP, errBlockedIP,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -381,7 +172,16 @@ func (g *Guard) validateHostname(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (g *Guard) ssrfDialContext(
|
// NewSSRFSafeTransport creates an http.Transport with a
|
||||||
|
// custom DialContext that blocks connections to
|
||||||
|
// private/reserved IP addresses.
|
||||||
|
func NewSSRFSafeTransport() *http.Transport {
|
||||||
|
return &http.Transport{
|
||||||
|
DialContext: ssrfDialContext,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func ssrfDialContext(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
network, addr string,
|
network, addr string,
|
||||||
) (net.Conn, error) {
|
) (net.Conn, error) {
|
||||||
@@ -404,11 +204,11 @@ func (g *Guard) ssrfDialContext(
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, ipAddr := range ips {
|
for _, ipAddr := range ips {
|
||||||
err = g.checkIP(ipAddr.IP)
|
if isBlockedIP(ipAddr.IP) {
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf(
|
||||||
"ssrf: connection to %s blocked: %w",
|
"ssrf: connection to %s (%s) "+
|
||||||
host, err,
|
"blocked: %w",
|
||||||
|
host, ipAddr.IP, errBlockedIP,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -420,14 +220,3 @@ func (g *Guard) ssrfDialContext(
|
|||||||
net.JoinHostPort(ips[0].IP.String(), port),
|
net.JoinHostPort(ips[0].IP.String(), port),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateScheme(scheme string) error {
|
|
||||||
if scheme != "http" && scheme != "https" {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"unsupported URL scheme %q: %w",
|
|
||||||
scheme, errInvalidScheme,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,562 +0,0 @@
|
|||||||
package delivery_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/netip"
|
|
||||||
"net/url"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Addresses the SSRF tests in this package share.
|
|
||||||
const (
|
|
||||||
// metadataIP is the cloud instance metadata address, and
|
|
||||||
// metadataURL an endpoint on it. The guard must never reach
|
|
||||||
// either, whatever an operator lists.
|
|
||||||
metadataIP = "169.254.169.254"
|
|
||||||
metadataURL = "http://" + metadataIP + "/latest/meta-data/"
|
|
||||||
|
|
||||||
// loopbackHookURL is a target on this host: blocked by
|
|
||||||
// default, reachable only once an operator allowlists
|
|
||||||
// loopback.
|
|
||||||
loopbackHookURL = "http://127.0.0.1/hook"
|
|
||||||
|
|
||||||
// publicIP is an ordinary public address, which the guard
|
|
||||||
// permits with or without an allowlist.
|
|
||||||
publicIP = "93.184.216.34"
|
|
||||||
|
|
||||||
// allowAllIPv4 and allowAllIPv6 are the widest allowlist
|
|
||||||
// entries expressible: the whole internet, in each family.
|
|
||||||
// Nothing unconditionally blocked may be reachable under
|
|
||||||
// them.
|
|
||||||
allowAllIPv4 = "0.0.0.0/0"
|
|
||||||
allowAllIPv6 = "::/0"
|
|
||||||
|
|
||||||
// allowAllULA is the ordinary ULA block an operator lists to
|
|
||||||
// reach their own IPv6 network. Several providers park a
|
|
||||||
// metadata endpoint inside it.
|
|
||||||
allowAllULA = "fd00::/8"
|
|
||||||
|
|
||||||
// metadataRefusalClause is the part of the refusal that only
|
|
||||||
// alwaysBlockedNetworks produces. Asserting it, rather than
|
|
||||||
// the bare word "blocked", is what proves the unconditional
|
|
||||||
// set did the refusing and not the default blocklist.
|
|
||||||
metadataRefusalClause = "ALLOWED_EGRESS_CIDRS cannot open it"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestGuardAllowlist_PermittedCIDRDelivers proves the escape
|
|
||||||
// hatch actually works end to end: with 127.0.0.0/8 allowed, the
|
|
||||||
// guard's own transport connects to a loopback server and gets a
|
|
||||||
// response back. The default guard, given the identical URL,
|
|
||||||
// refuses it — so the delivery succeeds because of the allowlist
|
|
||||||
// and nothing else.
|
|
||||||
func TestGuardAllowlist_PermittedCIDRDelivers(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
srv := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusNoContent)
|
|
||||||
},
|
|
||||||
))
|
|
||||||
t.Cleanup(srv.Close)
|
|
||||||
|
|
||||||
// httptest listens on loopback, which the default blocklist
|
|
||||||
// covers: exactly the "forward to a service on this host"
|
|
||||||
// case the allowlist exists for.
|
|
||||||
requireLoopback(t, srv.URL)
|
|
||||||
|
|
||||||
guard := delivery.NewTestGuard(
|
|
||||||
netip.MustParsePrefix("127.0.0.0/8"),
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t,
|
|
||||||
guard.ValidateTargetURL(context.Background(), srv.URL),
|
|
||||||
"an allowlisted loopback target must pass validation",
|
|
||||||
)
|
|
||||||
|
|
||||||
client := &http.Client{
|
|
||||||
Timeout: 5 * time.Second,
|
|
||||||
Transport: guard.NewSSRFSafeTransport(),
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(
|
|
||||||
context.Background(), http.MethodPost, srv.URL, nil,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
resp, err := client.Do(req)
|
|
||||||
require.NoError(t,
|
|
||||||
err, "an allowlisted loopback target must be deliverable",
|
|
||||||
)
|
|
||||||
|
|
||||||
defer func() { _ = resp.Body.Close() }()
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusNoContent, resp.StatusCode)
|
|
||||||
|
|
||||||
// The same URL through the default guard must still fail, or
|
|
||||||
// this test would pass without the allowlist doing anything.
|
|
||||||
assert.Error(t,
|
|
||||||
delivery.NewTestGuard().ValidateTargetURL(
|
|
||||||
context.Background(), srv.URL,
|
|
||||||
),
|
|
||||||
"without the allowlist the same target must be refused",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGuardAllowlist_UnlistedPrivateStillRefused proves the
|
|
||||||
// allowlist grants only what it names. A guard that opens one
|
|
||||||
// private block must keep refusing every other one, at both the
|
|
||||||
// validation and the delivery entry point.
|
|
||||||
func TestGuardAllowlist_UnlistedPrivateStillRefused(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Only 10.1.0.0/16 is open — a narrow block inside a much
|
|
||||||
// wider private range, so the test can tell "permits the
|
|
||||||
// listed block" from "permits anything private".
|
|
||||||
guard := delivery.NewTestGuard(
|
|
||||||
netip.MustParsePrefix("10.1.0.0/16"),
|
|
||||||
)
|
|
||||||
|
|
||||||
refused := []string{
|
|
||||||
"http://192.168.1.10/hook",
|
|
||||||
"http://172.16.0.1/hook",
|
|
||||||
loopbackHookURL,
|
|
||||||
"http://[fc00::1]/hook",
|
|
||||||
"http://100.64.0.1/hook",
|
|
||||||
// Private, adjacent to the allowed block, outside it.
|
|
||||||
"http://10.2.0.1/hook",
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, target := range refused {
|
|
||||||
t.Run(target, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
err := guard.ValidateTargetURL(
|
|
||||||
context.Background(), target,
|
|
||||||
)
|
|
||||||
require.Error(t,
|
|
||||||
err, "%s is not allowlisted and must be refused",
|
|
||||||
target,
|
|
||||||
)
|
|
||||||
assert.Contains(t, err.Error(), "blocked")
|
|
||||||
|
|
||||||
assertDialRefused(t, guard, target)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// The block that is listed must in fact be permitted, so the
|
|
||||||
// refusals above are selective rather than a guard that
|
|
||||||
// ignores its allowlist entirely.
|
|
||||||
assert.NoError(t,
|
|
||||||
guard.ValidateTargetURL(
|
|
||||||
context.Background(), "http://10.1.2.3/hook",
|
|
||||||
),
|
|
||||||
"the allowlisted block must be permitted",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGuardAllowlist_MetadataAlwaysRefused is the load-bearing
|
|
||||||
// case: cloud instance metadata endpoints are credential theft
|
|
||||||
// rather than delivery to an internal service, so no allowlist
|
|
||||||
// reaches one. Every guard below names a CIDR that covers its
|
|
||||||
// target — including 0.0.0.0/0, ::/0, and the ordinary ULA and
|
|
||||||
// CGNAT blocks an operator would really list — and the address
|
|
||||||
// must stay refused anyway, on both the validation and the
|
|
||||||
// delivery path.
|
|
||||||
func TestGuardAllowlist_MetadataAlwaysRefused(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tt := range metadataAlwaysRefusedCases() {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
guard := delivery.NewTestGuard(
|
|
||||||
netip.MustParsePrefix(tt.allow),
|
|
||||||
)
|
|
||||||
|
|
||||||
err := guard.ValidateTargetURL(
|
|
||||||
context.Background(), tt.target,
|
|
||||||
)
|
|
||||||
require.Error(t,
|
|
||||||
err,
|
|
||||||
"%s must stay blocked even though %s covers it",
|
|
||||||
tt.target, tt.allow,
|
|
||||||
)
|
|
||||||
assert.Contains(t,
|
|
||||||
err.Error(),
|
|
||||||
metadataRefusalClause,
|
|
||||||
"the refusal must say why it cannot be opened",
|
|
||||||
)
|
|
||||||
|
|
||||||
// The metadata clause, not just "blocked": that is
|
|
||||||
// what distinguishes the unconditional set from the
|
|
||||||
// ordinary blocklist.
|
|
||||||
assertDialRefusedWith(
|
|
||||||
t, guard, tt.target, metadataRefusalClause,
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// metadataAlwaysRefusedCase is one (allowlist, target) pair that
|
|
||||||
// must be refused: allow covers target, and target must stay
|
|
||||||
// blocked regardless.
|
|
||||||
type metadataAlwaysRefusedCase struct {
|
|
||||||
name string
|
|
||||||
allow string
|
|
||||||
target string
|
|
||||||
}
|
|
||||||
|
|
||||||
// metadataAlwaysRefusedCases enumerates every unconditionally
|
|
||||||
// blocked address together with an allowlist entry that would
|
|
||||||
// otherwise reach it. Split by family of address only to stay
|
|
||||||
// under the function-length limit.
|
|
||||||
func metadataAlwaysRefusedCases() []metadataAlwaysRefusedCase {
|
|
||||||
cases := linkLocalRefusedCases()
|
|
||||||
cases = append(cases, ulaMetadataRefusedCases()...)
|
|
||||||
cases = append(cases, ipv4MetadataRefusedCases()...)
|
|
||||||
|
|
||||||
return append(cases, encodedMetadataRefusedCases()...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// linkLocalRefusedCases covers the link-local blocks, including
|
|
||||||
// an operator naming the metadata address outright.
|
|
||||||
func linkLocalRefusedCases() []metadataAlwaysRefusedCase {
|
|
||||||
return []metadataAlwaysRefusedCase{
|
|
||||||
{
|
|
||||||
name: "exact metadata host",
|
|
||||||
allow: "169.254.169.254/32",
|
|
||||||
target: metadataURL,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "whole link-local block",
|
|
||||||
allow: "169.254.0.0/16",
|
|
||||||
target: metadataURL,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "supernet covering link-local",
|
|
||||||
allow: "169.0.0.0/8",
|
|
||||||
target: metadataURL,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "the entire IPv4 internet",
|
|
||||||
allow: allowAllIPv4,
|
|
||||||
target: metadataURL,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "other link-local address",
|
|
||||||
allow: allowAllIPv4,
|
|
||||||
target: "http://169.254.1.1/",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "IPv6 link-local",
|
|
||||||
allow: allowAllIPv6,
|
|
||||||
target: "http://[fe80::1]/",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ulaMetadataRefusedCases covers the metadata endpoints parked
|
|
||||||
// in ULA space. Every one is opened by the single ordinary
|
|
||||||
// allowlist entry fd00::/8, which is the whole reason they need
|
|
||||||
// their own /128 host routes: fe80::/10 does not cover a ULA,
|
|
||||||
// whatever the vendor's documentation calls the address.
|
|
||||||
func ulaMetadataRefusedCases() []metadataAlwaysRefusedCase {
|
|
||||||
return []metadataAlwaysRefusedCase{
|
|
||||||
{
|
|
||||||
name: "AWS IPv6 IMDS under an allowlisted ULA block",
|
|
||||||
allow: allowAllULA,
|
|
||||||
target: "http://[fd00:ec2::254]/latest/meta-data/",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// A second AWS credential endpoint, distinct from
|
|
||||||
// IMDS. AWS's own docs call this one "localhost".
|
|
||||||
name: "AWS EKS Pod Identity under an allowlisted ULA block",
|
|
||||||
allow: allowAllULA,
|
|
||||||
target: "http://[fd00:ec2::23]/v1/credentials",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "GCP IPv6 metadata under an allowlisted ULA block",
|
|
||||||
allow: allowAllULA,
|
|
||||||
target: "http://[fd20:ce::254]/computeMetadata/v1/",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Oracle OCI IPv6 IMDS under an allowlisted ULA block",
|
|
||||||
allow: allowAllULA,
|
|
||||||
target: "http://[fd00:c1::a9fe:a9fe]/opc/v2/instance/",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Scaleway IPv6 metadata under an allowlisted ULA block",
|
|
||||||
allow: allowAllULA,
|
|
||||||
target: "http://[fd00:42::42]/conf",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// Akamai's docs call this "link-local"; it is a ULA,
|
|
||||||
// so fe80::/10 does not cover it.
|
|
||||||
name: "Linode IPv6 metadata under an allowlisted ULA block",
|
|
||||||
allow: allowAllULA,
|
|
||||||
target: "http://[fd00:a9fe:a9fe::1]/v1/instance",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ipv4MetadataRefusedCases covers the IPv4 metadata endpoints
|
|
||||||
// that sit outside link-local: one in CGNAT and one in the
|
|
||||||
// blocked 192.0.0.0/24, each reachable only through an allowlist
|
|
||||||
// that this set overrides.
|
|
||||||
func ipv4MetadataRefusedCases() []metadataAlwaysRefusedCase {
|
|
||||||
return []metadataAlwaysRefusedCase{
|
|
||||||
{
|
|
||||||
// Tailscale uses 100.64.0.0/10, so an operator
|
|
||||||
// forwarding to a Tailscale peer lists exactly this.
|
|
||||||
name: "Alibaba metadata under allowlisted CGNAT",
|
|
||||||
allow: "100.64.0.0/10",
|
|
||||||
target: "http://100.100.100.200/latest/meta-data/",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// Inside the already-blocked 192.0.0.0/24, so only
|
|
||||||
// an allowlist can reach it — and must not.
|
|
||||||
name: "Oracle Cloud Classic metadata under 0.0.0.0/0",
|
|
||||||
allow: allowAllIPv4,
|
|
||||||
target: "http://192.0.0.192/latest/meta-data/",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// encodedMetadataRefusedCases covers the alternate IPv6
|
|
||||||
// encodings of 169.254.169.254.
|
|
||||||
func encodedMetadataRefusedCases() []metadataAlwaysRefusedCase {
|
|
||||||
return []metadataAlwaysRefusedCase{
|
|
||||||
{
|
|
||||||
// To4() does not normalise the IPv4-compatible form,
|
|
||||||
// so this needs its own always-blocked entry.
|
|
||||||
name: "IPv4-compatible IPv6 form of the metadata IP",
|
|
||||||
allow: allowAllIPv6,
|
|
||||||
target: "http://[::a9fe:a9fe]/latest/meta-data/",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// Nor the NAT64 well-known prefix form.
|
|
||||||
name: "NAT64 form of the metadata IP",
|
|
||||||
allow: allowAllIPv6,
|
|
||||||
target: "http://[64:ff9b::a9fe:a9fe]/latest/meta-data/",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// Already refused before this change: IPNet.Contains
|
|
||||||
// calls To4() first, so the mapped form matches
|
|
||||||
// 169.254.0.0/16. Pinned so it cannot regress.
|
|
||||||
//
|
|
||||||
// Allowed under 0.0.0.0/0 rather than ::/0: allows()
|
|
||||||
// unmaps before matching, so ::/0 would not cover the
|
|
||||||
// unmapped v4 address and the case would not prove
|
|
||||||
// the allowlist was overridden.
|
|
||||||
name: "IPv4-mapped IPv6 form of the metadata IP",
|
|
||||||
allow: allowAllIPv4,
|
|
||||||
target: "http://[::ffff:169.254.169.254]/latest/meta-data/",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGuardAllowlist_PublicUnaffected asserts the allowlist does
|
|
||||||
// not narrow anything: public addresses were reachable before it
|
|
||||||
// existed and stay reachable, whether or not a list is set.
|
|
||||||
func TestGuardAllowlist_PublicUnaffected(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
guards := map[string]*delivery.Guard{
|
|
||||||
"default": delivery.NewTestGuard(),
|
|
||||||
"with allowlist": delivery.NewTestGuard(
|
|
||||||
netip.MustParsePrefix("10.0.0.0/8"),
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, guard := range guards {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
assert.NoError(t,
|
|
||||||
guard.ValidateTargetURL(
|
|
||||||
context.Background(),
|
|
||||||
"http://"+publicIP+"/webhook",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGuardCheckIP_BothPathsShareOneDecision asserts that the
|
|
||||||
// validator and the dialer are not two policies that happen to
|
|
||||||
// agree: both are defined in terms of checkIP, so the exported
|
|
||||||
// decision function is the whole answer for a given address.
|
|
||||||
func TestGuardCheckIP_BothPathsShareOneDecision(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
guard := delivery.NewTestGuard(
|
|
||||||
netip.MustParsePrefix("10.0.0.0/8"),
|
|
||||||
)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
ip string
|
|
||||||
allowed bool
|
|
||||||
}{
|
|
||||||
{"10.1.2.3", true},
|
|
||||||
{publicIP, true},
|
|
||||||
{"192.168.1.1", false},
|
|
||||||
{"127.0.0.1", false},
|
|
||||||
{metadataIP, false},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.ip, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ip := net.ParseIP(tt.ip)
|
|
||||||
require.NotNil(t, ip)
|
|
||||||
|
|
||||||
decision := guard.ExportCheckIP(ip)
|
|
||||||
|
|
||||||
validation := guard.ValidateTargetURL(
|
|
||||||
context.Background(), "http://"+hostFor(tt.ip)+"/x",
|
|
||||||
)
|
|
||||||
|
|
||||||
if tt.allowed {
|
|
||||||
require.NoError(t, decision)
|
|
||||||
require.NoError(t, validation)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
require.Error(t, decision)
|
|
||||||
require.Error(t, validation,
|
|
||||||
"validation must refuse what checkIP refuses",
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestAlwaysBlockedNetworks_PinnedSet pins the unconditional set
|
|
||||||
// exactly, so it cannot quietly grow or shrink.
|
|
||||||
//
|
|
||||||
// It stays deliberately small. Everything else in the default
|
|
||||||
// blocklist is an operator's own network and must remain
|
|
||||||
// openable, or the escape hatch would not work — which is why
|
|
||||||
// the metadata endpoints outside the link-local range are host
|
|
||||||
// routes rather than the blocks that contain them.
|
|
||||||
func TestAlwaysBlockedNetworks_PinnedSet(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
nets := delivery.ExportAlwaysBlockedNetworks()
|
|
||||||
|
|
||||||
got := make([]string, 0, len(nets))
|
|
||||||
for _, n := range nets {
|
|
||||||
got = append(got, n.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
want := []string{
|
|
||||||
// IPv4 link-local: the 169.254.169.254 metadata
|
|
||||||
// service on AWS, Azure and others.
|
|
||||||
"169.254.0.0/16",
|
|
||||||
// IPv6 link-local.
|
|
||||||
"fe80::/10",
|
|
||||||
// AWS IPv6 IMDS, inside the ULA space an operator may
|
|
||||||
// legitimately allowlist.
|
|
||||||
"fd00:ec2::254/128",
|
|
||||||
// AWS EKS Pod Identity Agent, likewise ULA.
|
|
||||||
"fd00:ec2::23/128",
|
|
||||||
// GCP metadata for IPv6-only instances, likewise ULA.
|
|
||||||
"fd20:ce::254/128",
|
|
||||||
// Oracle OCI IMDS over IPv6, likewise ULA.
|
|
||||||
"fd00:c1::a9fe:a9fe/128",
|
|
||||||
// Scaleway metadata over IPv6, likewise ULA.
|
|
||||||
"fd00:42::42/128",
|
|
||||||
// Linode/Akamai metadata over IPv6, likewise ULA.
|
|
||||||
"fd00:a9fe:a9fe::1/128",
|
|
||||||
// Alibaba Cloud metadata, inside CGNAT.
|
|
||||||
"100.100.100.200/32",
|
|
||||||
// Oracle Cloud Classic metadata, inside the blocked
|
|
||||||
// 192.0.0.0/24.
|
|
||||||
"192.0.0.192/32",
|
|
||||||
// 169.254.169.254 as an IPv4-compatible IPv6 address.
|
|
||||||
"::a9fe:a9fe/128",
|
|
||||||
// 169.254.169.254 behind the NAT64 well-known prefix.
|
|
||||||
"64:ff9b::a9fe:a9fe/128",
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, want, got)
|
|
||||||
}
|
|
||||||
|
|
||||||
// requireLoopback fails the test unless rawURL's host is a
|
|
||||||
// loopback address, so the allowlist test cannot silently stop
|
|
||||||
// exercising a blocked range.
|
|
||||||
func requireLoopback(t *testing.T, rawURL string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
parsed, err := url.Parse(rawURL)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
ip := net.ParseIP(parsed.Hostname())
|
|
||||||
require.NotNil(t, ip, "test server host must be an IP literal")
|
|
||||||
require.True(t, ip.IsLoopback(),
|
|
||||||
"test server must listen on loopback, got %s", ip,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertDialRefused asserts the guard's transport refuses to
|
|
||||||
// connect to target, which is the delivery-time half of the
|
|
||||||
// policy. It never reaches the network: the guard checks the
|
|
||||||
// resolved address before dialling.
|
|
||||||
func assertDialRefused(
|
|
||||||
t *testing.T, guard *delivery.Guard, target string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
assertDialRefusedWith(t, guard, target, "blocked")
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertDialRefusedWith is assertDialRefused with the refusal
|
|
||||||
// text pinned. Callers testing the unconditional set pass
|
|
||||||
// metadataRefusalClause so the subtest cannot pass on an
|
|
||||||
// ordinary blocklist refusal instead.
|
|
||||||
func assertDialRefusedWith(
|
|
||||||
t *testing.T, guard *delivery.Guard, target, clause string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
client := &http.Client{
|
|
||||||
Timeout: 5 * time.Second,
|
|
||||||
Transport: guard.NewSSRFSafeTransport(),
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(
|
|
||||||
context.Background(), http.MethodPost, target, nil,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
resp, err := client.Do(req)
|
|
||||||
if resp != nil {
|
|
||||||
_ = resp.Body.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
require.Error(t, err,
|
|
||||||
"delivery to %s must be refused by the dialer", target,
|
|
||||||
)
|
|
||||||
assert.Contains(t, err.Error(), clause,
|
|
||||||
"the refusal must come from the SSRF guard",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// hostFor renders an IP as it appears in a URL host, bracketing
|
|
||||||
// IPv6 literals.
|
|
||||||
func hostFor(ip string) string {
|
|
||||||
if net.ParseIP(ip).To4() == nil {
|
|
||||||
return "[" + ip + "]"
|
|
||||||
}
|
|
||||||
|
|
||||||
return ip
|
|
||||||
}
|
|
||||||
@@ -31,10 +31,10 @@ func TestIsBlockedIP_PrivateRanges(t *testing.T) {
|
|||||||
{"192.168.0.1", "192.168.0.1", true},
|
{"192.168.0.1", "192.168.0.1", true},
|
||||||
{"192.168.255.255", "192.168.255.255", true},
|
{"192.168.255.255", "192.168.255.255", true},
|
||||||
{"169.254.0.1", "169.254.0.1", true},
|
{"169.254.0.1", "169.254.0.1", true},
|
||||||
{metadataIP, metadataIP, true},
|
{"169.254.169.254", "169.254.169.254", true},
|
||||||
{"8.8.8.8", "8.8.8.8", false},
|
{"8.8.8.8", "8.8.8.8", false},
|
||||||
{"1.1.1.1", "1.1.1.1", false},
|
{"1.1.1.1", "1.1.1.1", false},
|
||||||
{publicIP, publicIP, false},
|
{"93.184.216.34", "93.184.216.34", false},
|
||||||
{"::1", "::1", true},
|
{"::1", "::1", true},
|
||||||
{"fd00::1", "fd00::1", true},
|
{"fd00::1", "fd00::1", true},
|
||||||
{"fc00::1", "fc00::1", true},
|
{"fc00::1", "fc00::1", true},
|
||||||
@@ -72,12 +72,12 @@ func TestValidateTargetURL_Blocked(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
blockedURLs := []string{
|
blockedURLs := []string{
|
||||||
loopbackHookURL,
|
"http://127.0.0.1/hook",
|
||||||
"http://127.0.0.1:8080/hook",
|
"http://127.0.0.1:8080/hook",
|
||||||
"https://10.0.0.1/hook",
|
"https://10.0.0.1/hook",
|
||||||
"http://192.168.1.1/webhook",
|
"http://192.168.1.1/webhook",
|
||||||
"http://172.16.0.1/api",
|
"http://172.16.0.1/api",
|
||||||
metadataURL,
|
"http://169.254.169.254/latest/meta-data/",
|
||||||
"http://[::1]/hook",
|
"http://[::1]/hook",
|
||||||
"http://[fc00::1]/hook",
|
"http://[fc00::1]/hook",
|
||||||
"http://[fe80::1]/hook",
|
"http://[fe80::1]/hook",
|
||||||
@@ -88,7 +88,7 @@ func TestValidateTargetURL_Blocked(t *testing.T) {
|
|||||||
t.Run(u, func(t *testing.T) {
|
t.Run(u, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
err := delivery.NewTestGuard().ValidateTargetURL(
|
err := delivery.ValidateTargetURL(
|
||||||
context.Background(), u,
|
context.Background(), u,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -112,7 +112,7 @@ func TestValidateTargetURL_Allowed(t *testing.T) {
|
|||||||
t.Run(u, func(t *testing.T) {
|
t.Run(u, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
err := delivery.NewTestGuard().ValidateTargetURL(
|
err := delivery.ValidateTargetURL(
|
||||||
context.Background(), u,
|
context.Background(), u,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -126,7 +126,7 @@ func TestValidateTargetURL_Allowed(t *testing.T) {
|
|||||||
func TestValidateTargetURL_InvalidScheme(t *testing.T) {
|
func TestValidateTargetURL_InvalidScheme(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
err := delivery.NewTestGuard().ValidateTargetURL(
|
err := delivery.ValidateTargetURL(
|
||||||
context.Background(), "ftp://example.com/hook",
|
context.Background(), "ftp://example.com/hook",
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -140,7 +140,7 @@ func TestValidateTargetURL_InvalidScheme(t *testing.T) {
|
|||||||
func TestValidateTargetURL_EmptyHost(t *testing.T) {
|
func TestValidateTargetURL_EmptyHost(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
err := delivery.NewTestGuard().ValidateTargetURL(
|
err := delivery.ValidateTargetURL(
|
||||||
context.Background(), "http:///path",
|
context.Background(), "http:///path",
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -150,7 +150,7 @@ func TestValidateTargetURL_EmptyHost(t *testing.T) {
|
|||||||
func TestValidateTargetURL_InvalidURL(t *testing.T) {
|
func TestValidateTargetURL_InvalidURL(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
err := delivery.NewTestGuard().ValidateTargetURL(
|
err := delivery.ValidateTargetURL(
|
||||||
context.Background(), "://invalid",
|
context.Background(), "://invalid",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -27,12 +27,6 @@ type Scheduler interface {
|
|||||||
// own circuit breaker, and reschedules via the injected
|
// own circuit breaker, and reschedules via the injected
|
||||||
// Scheduler. Fire-and-forget targets simply record a single
|
// Scheduler. Fire-and-forget targets simply record a single
|
||||||
// attempt.
|
// attempt.
|
||||||
//
|
|
||||||
// An implementation reports each attempt it actually dispatches to
|
|
||||||
// Engine.observeAttempt, alongside the DeliveryResult it records for
|
|
||||||
// it. Deliver is also entered for attempts that never happen — an
|
|
||||||
// open circuit breaker refuses one — so the count cannot be taken
|
|
||||||
// from around this call.
|
|
||||||
type Target interface {
|
type Target interface {
|
||||||
Deliver(
|
Deliver(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
@@ -80,12 +74,6 @@ type attemptResult struct {
|
|||||||
errMsg string
|
errMsg string
|
||||||
}
|
}
|
||||||
|
|
||||||
// elapsed returns how long the attempt took. The field is stored in
|
|
||||||
// milliseconds because that is what DeliveryResult persists.
|
|
||||||
func (r attemptResult) elapsed() time.Duration {
|
|
||||||
return time.Duration(r.duration) * time.Millisecond
|
|
||||||
}
|
|
||||||
|
|
||||||
// initTargets builds the target registry, wiring each target
|
// initTargets builds the target registry, wiring each target
|
||||||
// to the engine's persistence helpers and giving the HTTP and
|
// to the engine's persistence helpers and giving the HTTP and
|
||||||
// Slack targets the shared SSRF-safe client. It is called by
|
// Slack targets the shared SSRF-safe client. It is called by
|
||||||
|
|||||||
@@ -1,119 +0,0 @@
|
|||||||
package delivery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// errUnknownTargetTypeForEdit is returned when a stored target has a
|
|
||||||
// type the edit form has no field set for.
|
|
||||||
var errUnknownTargetTypeForEdit = errors.New(
|
|
||||||
"unknown target type",
|
|
||||||
)
|
|
||||||
|
|
||||||
// TargetConfigForm is the UNMASKED projection of a target's stored
|
|
||||||
// configuration, for pre-filling the target edit form.
|
|
||||||
//
|
|
||||||
// It is the deliberate exception to the rule TargetView enforces
|
|
||||||
// everywhere else: TargetView exists so that no template can render
|
|
||||||
// a target's stored blob, because a destination URL's path segments
|
|
||||||
// and a header value are both routinely the credential. An operator
|
|
||||||
// cannot correct a value they cannot see, so the edit form — and
|
|
||||||
// only the edit form — is shown the full value.
|
|
||||||
//
|
|
||||||
// Everything that keeps that exception narrow lives at the call
|
|
||||||
// site: the route is behind RequireAuth and the webhook's ownership
|
|
||||||
// check, and its group sets NoCache so the rendered secret is not
|
|
||||||
// written to a shared cache. Do not reach for this type from any
|
|
||||||
// other page.
|
|
||||||
type TargetConfigForm struct {
|
|
||||||
// URL is the destination for an HTTP target and the webhook
|
|
||||||
// URL for a Slack target.
|
|
||||||
URL string
|
|
||||||
// Headers is the HTTP target's configured headers in the
|
|
||||||
// textarea representation, one "Name: value" per line.
|
|
||||||
Headers string
|
|
||||||
// Timeout is the HTTP target's per-request timeout in seconds,
|
|
||||||
// empty when unset.
|
|
||||||
Timeout string
|
|
||||||
// Expiry is the database (archive) target's row expiry.
|
|
||||||
Expiry string
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewTargetConfigForm parses a target's stored configuration into
|
|
||||||
// the edit form's fields.
|
|
||||||
//
|
|
||||||
// A configuration that does not parse is an error rather than a
|
|
||||||
// zero-valued form that silently looks like a target with no
|
|
||||||
// settings. The caller shows the operator that the stored value
|
|
||||||
// could not be read, so that saving the form is understood as
|
|
||||||
// replacing it rather than preserving it.
|
|
||||||
func NewTargetConfigForm(
|
|
||||||
t *database.Target,
|
|
||||||
) (TargetConfigForm, error) {
|
|
||||||
switch t.Type {
|
|
||||||
case database.TargetTypeHTTP:
|
|
||||||
cfg, err := parseHTTPConfig(t.Config)
|
|
||||||
if err != nil {
|
|
||||||
return TargetConfigForm{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return TargetConfigForm{
|
|
||||||
URL: cfg.URL,
|
|
||||||
Headers: FormatTargetHeaders(cfg.Headers),
|
|
||||||
Timeout: FormatTargetTimeout(cfg.Timeout),
|
|
||||||
}, nil
|
|
||||||
case database.TargetTypeSlack:
|
|
||||||
cfg, err := parseSlackConfig(t.Config)
|
|
||||||
if err != nil {
|
|
||||||
return TargetConfigForm{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return TargetConfigForm{URL: cfg.WebhookURL}, nil
|
|
||||||
case database.TargetTypeDatabase:
|
|
||||||
return databaseConfigForm(t.Config)
|
|
||||||
case database.TargetTypeLog:
|
|
||||||
// The log target takes no configuration.
|
|
||||||
return TargetConfigForm{}, nil
|
|
||||||
default:
|
|
||||||
return TargetConfigForm{}, fmt.Errorf(
|
|
||||||
"%w: %q", errUnknownTargetTypeForEdit, t.Type,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// databaseConfigForm parses an archive target's optional expiry.
|
|
||||||
// An absent or empty configuration is the keep-forever default and
|
|
||||||
// yields an empty field, so re-saving the form unchanged stores the
|
|
||||||
// same empty configuration it started with. An expiry that is set
|
|
||||||
// but not a valid duration is an error, not a blank field.
|
|
||||||
func databaseConfigForm(
|
|
||||||
configJSON string,
|
|
||||||
) (TargetConfigForm, error) {
|
|
||||||
if configJSON == "" {
|
|
||||||
return TargetConfigForm{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var cfg databaseTargetConfig
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(configJSON), &cfg)
|
|
||||||
if err != nil {
|
|
||||||
return TargetConfigForm{}, fmt.Errorf(
|
|
||||||
"parsing config JSON: %w", err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.Expiry == "" || cfg.Expiry == archiveExpiryNever {
|
|
||||||
return TargetConfigForm{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
err = ValidateArchiveExpiry(cfg.Expiry)
|
|
||||||
if err != nil {
|
|
||||||
return TargetConfigForm{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return TargetConfigForm{Expiry: cfg.Expiry}, nil
|
|
||||||
}
|
|
||||||
@@ -1,208 +0,0 @@
|
|||||||
package delivery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"strconv"
|
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// configUnavailable is what a target's configuration renders
|
|
||||||
// as when it is absent, of an unknown type, or does not
|
|
||||||
// parse. The stored blob is never shown as a fallback: it can
|
|
||||||
// hold a credential (a Slack incoming webhook URL is a bearer
|
|
||||||
// token) and a UI that prints it leaks that credential into
|
|
||||||
// browser history, screenshots and screen shares.
|
|
||||||
const configUnavailable = "(unavailable)"
|
|
||||||
|
|
||||||
// ConfigField is one labelled, display-safe value derived
|
|
||||||
// from a target's stored configuration.
|
|
||||||
type ConfigField struct {
|
|
||||||
Label string
|
|
||||||
Value string
|
|
||||||
}
|
|
||||||
|
|
||||||
// TargetView is the display-safe projection of a target for
|
|
||||||
// the UI. It deliberately has no raw configuration field, so
|
|
||||||
// no template — present or future — can render the stored
|
|
||||||
// blob.
|
|
||||||
type TargetView struct {
|
|
||||||
ID string
|
|
||||||
Name string
|
|
||||||
Type database.TargetType
|
|
||||||
Active bool
|
|
||||||
Config []ConfigField
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewTargetViews projects targets for rendering, replacing
|
|
||||||
// each stored configuration blob with named, display-safe
|
|
||||||
// fields.
|
|
||||||
func NewTargetViews(
|
|
||||||
targets []database.Target,
|
|
||||||
) []TargetView {
|
|
||||||
views := make([]TargetView, 0, len(targets))
|
|
||||||
|
|
||||||
for i := range targets {
|
|
||||||
t := &targets[i]
|
|
||||||
|
|
||||||
views = append(views, TargetView{
|
|
||||||
ID: t.ID,
|
|
||||||
Name: t.Name,
|
|
||||||
Type: t.Type,
|
|
||||||
Active: t.Active,
|
|
||||||
Config: targetConfigFields(t),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return views
|
|
||||||
}
|
|
||||||
|
|
||||||
// targetConfigFields returns the display-safe fields for a
|
|
||||||
// target's configuration. Anything it cannot parse becomes
|
|
||||||
// the neutral placeholder.
|
|
||||||
func targetConfigFields(
|
|
||||||
t *database.Target,
|
|
||||||
) []ConfigField {
|
|
||||||
switch t.Type {
|
|
||||||
case database.TargetTypeSlack:
|
|
||||||
return slackConfigFields(t.Config)
|
|
||||||
case database.TargetTypeHTTP:
|
|
||||||
return httpConfigFields(t)
|
|
||||||
case database.TargetTypeDatabase:
|
|
||||||
return databaseConfigFields(t.Config)
|
|
||||||
case database.TargetTypeLog:
|
|
||||||
// The log target takes no configuration.
|
|
||||||
return nil
|
|
||||||
default:
|
|
||||||
return unavailableConfigFields()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// unavailableConfigFields is the neutral placeholder shown
|
|
||||||
// for a configuration that could not be presented.
|
|
||||||
func unavailableConfigFields() []ConfigField {
|
|
||||||
return []ConfigField{{
|
|
||||||
Label: "Configuration",
|
|
||||||
Value: configUnavailable,
|
|
||||||
}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// slackConfigFields describes a Slack target. Only the masked
|
|
||||||
// webhook URL is shown; the full URL is the credential.
|
|
||||||
func slackConfigFields(configJSON string) []ConfigField {
|
|
||||||
cfg, err := parseSlackConfig(configJSON)
|
|
||||||
if err != nil {
|
|
||||||
return unavailableConfigFields()
|
|
||||||
}
|
|
||||||
|
|
||||||
return []ConfigField{{
|
|
||||||
Label: "Webhook URL",
|
|
||||||
Value: cfg.MaskedWebhookURL(),
|
|
||||||
}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// httpConfigFields describes an HTTP target: its destination
|
|
||||||
// and its retry settings. Header values are not shown — they
|
|
||||||
// routinely carry authorization tokens — only how many are
|
|
||||||
// configured.
|
|
||||||
//
|
|
||||||
// The destination is masked to scheme and host by the same
|
|
||||||
// rule the Slack target uses. An HTTP target's destination is
|
|
||||||
// commonly a Slack, Discord or Teams incoming-webhook endpoint
|
|
||||||
// whose path segments are the credential, and the field takes
|
|
||||||
// an arbitrary URL, so no segment can be assumed non-secret.
|
|
||||||
func httpConfigFields(t *database.Target) []ConfigField {
|
|
||||||
cfg, err := parseHTTPConfig(t.Config)
|
|
||||||
if err != nil {
|
|
||||||
return unavailableConfigFields()
|
|
||||||
}
|
|
||||||
|
|
||||||
fields := []ConfigField{{
|
|
||||||
Label: "Destination URL",
|
|
||||||
Value: MaskURL(cfg.URL),
|
|
||||||
}}
|
|
||||||
|
|
||||||
if cfg.Timeout > 0 {
|
|
||||||
fields = append(fields, ConfigField{
|
|
||||||
Label: "Timeout",
|
|
||||||
Value: strconv.Itoa(cfg.Timeout) + "s",
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(cfg.Headers) > 0 {
|
|
||||||
fields = append(fields, ConfigField{
|
|
||||||
Label: "Headers",
|
|
||||||
Value: fmt.Sprintf(
|
|
||||||
"%d configured", len(cfg.Headers),
|
|
||||||
),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return append(fields, retryFields(t)...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// retryFields describes a target's retry settings, which live
|
|
||||||
// on the target row rather than in its configuration blob.
|
|
||||||
func retryFields(t *database.Target) []ConfigField {
|
|
||||||
retries := strconv.Itoa(t.MaxRetries)
|
|
||||||
if t.MaxRetries == 0 {
|
|
||||||
retries += " (fire-and-forget)"
|
|
||||||
}
|
|
||||||
|
|
||||||
fields := []ConfigField{{
|
|
||||||
Label: "Max Retries",
|
|
||||||
Value: retries,
|
|
||||||
}}
|
|
||||||
|
|
||||||
if t.MaxQueueSize > 0 {
|
|
||||||
fields = append(fields, ConfigField{
|
|
||||||
Label: "Max Queue Size",
|
|
||||||
Value: strconv.Itoa(t.MaxQueueSize),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return fields
|
|
||||||
}
|
|
||||||
|
|
||||||
// databaseConfigFields describes an archive target. Its
|
|
||||||
// configuration is optional, and an absent or empty expiry
|
|
||||||
// means the archive is kept forever. An expiry that is set
|
|
||||||
// but not a valid duration is reported as unavailable rather
|
|
||||||
// than echoed back.
|
|
||||||
func databaseConfigFields(configJSON string) []ConfigField {
|
|
||||||
expiry := archiveExpiryNever
|
|
||||||
|
|
||||||
if configJSON != "" {
|
|
||||||
var cfg databaseTargetConfig
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(configJSON), &cfg)
|
|
||||||
if err != nil {
|
|
||||||
return unavailableConfigFields()
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.Expiry != "" {
|
|
||||||
if ValidateArchiveExpiry(cfg.Expiry) != nil {
|
|
||||||
return unavailableConfigFields()
|
|
||||||
}
|
|
||||||
|
|
||||||
expiry = cfg.Expiry
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return []ConfigField{{
|
|
||||||
Label: "Archive Expiry",
|
|
||||||
Value: expiry,
|
|
||||||
}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// MaskedWebhookURL returns the Slack webhook URL reduced to
|
|
||||||
// its scheme and host, with the path, query and any userinfo
|
|
||||||
// elided. The path segments are the credential, so none of
|
|
||||||
// them is shown: the field accepts an arbitrary URL, so no
|
|
||||||
// segment can be assumed non-secret. A URL that does not
|
|
||||||
// parse into a scheme and host yields the neutral
|
|
||||||
// placeholder, never the raw string.
|
|
||||||
func (c *SlackTargetConfig) MaskedWebhookURL() string {
|
|
||||||
return MaskURL(c.WebhookURL)
|
|
||||||
}
|
|
||||||
@@ -1,328 +0,0 @@
|
|||||||
package delivery_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// slackSecretPath is the credential-bearing part of a
|
|
||||||
// Slack incoming webhook URL: everything after the host.
|
|
||||||
slackSecretPath = "/services/T00000000/B00000000/" +
|
|
||||||
"XXXXXXXXXXXXXXXXXXXXXXXX"
|
|
||||||
slackWebhookURL = "https://hooks.slack.com" +
|
|
||||||
slackSecretPath
|
|
||||||
|
|
||||||
viewExampleOrigin = "https://example.com"
|
|
||||||
viewExampleHook = viewExampleOrigin + "/hook"
|
|
||||||
viewMaskedOrigin = viewExampleOrigin + "/..."
|
|
||||||
viewUnavailable = "(unavailable)"
|
|
||||||
viewExpiryNever = "never"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestMaskedWebhookURL(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := map[string]struct {
|
|
||||||
url string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
"slack webhook": {
|
|
||||||
url: slackWebhookURL,
|
|
||||||
want: "https://hooks.slack.com/...",
|
|
||||||
},
|
|
||||||
"query string dropped": {
|
|
||||||
url: viewExampleOrigin + "/a?token=secret",
|
|
||||||
want: viewExampleOrigin + "/...",
|
|
||||||
},
|
|
||||||
// Fabricated userinfo in a test URL, not a real
|
|
||||||
// credential.
|
|
||||||
//nolint:gosec // G101
|
|
||||||
"userinfo dropped": {
|
|
||||||
url: "https://user:pw@example.com/a/b",
|
|
||||||
want: viewExampleOrigin + "/...",
|
|
||||||
},
|
|
||||||
"no path": {
|
|
||||||
url: viewExampleOrigin,
|
|
||||||
want: viewExampleOrigin,
|
|
||||||
},
|
|
||||||
"root path": {
|
|
||||||
url: viewExampleOrigin + "/",
|
|
||||||
want: viewExampleOrigin,
|
|
||||||
},
|
|
||||||
"not a url": {
|
|
||||||
url: "definitely not a url",
|
|
||||||
want: viewUnavailable,
|
|
||||||
},
|
|
||||||
"empty": {
|
|
||||||
url: "",
|
|
||||||
want: viewUnavailable,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, tc := range tests {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg := &delivery.SlackTargetConfig{
|
|
||||||
WebhookURL: tc.url,
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t, tc.want, cfg.MaskedWebhookURL(),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestMaskedWebhookURL_NeverLeaksPath is the direct
|
|
||||||
// expression of the rule: whatever the input, the masked
|
|
||||||
// value never contains a path segment of it.
|
|
||||||
func TestMaskedWebhookURL_NeverLeaksPath(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cfg := &delivery.SlackTargetConfig{
|
|
||||||
WebhookURL: slackWebhookURL,
|
|
||||||
}
|
|
||||||
|
|
||||||
masked := cfg.MaskedWebhookURL()
|
|
||||||
|
|
||||||
assert.NotContains(t, masked, "T00000000")
|
|
||||||
assert.NotContains(t, masked, "B00000000")
|
|
||||||
assert.NotContains(
|
|
||||||
t, masked, "XXXXXXXXXXXXXXXXXXXXXXXX",
|
|
||||||
)
|
|
||||||
assert.NotContains(t, masked, slackSecretPath)
|
|
||||||
}
|
|
||||||
|
|
||||||
// fieldMap turns a view's config fields into a lookup so
|
|
||||||
// assertions read by label.
|
|
||||||
func fieldMap(fields []delivery.ConfigField) map[string]string {
|
|
||||||
out := make(map[string]string, len(fields))
|
|
||||||
for _, f := range fields {
|
|
||||||
out[f.Label] = f.Value
|
|
||||||
}
|
|
||||||
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// viewFor projects a single target and returns its view.
|
|
||||||
func viewFor(
|
|
||||||
t *testing.T,
|
|
||||||
target database.Target,
|
|
||||||
) delivery.TargetView {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
views := delivery.NewTargetViews(
|
|
||||||
[]database.Target{target},
|
|
||||||
)
|
|
||||||
require.Len(t, views, 1)
|
|
||||||
|
|
||||||
return views[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewTargetViews_Slack(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
view := viewFor(t, database.Target{
|
|
||||||
Name: "slack-target",
|
|
||||||
Type: database.TargetTypeSlack,
|
|
||||||
Active: true,
|
|
||||||
Config: `{"webhookUrl":"` +
|
|
||||||
slackWebhookURL + `"}`,
|
|
||||||
})
|
|
||||||
|
|
||||||
assert.Equal(t, "slack-target", view.Name)
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
map[string]string{
|
|
||||||
"Webhook URL": "https://hooks.slack.com/...",
|
|
||||||
},
|
|
||||||
fieldMap(view.Config),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewTargetViews_HTTP(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
view := viewFor(t, database.Target{
|
|
||||||
Type: database.TargetTypeHTTP,
|
|
||||||
Config: `{"url":"` + viewExampleHook + `",` +
|
|
||||||
`"timeout":30,` +
|
|
||||||
`"headers":{"Authorization":"Bearer sekrit"}}`,
|
|
||||||
MaxRetries: 5,
|
|
||||||
MaxQueueSize: 100,
|
|
||||||
})
|
|
||||||
|
|
||||||
fields := fieldMap(view.Config)
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
map[string]string{
|
|
||||||
"Destination URL": viewMaskedOrigin,
|
|
||||||
"Timeout": "30s",
|
|
||||||
"Headers": "1 configured",
|
|
||||||
"Max Retries": "5",
|
|
||||||
"Max Queue Size": "100",
|
|
||||||
},
|
|
||||||
fields,
|
|
||||||
)
|
|
||||||
|
|
||||||
// Header values can be credentials and are never shown.
|
|
||||||
for _, v := range fields {
|
|
||||||
assert.NotContains(t, v, "sekrit")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewTargetViews_HTTPFireAndForget(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
view := viewFor(t, database.Target{
|
|
||||||
Type: database.TargetTypeHTTP,
|
|
||||||
Config: `{"url":"` + viewExampleHook + `"}`,
|
|
||||||
})
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
map[string]string{
|
|
||||||
"Destination URL": viewMaskedOrigin,
|
|
||||||
"Max Retries": "0 (fire-and-forget)",
|
|
||||||
},
|
|
||||||
fieldMap(view.Config),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestNewTargetViews_HTTPMasksDestinationURL proves the rule
|
|
||||||
// holds for the http target too: an http destination is
|
|
||||||
// routinely an incoming-webhook endpoint whose path segments
|
|
||||||
// are the credential, so none of them is shown.
|
|
||||||
func TestNewTargetViews_HTTPMasksDestinationURL(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
view := viewFor(t, database.Target{
|
|
||||||
Type: database.TargetTypeHTTP,
|
|
||||||
Config: `{"url":"` + slackWebhookURL + `"}`,
|
|
||||||
})
|
|
||||||
|
|
||||||
fields := fieldMap(view.Config)
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
"https://hooks.slack.com/...",
|
|
||||||
fields["Destination URL"],
|
|
||||||
)
|
|
||||||
|
|
||||||
for _, v := range fields {
|
|
||||||
assert.NotContains(t, v, slackSecretPath)
|
|
||||||
assert.NotContains(t, v, "T00000000")
|
|
||||||
assert.NotContains(t, v, "B00000000")
|
|
||||||
assert.NotContains(t, v, "XXXXXXXXXXXXXXXXXXXXXXXX")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewTargetViews_Database(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
tests := map[string]struct {
|
|
||||||
config string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
"empty config": {config: "", want: viewExpiryNever},
|
|
||||||
"empty expiry": {config: `{}`, want: viewExpiryNever},
|
|
||||||
"explicit": {
|
|
||||||
config: `{"expiry":"720h"}`,
|
|
||||||
want: "720h",
|
|
||||||
},
|
|
||||||
"never literal": {
|
|
||||||
config: `{"expiry":"` + viewExpiryNever + `"}`,
|
|
||||||
want: viewExpiryNever,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, tc := range tests {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
view := viewFor(t, database.Target{
|
|
||||||
Type: database.TargetTypeDatabase,
|
|
||||||
Config: tc.config,
|
|
||||||
})
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
map[string]string{"Archive Expiry": tc.want},
|
|
||||||
fieldMap(view.Config),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewTargetViews_Log(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
view := viewFor(t, database.Target{
|
|
||||||
Type: database.TargetTypeLog,
|
|
||||||
Config: "",
|
|
||||||
})
|
|
||||||
|
|
||||||
assert.Empty(t, view.Config)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestNewTargetViews_Unpresentable proves that no config the
|
|
||||||
// view cannot present falls back to the stored blob.
|
|
||||||
func TestNewTargetViews_Unpresentable(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const blob = `{"webhookUrl":"https://hooks.slack.com` +
|
|
||||||
slackSecretPath + `"`
|
|
||||||
|
|
||||||
tests := map[string]database.Target{
|
|
||||||
"unknown target type": {
|
|
||||||
Type: database.TargetType("carrier-pigeon"),
|
|
||||||
Config: blob,
|
|
||||||
},
|
|
||||||
"unparseable json": {
|
|
||||||
Type: database.TargetTypeSlack,
|
|
||||||
Config: blob,
|
|
||||||
},
|
|
||||||
"empty slack config": {
|
|
||||||
Type: database.TargetTypeSlack,
|
|
||||||
},
|
|
||||||
"slack config without url": {
|
|
||||||
Type: database.TargetTypeSlack,
|
|
||||||
Config: `{}`,
|
|
||||||
},
|
|
||||||
"unparseable http json": {
|
|
||||||
Type: database.TargetTypeHTTP,
|
|
||||||
Config: `{"url":`,
|
|
||||||
},
|
|
||||||
"unparseable archive json": {
|
|
||||||
Type: database.TargetTypeDatabase,
|
|
||||||
Config: `{"expiry":`,
|
|
||||||
},
|
|
||||||
"invalid archive expiry": {
|
|
||||||
Type: database.TargetTypeDatabase,
|
|
||||||
Config: `{"expiry":"a fortnight"}`,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, target := range tests {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
view := viewFor(t, target)
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
map[string]string{
|
|
||||||
"Configuration": viewUnavailable,
|
|
||||||
},
|
|
||||||
fieldMap(view.Config),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -42,14 +42,7 @@ func (t *databaseTarget) Deliver(
|
|||||||
_ *Task,
|
_ *Task,
|
||||||
_ Scheduler,
|
_ Scheduler,
|
||||||
) {
|
) {
|
||||||
start := time.Now()
|
|
||||||
|
|
||||||
err := t.archive(d)
|
err := t.archive(d)
|
||||||
|
|
||||||
elapsed := time.Since(start)
|
|
||||||
|
|
||||||
t.eng.observeAttempt(d.Target.Type, elapsed)
|
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.eng.log.Error(
|
t.eng.log.Error(
|
||||||
"failed to archive event to database target",
|
"failed to archive event to database target",
|
||||||
@@ -60,25 +53,22 @@ func (t *databaseTarget) Deliver(
|
|||||||
|
|
||||||
t.eng.recordResult(
|
t.eng.recordResult(
|
||||||
webhookDB, d, 1, false, 0, "",
|
webhookDB, d, 1, false, 0, "",
|
||||||
err.Error(), elapsed.Milliseconds(),
|
err.Error(), 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
t.eng.updateDeliveryStatus(
|
t.eng.updateDeliveryStatus(
|
||||||
webhookDB, d, d.Target.Type,
|
webhookDB, d, database.DeliveryStatusFailed,
|
||||||
database.DeliveryStatusFailed,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
t.eng.recordResult(
|
t.eng.recordResult(
|
||||||
webhookDB, d, 1, true, 0, "", "",
|
webhookDB, d, 1, true, 0, "", "", 0,
|
||||||
elapsed.Milliseconds(),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
t.eng.updateDeliveryStatus(
|
t.eng.updateDeliveryStatus(
|
||||||
webhookDB, d, d.Target.Type,
|
webhookDB, d, database.DeliveryStatusDelivered,
|
||||||
database.DeliveryStatusDelivered,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ import (
|
|||||||
|
|
||||||
"gorm.io/driver/sqlite"
|
"gorm.io/driver/sqlite"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// archiveExpiryNever is the expiry sentinel (and default) that
|
// archiveExpiryNever is the expiry sentinel (and default) that
|
||||||
@@ -283,11 +282,7 @@ func (w *archiveWriter) openMode(
|
|||||||
}
|
}
|
||||||
|
|
||||||
gdb, err := gorm.Open(
|
gdb, err := gorm.Open(
|
||||||
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{
|
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
|
||||||
// Never leave this at GORM's default. See
|
|
||||||
// internal/gormlog.
|
|
||||||
Logger: gormlog.New(w.log),
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
_ = sqlDB.Close()
|
_ = sqlDB.Close()
|
||||||
|
|||||||
@@ -1,155 +0,0 @@
|
|||||||
package delivery_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"log"
|
|
||||||
"log/slog"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
gormlogger "gorm.io/gorm/logger"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
"sneak.berlin/go/webhooker/internal/middleware"
|
|
||||||
)
|
|
||||||
|
|
||||||
// archiveGORMTailMarker sits at the far end of the value this file
|
|
||||||
// drives into an archive lookup. Its presence in a log line means the
|
|
||||||
// whole value reached the log, so nothing truncated it.
|
|
||||||
const archiveGORMTailMarker = "ENDOFCLIENTVALUE"
|
|
||||||
|
|
||||||
// archiveGORMFillBytes is how much text the lookup carries. It is far
|
|
||||||
// past every budget in play.
|
|
||||||
const archiveGORMFillBytes = 8 << 10
|
|
||||||
|
|
||||||
// gormDefaultBuf collects what GORM's package-level default logger
|
|
||||||
// writes, if anything reaches it.
|
|
||||||
type gormDefaultBuf struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
b bytes.Buffer
|
|
||||||
}
|
|
||||||
|
|
||||||
func (g *gormDefaultBuf) Write(p []byte) (int, error) {
|
|
||||||
g.mu.Lock()
|
|
||||||
defer g.mu.Unlock()
|
|
||||||
|
|
||||||
return g.b.Write(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (g *gormDefaultBuf) String() string {
|
|
||||||
g.mu.Lock()
|
|
||||||
defer g.mu.Unlock()
|
|
||||||
|
|
||||||
return g.b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// captureArchiveGORMDefault replaces GORM's package-level default
|
|
||||||
// logger with one configured exactly as GORM configures its own,
|
|
||||||
// writing to a buffer.
|
|
||||||
//
|
|
||||||
// This duplicates the detector in internal/handlers rather than
|
|
||||||
// sharing it: a test helper cannot cross a package's test boundary
|
|
||||||
// without exporting production code to carry it, and a logging
|
|
||||||
// detector is not worth a production symbol. What it detects is the
|
|
||||||
// third gorm.Open in this service, at
|
|
||||||
// internal/delivery/target_database_archive.go — the archive writer,
|
|
||||||
// whose type is unexported, so nothing outside this package can drive
|
|
||||||
// it.
|
|
||||||
func captureArchiveGORMDefault(t *testing.T) *gormDefaultBuf {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
buf := &gormDefaultBuf{}
|
|
||||||
orig := gormlogger.Default
|
|
||||||
|
|
||||||
gormlogger.Default = gormlogger.New(
|
|
||||||
log.New(buf, "", log.LstdFlags),
|
|
||||||
gormlogger.Config{
|
|
||||||
SlowThreshold: 200 * time.Millisecond,
|
|
||||||
LogLevel: gormlogger.Warn,
|
|
||||||
IgnoreRecordNotFoundError: false,
|
|
||||||
Colorful: false,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
t.Cleanup(func() { gormlogger.Default = orig })
|
|
||||||
|
|
||||||
return buf
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveWriter_NeverUsesGORMsDefaultLogger pins the archive
|
|
||||||
// writer's gorm.Open to the adapter.
|
|
||||||
//
|
|
||||||
// Restore a bare &gorm.Config{} at
|
|
||||||
// internal/delivery/target_database_archive.go and this fails: the
|
|
||||||
// default logger prints the fully interpolated SELECT on every
|
|
||||||
// ErrRecordNotFound, so the client-chosen event id below arrives whole
|
|
||||||
// and unbounded on stdout, answering to no level the operator set.
|
|
||||||
//
|
|
||||||
// Not parallel: gormlogger.Default is process-global. Go runs every
|
|
||||||
// non-parallel top-level test to completion before it resumes the
|
|
||||||
// parallel ones.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // Deliberately sequential; see above.
|
|
||||||
func TestArchiveWriter_NeverUsesGORMsDefaultLogger(t *testing.T) {
|
|
||||||
var captured bytes.Buffer
|
|
||||||
|
|
||||||
gormDefault := captureArchiveGORMDefault(t)
|
|
||||||
|
|
||||||
w := delivery.NewExportArchiveWriter(
|
|
||||||
filepath.Join(t.TempDir(), "archive.db"),
|
|
||||||
slog.New(slog.NewTextHandler(
|
|
||||||
&captured, &slog.HandlerOptions{Level: slog.LevelDebug},
|
|
||||||
)),
|
|
||||||
0,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, w.Open(0))
|
|
||||||
|
|
||||||
t.Cleanup(w.Evict)
|
|
||||||
|
|
||||||
// A lookup that misses, carrying a value the size of an inbound
|
|
||||||
// event id. Under the default logger this is the line that gets
|
|
||||||
// interpolated and printed.
|
|
||||||
value := strings.Repeat("\x01", archiveGORMFillBytes) +
|
|
||||||
archiveGORMTailMarker
|
|
||||||
|
|
||||||
var row delivery.ExportArchivedEvent
|
|
||||||
|
|
||||||
err := w.DB().Where("event_id = ?", value).First(&row).Error
|
|
||||||
require.ErrorIs(t, err, gorm.ErrRecordNotFound)
|
|
||||||
|
|
||||||
got := gormDefault.String()
|
|
||||||
assert.Empty(
|
|
||||||
t, got,
|
|
||||||
"GORM's default logger wrote %d bytes, so the archive "+
|
|
||||||
"writer's gorm.Open is back on a bare &gorm.Config{}; "+
|
|
||||||
"the first of them: %s",
|
|
||||||
len(got), got[:min(len(got), 300)],
|
|
||||||
)
|
|
||||||
|
|
||||||
// The adapter drops a miss, so this should be silent too — and
|
|
||||||
// whatever it does write stays inside the stated ceiling.
|
|
||||||
out := captured.String()
|
|
||||||
|
|
||||||
assert.NotContains(
|
|
||||||
t, out, archiveGORMTailMarker,
|
|
||||||
"the far end of the client-chosen value reached the log",
|
|
||||||
)
|
|
||||||
|
|
||||||
for line := range strings.SplitSeq(strings.TrimRight(out, "\n"), "\n") {
|
|
||||||
if line == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.LessOrEqual(
|
|
||||||
t, len(line), middleware.MaxAccessLogLineBytes,
|
|
||||||
"log line exceeded its bound: %s",
|
|
||||||
line[:min(len(line), 300)],
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,263 +0,0 @@
|
|||||||
package delivery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"slices"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// MaxTargetTimeoutSeconds bounds a per-target request timeout.
|
|
||||||
// A delivery attempt holds a worker for its whole duration, so an
|
|
||||||
// unbounded timeout lets one misconfigured target stall the queue
|
|
||||||
// indefinitely. Five minutes is far beyond any healthy webhook
|
|
||||||
// receiver and still finite.
|
|
||||||
const MaxTargetTimeoutSeconds = 300
|
|
||||||
|
|
||||||
// Errors returned when a target's header or timeout form input
|
|
||||||
// cannot be turned into a configuration.
|
|
||||||
//
|
|
||||||
// None of these ever quotes a header VALUE. A target header value
|
|
||||||
// is routinely an authorization token, and these messages are shown
|
|
||||||
// to the user in an error page body.
|
|
||||||
var (
|
|
||||||
errHeaderLineMalformed = errors.New(
|
|
||||||
`each header line must be "Name: value"`,
|
|
||||||
)
|
|
||||||
errHeaderNameInvalid = errors.New(
|
|
||||||
"header name must be a valid HTTP token",
|
|
||||||
)
|
|
||||||
errHeaderValueInvalid = errors.New(
|
|
||||||
"header value must not contain control characters",
|
|
||||||
)
|
|
||||||
errHeaderDuplicate = errors.New(
|
|
||||||
"header given more than once",
|
|
||||||
)
|
|
||||||
errHeaderReserved = errors.New(
|
|
||||||
"header is set by the delivery engine and cannot be " +
|
|
||||||
"overridden",
|
|
||||||
)
|
|
||||||
errTimeoutInvalid = errors.New(
|
|
||||||
"timeout must be a whole number of seconds",
|
|
||||||
)
|
|
||||||
errTimeoutOutOfRange = errors.New(
|
|
||||||
"timeout is out of range",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
// isReservedTargetHeader reports whether name (canonicalised) is a
|
|
||||||
// header a target configuration may not set, because the delivery
|
|
||||||
// path or net/http itself writes it regardless.
|
|
||||||
//
|
|
||||||
// These are rejected rather than accepted-and-ignored. Storing a
|
|
||||||
// header that provably never reaches the wire tells the operator
|
|
||||||
// their configuration took effect when it did not, which is the
|
|
||||||
// same failure mode as silently substituting a default for an
|
|
||||||
// invalid value.
|
|
||||||
func isReservedTargetHeader(name string) bool {
|
|
||||||
switch name {
|
|
||||||
case "Host", "Content-Length", "Transfer-Encoding", "Connection":
|
|
||||||
return true
|
|
||||||
case "User-Agent":
|
|
||||||
// applyRequestHeaders sets the User-Agent after it applies
|
|
||||||
// the configured headers, so a configured one would always
|
|
||||||
// be overwritten.
|
|
||||||
return true
|
|
||||||
case "Trailer":
|
|
||||||
// net/http strips Trailer from the request it writes
|
|
||||||
// (reqWriteExcludeHeader), so a configured one is accepted
|
|
||||||
// and stored and then provably never reaches the wire.
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ParseTargetHeaders turns the target form's headers field — one
|
|
||||||
// "Name: value" pair per line, blank lines ignored — into the map
|
|
||||||
// stored in HTTPTargetConfig.Headers. Names are canonicalised, so a
|
|
||||||
// name repeated in a different case is still a duplicate rather than
|
|
||||||
// one pair silently overwriting the other.
|
|
||||||
//
|
|
||||||
// An input with no pairs yields an empty map, which omitempty drops
|
|
||||||
// from the stored config: a target configured with no headers keeps
|
|
||||||
// the same config JSON it had before this field existed.
|
|
||||||
func ParseTargetHeaders(raw string) (map[string]string, error) {
|
|
||||||
headers := make(map[string]string)
|
|
||||||
|
|
||||||
for i, line := range strings.Split(raw, "\n") {
|
|
||||||
lineNum := i + 1
|
|
||||||
|
|
||||||
line = strings.TrimSpace(line)
|
|
||||||
if line == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
name, value, err := parseHeaderLine(line)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("line %d: %w", lineNum, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, dup := headers[name]; dup {
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"line %d: %w: %q", lineNum,
|
|
||||||
errHeaderDuplicate, name,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
headers[name] = value
|
|
||||||
}
|
|
||||||
|
|
||||||
return headers, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseHeaderLine splits and validates one "Name: value" line,
|
|
||||||
// returning the canonicalised name and the trimmed value.
|
|
||||||
func parseHeaderLine(line string) (string, string, error) {
|
|
||||||
rawName, value, found := strings.Cut(line, ":")
|
|
||||||
if !found {
|
|
||||||
return "", "", errHeaderLineMalformed
|
|
||||||
}
|
|
||||||
|
|
||||||
rawName = strings.TrimSpace(rawName)
|
|
||||||
if !validHeaderName(rawName) {
|
|
||||||
// Quotes nothing. The text before the first colon is only
|
|
||||||
// a name if it parses as one; when it does not, it is as
|
|
||||||
// likely to be a pasted value whose own colon split the
|
|
||||||
// line, and half of a token would be echoed into the 400.
|
|
||||||
return "", "", errHeaderNameInvalid
|
|
||||||
}
|
|
||||||
|
|
||||||
name := http.CanonicalHeaderKey(rawName)
|
|
||||||
if isReservedTargetHeader(name) {
|
|
||||||
return "", "", fmt.Errorf(
|
|
||||||
"%w: %q", errHeaderReserved, name,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
value = strings.TrimSpace(value)
|
|
||||||
if !validHeaderValue(value) {
|
|
||||||
return "", "", fmt.Errorf(
|
|
||||||
"%w: %q", errHeaderValueInvalid, name,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return name, value, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// validHeaderName reports whether name is a non-empty RFC 9110
|
|
||||||
// field name. Rejecting anything else here is what keeps a value
|
|
||||||
// containing CR or LF from being smuggled in as part of a name and
|
|
||||||
// injecting a second header into the outbound request.
|
|
||||||
func validHeaderName(name string) bool {
|
|
||||||
if name == "" {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := range len(name) {
|
|
||||||
if !isTokenByte(name[i]) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// isTokenByte reports whether c is a "tchar" per RFC 9110 5.6.2.
|
|
||||||
func isTokenByte(c byte) bool {
|
|
||||||
switch {
|
|
||||||
case c >= 'a' && c <= 'z',
|
|
||||||
c >= 'A' && c <= 'Z',
|
|
||||||
c >= '0' && c <= '9':
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
return strings.IndexByte("!#$%&'*+-.^_`|~", c) >= 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// validHeaderValue reports whether value is a legal field value:
|
|
||||||
// no control characters, which is the other half of the header
|
|
||||||
// injection guard. An empty value is legal.
|
|
||||||
func validHeaderValue(value string) bool {
|
|
||||||
for i := range len(value) {
|
|
||||||
c := value[i]
|
|
||||||
if c < 0x20 || c == 0x7f {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// FormatTargetHeaders renders a stored header map back into the
|
|
||||||
// form's textarea representation, one "Name: value" per line.
|
|
||||||
//
|
|
||||||
// Names are sorted so that loading the edit form twice without
|
|
||||||
// saving produces identical text; Go map iteration order would
|
|
||||||
// otherwise reshuffle the field on every render.
|
|
||||||
func FormatTargetHeaders(headers map[string]string) string {
|
|
||||||
if len(headers) == 0 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
names := make([]string, 0, len(headers))
|
|
||||||
for name := range headers {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
|
|
||||||
slices.Sort(names)
|
|
||||||
|
|
||||||
var b strings.Builder
|
|
||||||
|
|
||||||
for _, name := range names {
|
|
||||||
b.WriteString(name)
|
|
||||||
b.WriteString(": ")
|
|
||||||
b.WriteString(headers[name])
|
|
||||||
b.WriteString("\n")
|
|
||||||
}
|
|
||||||
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// ParseTargetTimeout interprets the target form's timeout field as
|
|
||||||
// a whole number of seconds. An empty field means "unset" and yields
|
|
||||||
// 0, which omitempty drops from the stored config and which the
|
|
||||||
// delivery path reads as "use the shared client's timeout".
|
|
||||||
//
|
|
||||||
// Anything else that is not a whole number in range is an error, not
|
|
||||||
// a silently substituted default: a target whose timeout was typed
|
|
||||||
// wrong must say so at the form rather than deliver on a timeout its
|
|
||||||
// operator did not choose.
|
|
||||||
func ParseTargetTimeout(raw string) (int, error) {
|
|
||||||
raw = strings.TrimSpace(raw)
|
|
||||||
if raw == "" {
|
|
||||||
return 0, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
v, err := strconv.Atoi(raw)
|
|
||||||
if err != nil || v < 0 {
|
|
||||||
return 0, errTimeoutInvalid
|
|
||||||
}
|
|
||||||
|
|
||||||
if v > MaxTargetTimeoutSeconds {
|
|
||||||
return 0, fmt.Errorf(
|
|
||||||
"%w: at most %d seconds",
|
|
||||||
errTimeoutOutOfRange, MaxTargetTimeoutSeconds,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return v, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// FormatTargetTimeout renders a stored timeout for the form field.
|
|
||||||
// An unset timeout renders as an empty field rather than "0", so the
|
|
||||||
// placeholder can describe the default the target actually uses.
|
|
||||||
func FormatTargetTimeout(timeout int) string {
|
|
||||||
if timeout <= 0 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
return strconv.Itoa(timeout)
|
|
||||||
}
|
|
||||||
@@ -1,292 +0,0 @@
|
|||||||
package delivery_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"strconv"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Literals these tests repeat, named so that the header name and the
|
|
||||||
// keep-forever archive config each have one definition.
|
|
||||||
const (
|
|
||||||
headerAuthorization = "Authorization"
|
|
||||||
bearerValue = "Bearer abc"
|
|
||||||
archiveConfigNever = "{\"expiry\":\"never\"}"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestParseTargetHeaders_AcceptsPairs(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
got, err := delivery.ParseTargetHeaders(
|
|
||||||
" Authorization: Bearer abc \n\n" +
|
|
||||||
"x-tenant:acme\r\n" +
|
|
||||||
"X-Empty:\n",
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
map[string]string{
|
|
||||||
headerAuthorization: bearerValue,
|
|
||||||
"X-Tenant": "acme",
|
|
||||||
"X-Empty": "",
|
|
||||||
},
|
|
||||||
got,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A configuration with no headers must stay indistinguishable from
|
|
||||||
// one written before the field existed, so omitempty drops the key.
|
|
||||||
func TestParseTargetHeaders_EmptyInputYieldsNoHeaders(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
got, err := delivery.ParseTargetHeaders("\n \n")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Empty(t, got)
|
|
||||||
|
|
||||||
encoded, err := json.Marshal(delivery.HTTPTargetConfig{
|
|
||||||
URL: "https://example.com/h",
|
|
||||||
Headers: got,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.JSONEq(
|
|
||||||
t, `{"url":"https://example.com/h"}`, string(encoded),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseTargetHeaders_Rejects(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cases := map[string]string{
|
|
||||||
"no colon": "Authorization Bearer abc",
|
|
||||||
"empty name": ": value",
|
|
||||||
"space in name": "X Bad: value",
|
|
||||||
"reserved host": "Host: evil.example",
|
|
||||||
"reserved ua": "User-Agent: curl/8",
|
|
||||||
"reserved length": "Content-Length: 0",
|
|
||||||
"duplicate any case": "X-A: 1\nx-a: 2",
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, input := range cases {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := delivery.ParseTargetHeaders(input)
|
|
||||||
require.Error(t, err)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// net/http strips Trailer from the request it writes, so accepting
|
|
||||||
// one would store a header that never reaches the target.
|
|
||||||
func TestParseTargetHeaders_RejectsTrailer(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
_, err := delivery.ParseTargetHeaders("Trailer: X-Checksum")
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), "Trailer")
|
|
||||||
}
|
|
||||||
|
|
||||||
// A header value is routinely a bearer token and these errors are
|
|
||||||
// rendered into a 400 body, so no message may quote one.
|
|
||||||
func TestParseTargetHeaders_ErrorsNeverQuoteAValue(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const secret = "QQNEVERINAMESSAGEQQ"
|
|
||||||
|
|
||||||
inputs := []string{
|
|
||||||
// The value, after the colon, in a duplicate name.
|
|
||||||
"X-A: " + secret + "\nx-a: " + secret,
|
|
||||||
// The value after the colon of an unusable name.
|
|
||||||
"X Bad Name: " + secret,
|
|
||||||
// The line splits on the value's own colon, so the
|
|
||||||
// secret lands in the text an unusable-name error is
|
|
||||||
// tempted to quote as the name.
|
|
||||||
"X-Api-Key " + secret + ":x",
|
|
||||||
// The same, with nothing before the secret at all.
|
|
||||||
secret + " and more:x",
|
|
||||||
// A control character in the value.
|
|
||||||
"X-A: " + secret + "\x01",
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, input := range inputs {
|
|
||||||
_, err := delivery.ParseTargetHeaders(input)
|
|
||||||
require.Error(t, err, input)
|
|
||||||
assert.NotContains(t, err.Error(), secret, input)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Loading the edit form twice without saving must not reshuffle
|
|
||||||
// the textarea, which Go's map iteration order would otherwise do.
|
|
||||||
func TestFormatTargetHeaders_IsSorted(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
got := delivery.FormatTargetHeaders(map[string]string{
|
|
||||||
"X-Zed": "z",
|
|
||||||
headerAuthorization: bearerValue,
|
|
||||||
"X-Alpha": "a",
|
|
||||||
})
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
"Authorization: Bearer abc\nX-Alpha: a\nX-Zed: z\n",
|
|
||||||
got,
|
|
||||||
)
|
|
||||||
assert.Empty(t, delivery.FormatTargetHeaders(nil))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFormatTargetHeaders_RoundTripsThroughParse(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
want := map[string]string{
|
|
||||||
headerAuthorization: bearerValue,
|
|
||||||
"X-Tenant": "acme",
|
|
||||||
}
|
|
||||||
|
|
||||||
got, err := delivery.ParseTargetHeaders(
|
|
||||||
delivery.FormatTargetHeaders(want),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, want, got)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseTargetTimeout(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
got, err := delivery.ParseTargetTimeout(" 30 ")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, 30, got)
|
|
||||||
|
|
||||||
got, err = delivery.ParseTargetTimeout("")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Zero(t, got)
|
|
||||||
|
|
||||||
for _, bad := range []string{"soon", "-1", "1e3", "100000"} {
|
|
||||||
_, err = delivery.ParseTargetTimeout(bad)
|
|
||||||
require.Error(t, err, bad)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFormatTargetTimeout(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
assert.Equal(t, "30", delivery.FormatTargetTimeout(30))
|
|
||||||
assert.Empty(t, delivery.FormatTargetTimeout(0))
|
|
||||||
assert.Empty(t, delivery.FormatTargetTimeout(-1))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewTargetConfigForm(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
form, err := delivery.NewTargetConfigForm(&database.Target{
|
|
||||||
Type: database.TargetTypeHTTP,
|
|
||||||
Config: `{"url":"https://example.com/h",` +
|
|
||||||
`"headers":{"Authorization":"Bearer abc"},` +
|
|
||||||
`"timeout":9}`,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "https://example.com/h", form.URL)
|
|
||||||
assert.Equal(t, "Authorization: Bearer abc\n", form.Headers)
|
|
||||||
assert.Equal(t, "9", form.Timeout)
|
|
||||||
|
|
||||||
form, err = delivery.NewTargetConfigForm(&database.Target{
|
|
||||||
Type: database.TargetTypeSlack,
|
|
||||||
Config: `{"webhookUrl":"https://hooks.example/s"}`,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "https://hooks.example/s", form.URL)
|
|
||||||
|
|
||||||
form, err = delivery.NewTargetConfigForm(&database.Target{
|
|
||||||
Type: database.TargetTypeDatabase,
|
|
||||||
Config: `{"expiry":"720h"}`,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "720h", form.Expiry)
|
|
||||||
|
|
||||||
form, err = delivery.NewTargetConfigForm(&database.Target{
|
|
||||||
Type: database.TargetTypeLog,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Empty(t, form.URL)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A keep-forever archive target must pre-fill as an empty field, so
|
|
||||||
// saving the form back unchanged stores the same empty config.
|
|
||||||
func TestNewTargetConfigForm_DatabaseNeverIsBlank(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, cfg := range []string{"", `{}`, archiveConfigNever} {
|
|
||||||
form, err := delivery.NewTargetConfigForm(
|
|
||||||
&database.Target{
|
|
||||||
Type: database.TargetTypeDatabase,
|
|
||||||
Config: cfg,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
require.NoError(t, err, cfg)
|
|
||||||
assert.Empty(t, form.Expiry, cfg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// An unreadable stored config is an error rather than a blank form
|
|
||||||
// that looks like a target with no settings, so the caller can tell
|
|
||||||
// the operator that saving replaces the stored value.
|
|
||||||
func TestNewTargetConfigForm_UnreadableConfigErrors(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cases := []*database.Target{
|
|
||||||
{Type: database.TargetTypeHTTP, Config: "not json"},
|
|
||||||
{Type: database.TargetTypeHTTP, Config: `{}`},
|
|
||||||
{Type: database.TargetTypeSlack, Config: ""},
|
|
||||||
{
|
|
||||||
Type: database.TargetTypeDatabase,
|
|
||||||
Config: `{"expiry":"soon"}`,
|
|
||||||
},
|
|
||||||
{Type: database.TargetType("nope")},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, target := range cases {
|
|
||||||
_, err := delivery.NewTargetConfigForm(target)
|
|
||||||
require.Error(t, err, target.Type)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The ceiling exists so one misconfigured target cannot hold a
|
|
||||||
// delivery worker indefinitely, and it is inclusive.
|
|
||||||
func TestParseTargetTimeout_CeilingIsInclusive(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
assert.Positive(t, delivery.MaxTargetTimeoutSeconds)
|
|
||||||
|
|
||||||
got, err := delivery.ParseTargetTimeout(
|
|
||||||
strconv.Itoa(delivery.MaxTargetTimeoutSeconds),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, delivery.MaxTargetTimeoutSeconds, got)
|
|
||||||
|
|
||||||
_, err = delivery.ParseTargetTimeout(
|
|
||||||
strconv.Itoa(delivery.MaxTargetTimeoutSeconds + 1),
|
|
||||||
)
|
|
||||||
require.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Control characters in a value are how a second header would be
|
|
||||||
// smuggled into the outbound request.
|
|
||||||
func TestParseTargetHeaders_RejectsControlCharactersInValues(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, bad := range []string{
|
|
||||||
"X-A: one\x01two",
|
|
||||||
"X-A: one\ttwo",
|
|
||||||
"X-A: one\x7ftwo",
|
|
||||||
} {
|
|
||||||
_, err := delivery.ParseTargetHeaders(bad)
|
|
||||||
require.Error(t, err, bad)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -8,7 +8,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"sort"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -75,8 +74,6 @@ func (c *httpCore) fireAndForget(
|
|||||||
d *database.Delivery,
|
d *database.Delivery,
|
||||||
res attemptResult,
|
res attemptResult,
|
||||||
) {
|
) {
|
||||||
c.eng.observeAttempt(d.Target.Type, res.elapsed())
|
|
||||||
|
|
||||||
c.eng.recordResult(
|
c.eng.recordResult(
|
||||||
webhookDB, d, 1, res.success,
|
webhookDB, d, 1, res.success,
|
||||||
res.statusCode, res.respBody, res.errMsg,
|
res.statusCode, res.respBody, res.errMsg,
|
||||||
@@ -85,7 +82,7 @@ func (c *httpCore) fireAndForget(
|
|||||||
|
|
||||||
if res.success {
|
if res.success {
|
||||||
c.eng.updateDeliveryStatus(
|
c.eng.updateDeliveryStatus(
|
||||||
webhookDB, d, d.Target.Type,
|
webhookDB, d,
|
||||||
database.DeliveryStatusDelivered,
|
database.DeliveryStatusDelivered,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -93,8 +90,7 @@ func (c *httpCore) fireAndForget(
|
|||||||
}
|
}
|
||||||
|
|
||||||
c.eng.updateDeliveryStatus(
|
c.eng.updateDeliveryStatus(
|
||||||
webhookDB, d, d.Target.Type,
|
webhookDB, d, database.DeliveryStatusFailed,
|
||||||
database.DeliveryStatusFailed,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -111,17 +107,10 @@ func (c *httpCore) withRetry(
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Allow may have moved the breaker to half-open, and the
|
|
||||||
// attempt below may open or close it, so the gauge is
|
|
||||||
// republished on every exit from here.
|
|
||||||
defer c.publishCircuitState(d.Target.Type)
|
|
||||||
|
|
||||||
attemptNum := task.AttemptNum
|
attemptNum := task.AttemptNum
|
||||||
|
|
||||||
res := attempt()
|
res := attempt()
|
||||||
|
|
||||||
c.eng.observeAttempt(d.Target.Type, res.elapsed())
|
|
||||||
|
|
||||||
c.eng.recordResult(
|
c.eng.recordResult(
|
||||||
webhookDB, d, attemptNum, res.success,
|
webhookDB, d, attemptNum, res.success,
|
||||||
res.statusCode, res.respBody, res.errMsg,
|
res.statusCode, res.respBody, res.errMsg,
|
||||||
@@ -132,7 +121,7 @@ func (c *httpCore) withRetry(
|
|||||||
cb.RecordSuccess()
|
cb.RecordSuccess()
|
||||||
|
|
||||||
c.eng.updateDeliveryStatus(
|
c.eng.updateDeliveryStatus(
|
||||||
webhookDB, d, d.Target.Type,
|
webhookDB, d,
|
||||||
database.DeliveryStatusDelivered,
|
database.DeliveryStatusDelivered,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -157,8 +146,6 @@ func (c *httpCore) circuitBreakerBlock(
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
defer c.publishCircuitState(d.Target.Type)
|
|
||||||
|
|
||||||
remaining := cb.CooldownRemaining()
|
remaining := cb.CooldownRemaining()
|
||||||
|
|
||||||
c.eng.log.Info(
|
c.eng.log.Info(
|
||||||
@@ -170,7 +157,7 @@ func (c *httpCore) circuitBreakerBlock(
|
|||||||
)
|
)
|
||||||
|
|
||||||
c.eng.updateDeliveryStatus(
|
c.eng.updateDeliveryStatus(
|
||||||
webhookDB, d, d.Target.Type,
|
webhookDB, d,
|
||||||
database.DeliveryStatusRetrying,
|
database.DeliveryStatusRetrying,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -190,7 +177,7 @@ func (c *httpCore) handleRetry(
|
|||||||
) {
|
) {
|
||||||
if attemptNum >= maxRetries {
|
if attemptNum >= maxRetries {
|
||||||
c.eng.updateDeliveryStatus(
|
c.eng.updateDeliveryStatus(
|
||||||
webhookDB, d, d.Target.Type,
|
webhookDB, d,
|
||||||
database.DeliveryStatusFailed,
|
database.DeliveryStatusFailed,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -198,8 +185,7 @@ func (c *httpCore) handleRetry(
|
|||||||
}
|
}
|
||||||
|
|
||||||
c.eng.updateDeliveryStatus(
|
c.eng.updateDeliveryStatus(
|
||||||
webhookDB, d, d.Target.Type,
|
webhookDB, d, database.DeliveryStatusRetrying,
|
||||||
database.DeliveryStatusRetrying,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
backoff := calcBackoff(attemptNum)
|
backoff := calcBackoff(attemptNum)
|
||||||
@@ -229,28 +215,6 @@ func (c *httpCore) getCircuitBreaker(
|
|||||||
return cb
|
return cb
|
||||||
}
|
}
|
||||||
|
|
||||||
// publishCircuitState recounts this core's open breakers and
|
|
||||||
// publishes the gauge. Each core holds the breakers of exactly one
|
|
||||||
// target type, so the recount is over that type's targets alone.
|
|
||||||
// Counting rather than adjusting a delta keeps the gauge honest
|
|
||||||
// however a breaker changed state.
|
|
||||||
func (c *httpCore) publishCircuitState(
|
|
||||||
targetType database.TargetType,
|
|
||||||
) {
|
|
||||||
open := 0
|
|
||||||
|
|
||||||
c.circuitBreakers.Range(func(_, val any) bool {
|
|
||||||
cb, ok := val.(*CircuitBreaker)
|
|
||||||
if ok && cb.State() == CircuitOpen {
|
|
||||||
open++
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
})
|
|
||||||
|
|
||||||
c.eng.mtr.SetCircuitBreakersOpen(targetType, open)
|
|
||||||
}
|
|
||||||
|
|
||||||
// remainingBackoff returns how long remains of the backoff
|
// remainingBackoff returns how long remains of the backoff
|
||||||
// window for the last attempt of a recovered retrying
|
// window for the last attempt of a recovered retrying
|
||||||
// delivery. It implements rescheduler.
|
// delivery. It implements rescheduler.
|
||||||
@@ -338,8 +302,7 @@ func (t *httpTarget) Deliver(
|
|||||||
)
|
)
|
||||||
|
|
||||||
t.eng.updateDeliveryStatus(
|
t.eng.updateDeliveryStatus(
|
||||||
webhookDB, d, d.Target.Type,
|
webhookDB, d, database.DeliveryStatusFailed,
|
||||||
database.DeliveryStatusFailed,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -400,14 +363,13 @@ func (t *httpTarget) doHTTPRequest(
|
|||||||
)
|
)
|
||||||
if reqErr != nil {
|
if reqErr != nil {
|
||||||
return 0, "", 0, fmt.Errorf(
|
return 0, "", 0, fmt.Errorf(
|
||||||
"creating request: %w",
|
"creating request: %w", reqErr,
|
||||||
maskURLError(reqErr),
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
originScoped := applyRequestHeaders(req, event, cfg)
|
applyRequestHeaders(req, event, cfg)
|
||||||
|
|
||||||
client := t.clientForRequest(cfg, originScoped)
|
client := t.clientForConfig(cfg)
|
||||||
|
|
||||||
resp, doErr := executeHTTPRequest(client, req)
|
resp, doErr := executeHTTPRequest(client, req)
|
||||||
|
|
||||||
@@ -433,41 +395,23 @@ func (t *httpTarget) doHTTPRequest(
|
|||||||
return resp.StatusCode, string(body), dur, nil
|
return resp.StatusCode, string(body), dur, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// clientForRequest returns the client for one delivery attempt.
|
func (t *httpTarget) clientForConfig(
|
||||||
// originScoped is the header set applyRequestHeaders built for that
|
|
||||||
// attempt; a request with neither a per-target timeout nor an
|
|
||||||
// origin-scoped header gets the shared client, because there is
|
|
||||||
// then nothing for the redirect policy to strip and net/http's
|
|
||||||
// default policy already withholds Authorization and Cookie across
|
|
||||||
// hosts.
|
|
||||||
func (t *httpTarget) clientForRequest(
|
|
||||||
cfg *HTTPTargetConfig,
|
cfg *HTTPTargetConfig,
|
||||||
originScoped []string,
|
|
||||||
) *http.Client {
|
) *http.Client {
|
||||||
if cfg.Timeout <= 0 && len(originScoped) == 0 {
|
if cfg.Timeout > 0 {
|
||||||
return t.client
|
// Reuse the shared client's SSRF-safe transport so
|
||||||
}
|
// a per-target timeout does not drop the
|
||||||
|
// request-time private-IP guard. Only the timeout
|
||||||
// Reuse the shared client's SSRF-safe transport so neither a
|
// is overridden.
|
||||||
// per-target timeout nor the redirect policy drops the
|
return &http.Client{
|
||||||
// request-time private-IP guard — which, being a dial hook,
|
Timeout: time.Duration(
|
||||||
// also covers every redirect hop.
|
cfg.Timeout,
|
||||||
client := &http.Client{
|
) * time.Second,
|
||||||
Timeout: t.client.Timeout,
|
|
||||||
Transport: t.client.Transport,
|
Transport: t.client.Transport,
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.Timeout > 0 {
|
|
||||||
client.Timeout = time.Duration(
|
|
||||||
cfg.Timeout,
|
|
||||||
) * time.Second
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(originScoped) > 0 {
|
return t.client
|
||||||
client.CheckRedirect = offOriginHeaderPolicy(originScoped)
|
|
||||||
}
|
|
||||||
|
|
||||||
return client
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func parseHTTPConfig(
|
func parseHTTPConfig(
|
||||||
@@ -509,106 +453,47 @@ func isForwardableHeader(name string) bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// applyRequestHeaders builds one outbound delivery's header set and
|
|
||||||
// returns the canonical names of every header in it that is scoped
|
|
||||||
// to the configured origin: the inbound event headers this delivery
|
|
||||||
// forwarded, plus the operator's configured headers. The redirect
|
|
||||||
// policy strips exactly that set on a hop that leaves the origin,
|
|
||||||
// so the forward set is decided here and only here — a header added
|
|
||||||
// to it is covered off-origin without a second edit elsewhere.
|
|
||||||
func applyRequestHeaders(
|
func applyRequestHeaders(
|
||||||
req *http.Request,
|
req *http.Request,
|
||||||
event *database.Event,
|
event *database.Event,
|
||||||
cfg *HTTPTargetConfig,
|
cfg *HTTPTargetConfig,
|
||||||
) []string {
|
) {
|
||||||
if event.ContentType != "" {
|
if event.ContentType != "" {
|
||||||
req.Header.Set(
|
req.Header.Set(
|
||||||
"Content-Type", event.ContentType,
|
"Content-Type", event.ContentType,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
originScoped := forwardEventHeaders(req, event)
|
var originalHeaders map[string][]string
|
||||||
|
|
||||||
for k, v := range cfg.Headers {
|
|
||||||
req.Header.Set(k, v)
|
|
||||||
originScoped[http.CanonicalHeaderKey(k)] = struct{}{}
|
|
||||||
}
|
|
||||||
|
|
||||||
req.Header.Set("User-Agent", "webhooker/1.0")
|
|
||||||
|
|
||||||
// Content-Type describes the body being sent rather than the
|
|
||||||
// sender, and the delivery path sets it from the event itself.
|
|
||||||
// A 307/308 preserves the body across hosts, so stripping it
|
|
||||||
// would send that body untyped.
|
|
||||||
delete(originScoped, "Content-Type")
|
|
||||||
|
|
||||||
// User-Agent is overwritten just above, so an inbound one never
|
|
||||||
// reaches the wire and the value that does identifies this
|
|
||||||
// delivery path rather than the sender. Reporting it would strip
|
|
||||||
// it off-origin and leave net/http's own default in its place.
|
|
||||||
delete(originScoped, "User-Agent")
|
|
||||||
|
|
||||||
names := make([]string, 0, len(originScoped))
|
|
||||||
for name := range originScoped {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
|
|
||||||
sort.Strings(names)
|
|
||||||
|
|
||||||
return names
|
|
||||||
}
|
|
||||||
|
|
||||||
// forwardEventHeaders copies the inbound event's forwardable
|
|
||||||
// headers onto the outbound request and returns the canonical names
|
|
||||||
// it forwarded. Headers the event never carried are absent from the
|
|
||||||
// result, so the redirect policy strips what was actually sent.
|
|
||||||
func forwardEventHeaders(
|
|
||||||
req *http.Request,
|
|
||||||
event *database.Event,
|
|
||||||
) map[string]struct{} {
|
|
||||||
forwarded := make(map[string]struct{})
|
|
||||||
|
|
||||||
if event.Headers == "" {
|
|
||||||
return forwarded
|
|
||||||
}
|
|
||||||
|
|
||||||
var inbound map[string][]string
|
|
||||||
|
|
||||||
if json.Unmarshal([]byte(event.Headers), &inbound) != nil {
|
|
||||||
return forwarded
|
|
||||||
}
|
|
||||||
|
|
||||||
for k, vals := range inbound {
|
|
||||||
if !isForwardableHeader(k) || len(vals) == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
|
if event.Headers != "" {
|
||||||
|
jsonErr := json.Unmarshal(
|
||||||
|
[]byte(event.Headers),
|
||||||
|
&originalHeaders,
|
||||||
|
)
|
||||||
|
if jsonErr == nil {
|
||||||
|
for k, vals := range originalHeaders {
|
||||||
|
if isForwardableHeader(k) {
|
||||||
for _, v := range vals {
|
for _, v := range vals {
|
||||||
req.Header.Add(k, v)
|
req.Header.Add(k, v)
|
||||||
}
|
}
|
||||||
|
}
|
||||||
forwarded[http.CanonicalHeaderKey(k)] = struct{}{}
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return forwarded
|
for k, v := range cfg.Headers {
|
||||||
|
req.Header.Set(k, v)
|
||||||
|
}
|
||||||
|
|
||||||
|
req.Header.Set("User-Agent", "webhooker/1.0")
|
||||||
}
|
}
|
||||||
|
|
||||||
// executeHTTPRequest sends an HTTP request using the provided
|
// executeHTTPRequest sends an HTTP request using the provided
|
||||||
// client. URLs are validated by the config parsers and the
|
// client. URLs are validated by the config parsers and the
|
||||||
// SSRF-safe transport before reaching here.
|
// SSRF-safe transport before reaching here.
|
||||||
//
|
|
||||||
// Transport failures are masked here, at the single point
|
|
||||||
// where every target's request errors are born, because the
|
|
||||||
// caller stores them in DeliveryResult.Error: an unmasked
|
|
||||||
// *url.Error would write the target URL — the credential for
|
|
||||||
// a Slack incoming webhook — into the per-webhook database.
|
|
||||||
func executeHTTPRequest(
|
func executeHTTPRequest(
|
||||||
client *http.Client, req *http.Request,
|
client *http.Client, req *http.Request,
|
||||||
) (*http.Response, error) {
|
) (*http.Response, error) {
|
||||||
resp, err := client.Do(req) //#nosec G704 -- validated URL, SSRF-safe transport
|
return client.Do(req) //#nosec G704 -- validated URL, SSRF-safe transport
|
||||||
if err != nil {
|
|
||||||
return nil, maskURLError(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return resp, nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,142 +0,0 @@
|
|||||||
package delivery_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
"sneak.berlin/go/webhooker/internal/signature"
|
|
||||||
)
|
|
||||||
|
|
||||||
// gitlabDeliverySecret is the shared secret the entrypoint in these
|
|
||||||
// tests is configured with. No outbound request may contain it.
|
|
||||||
const gitlabDeliverySecret = "QQDELIVERYSECRETQQ"
|
|
||||||
|
|
||||||
// receivedEventHeaders builds the Event.Headers value the receiver
|
|
||||||
// stores for an inbound request, by running the request's headers
|
|
||||||
// through the same sanitizer the receive path uses. Going through
|
|
||||||
// signature.SanitizeHeaders rather than a literal is the point of
|
|
||||||
// the test: it joins the two egresses at the field they share, so a
|
|
||||||
// regression at either end shows up here.
|
|
||||||
func receivedEventHeaders(
|
|
||||||
t *testing.T,
|
|
||||||
scheme database.SignatureScheme,
|
|
||||||
inbound http.Header,
|
|
||||||
) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
ep := &database.Entrypoint{
|
|
||||||
SignatureScheme: scheme,
|
|
||||||
SignatureSecret: gitlabDeliverySecret,
|
|
||||||
}
|
|
||||||
|
|
||||||
encoded, err := json.Marshal(
|
|
||||||
signature.SanitizeHeaders(ep, inbound),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
return string(encoded)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestApplyRequestHeadersDropsInboundCredential proves a delivery to
|
|
||||||
// an HTTP target does not carry the GitLab shared secret.
|
|
||||||
//
|
|
||||||
// isForwardableHeader is a blocklist of hop-by-hop names, so it
|
|
||||||
// forwards X-Gitlab-Token like any other header; what keeps the
|
|
||||||
// secret out of the outbound request is that the receiver never
|
|
||||||
// stored it. Handing a target operator the token would hand them the
|
|
||||||
// ability to forge requests to the entrypoint it authenticates,
|
|
||||||
// which is the one control the receiver has.
|
|
||||||
func TestApplyRequestHeadersDropsInboundCredential(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
inbound := http.Header{}
|
|
||||||
inbound.Set(signature.HeaderGitLab, gitlabDeliverySecret)
|
|
||||||
inbound.Set("X-Gitlab-Event", "Push Hook")
|
|
||||||
|
|
||||||
event := &database.Event{
|
|
||||||
Headers: receivedEventHeaders(
|
|
||||||
t, database.SignatureSchemeGitLab, inbound,
|
|
||||||
),
|
|
||||||
ContentType: "application/json",
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(
|
|
||||||
context.Background(),
|
|
||||||
http.MethodPost,
|
|
||||||
"https://target.example.com/hook",
|
|
||||||
http.NoBody,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
delivery.ExportApplyRequestHeaders(
|
|
||||||
req, event, &delivery.HTTPTargetConfig{},
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Empty(
|
|
||||||
t,
|
|
||||||
req.Header.Values(signature.HeaderGitLab),
|
|
||||||
"the shared secret header must not reach a target",
|
|
||||||
)
|
|
||||||
|
|
||||||
// Header.Values canonicalises, so a differently-cased spelling
|
|
||||||
// would be caught above; this catches the value arriving under
|
|
||||||
// some other name.
|
|
||||||
for name, values := range req.Header {
|
|
||||||
for _, v := range values {
|
|
||||||
assert.NotContains(
|
|
||||||
t, v, gitlabDeliverySecret,
|
|
||||||
"secret present in outbound header %s", name,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The rest of the sender's headers still arrive. A fix that
|
|
||||||
// dropped everything would pass the assertions above while
|
|
||||||
// breaking delivery.
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
"Push Hook",
|
|
||||||
req.Header.Get("X-Gitlab-Event"),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestApplyRequestHeadersKeepsGitHubDigest proves the stripping is
|
|
||||||
// scoped to headers that carry the secret itself. GitHub's
|
|
||||||
// X-Hub-Signature-256 is an HMAC over the body, so a target can be
|
|
||||||
// shown it without being handed the key.
|
|
||||||
func TestApplyRequestHeadersKeepsGitHubDigest(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const digest = "sha256=deadbeef"
|
|
||||||
|
|
||||||
inbound := http.Header{}
|
|
||||||
inbound.Set(signature.HeaderGitHub, digest)
|
|
||||||
|
|
||||||
event := &database.Event{
|
|
||||||
Headers: receivedEventHeaders(
|
|
||||||
t, database.SignatureSchemeGitHub, inbound,
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(
|
|
||||||
context.Background(),
|
|
||||||
http.MethodPost,
|
|
||||||
"https://target.example.com/hook",
|
|
||||||
http.NoBody,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
delivery.ExportApplyRequestHeaders(
|
|
||||||
req, event, &delivery.HTTPTargetConfig{},
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t, digest, req.Header.Get(signature.HeaderGitHub),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -2,7 +2,6 @@ package delivery
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"time"
|
|
||||||
|
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
@@ -12,17 +11,6 @@ import (
|
|||||||
// inbound webhook — the full request body and headers, plus
|
// inbound webhook — the full request body and headers, plus
|
||||||
// the method, content type, and the webhook and entrypoint
|
// the method, content type, and the webhook and entrypoint
|
||||||
// ids — then records a single successful attempt.
|
// ids — then records a single successful attempt.
|
||||||
//
|
|
||||||
// This is the one log call in the service that deliberately writes
|
|
||||||
// unbounded client-chosen bytes, so it is the one exception to the
|
|
||||||
// per-field budgets in internal/logfield and to the ceiling stated on
|
|
||||||
// middleware.MaxAccessLogLineBytes. Capping here would defeat the
|
|
||||||
// target: emitting the payload IS the delivery. It costs nothing by
|
|
||||||
// default — an authenticated operator has to create a target of this
|
|
||||||
// type on a specific webhook before a single line is written — and the
|
|
||||||
// bytes it writes are bounded per event by maxWebhookBodySize (1 MB).
|
|
||||||
// An operator who adds one is choosing to spend log volume on the
|
|
||||||
// payloads that webhook receives.
|
|
||||||
type logTarget struct {
|
type logTarget struct {
|
||||||
eng *Engine
|
eng *Engine
|
||||||
}
|
}
|
||||||
@@ -35,8 +23,6 @@ func (t *logTarget) Deliver(
|
|||||||
_ *Task,
|
_ *Task,
|
||||||
_ Scheduler,
|
_ Scheduler,
|
||||||
) {
|
) {
|
||||||
start := time.Now()
|
|
||||||
|
|
||||||
t.eng.log.Info(
|
t.eng.log.Info(
|
||||||
"webhook event delivered to log target",
|
"webhook event delivered to log target",
|
||||||
"delivery_id", d.ID,
|
"delivery_id", d.ID,
|
||||||
@@ -51,17 +37,11 @@ func (t *logTarget) Deliver(
|
|||||||
"body", d.Event.Body,
|
"body", d.Event.Body,
|
||||||
)
|
)
|
||||||
|
|
||||||
elapsed := time.Since(start)
|
|
||||||
|
|
||||||
t.eng.observeAttempt(d.Target.Type, elapsed)
|
|
||||||
|
|
||||||
t.eng.recordResult(
|
t.eng.recordResult(
|
||||||
webhookDB, d, 1, true, 0, "", "",
|
webhookDB, d, 1, true, 0, "", "", 0,
|
||||||
elapsed.Milliseconds(),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
t.eng.updateDeliveryStatus(
|
t.eng.updateDeliveryStatus(
|
||||||
webhookDB, d, d.Target.Type,
|
webhookDB, d, database.DeliveryStatusDelivered,
|
||||||
database.DeliveryStatusDelivered,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,272 +0,0 @@
|
|||||||
package delivery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/url"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// RedactionMarker stands in for a target credential found in
|
|
||||||
// text the target's remote peer chose.
|
|
||||||
const RedactionMarker = "(redacted)"
|
|
||||||
|
|
||||||
// Redactor removes one target's own credential material from
|
|
||||||
// text that target's remote peer chose: a delivery response
|
|
||||||
// body, or a delivery error stored before the delivery path
|
|
||||||
// learned to mask the URLs it embeds.
|
|
||||||
//
|
|
||||||
// It removes byte-identical echoes of strings taken from the
|
|
||||||
// target's stored configuration, and nothing else. Anything
|
|
||||||
// the remote re-encodes survives: JSON "\/" escaping (what
|
|
||||||
// PHP's json_encode emits by default), percent-encoding, HTML
|
|
||||||
// entities, and an echo of only part of a path. It cannot
|
|
||||||
// remove a secret the remote invented.
|
|
||||||
//
|
|
||||||
// The zero Redactor removes nothing, which is what a caller
|
|
||||||
// holding no target for a delivery gets.
|
|
||||||
type Redactor struct {
|
|
||||||
secrets []string
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewRedactor builds the redactor for one target.
|
|
||||||
func NewRedactor(t *database.Target) Redactor {
|
|
||||||
// Drop empty strings here rather than at the site that
|
|
||||||
// produced one. strings.ReplaceAll with an empty old string
|
|
||||||
// inserts the marker at every byte boundary, so a single
|
|
||||||
// empty secret destroys every body and error the target
|
|
||||||
// renders; filtering at the collection point means no field
|
|
||||||
// added to targetSecrets later can reintroduce that.
|
|
||||||
// url.Parse("https://@example.com/in") is the known
|
|
||||||
// producer: a non-nil User whose String is "".
|
|
||||||
secrets := slices.DeleteFunc(
|
|
||||||
targetSecrets(t),
|
|
||||||
func(s string) bool { return s == "" },
|
|
||||||
)
|
|
||||||
|
|
||||||
// Longest first, so replacing a secret that is contained
|
|
||||||
// in a longer one cannot leave a fragment of the longer
|
|
||||||
// one behind. Configured headers arrive in map order, so
|
|
||||||
// the sort is also what makes the result deterministic.
|
|
||||||
slices.SortFunc(secrets, func(a, b string) int {
|
|
||||||
if d := len(b) - len(a); d != 0 {
|
|
||||||
return d
|
|
||||||
}
|
|
||||||
|
|
||||||
return strings.Compare(a, b)
|
|
||||||
})
|
|
||||||
|
|
||||||
return Redactor{secrets: secrets}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Redact replaces every occurrence of the target's credential
|
|
||||||
// material in s.
|
|
||||||
func (r Redactor) Redact(s string) string {
|
|
||||||
if s == "" {
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, secret := range r.secrets {
|
|
||||||
s = strings.ReplaceAll(s, secret, RedactionMarker)
|
|
||||||
}
|
|
||||||
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
// RedactCut redacts s, which its caller has already cut to a
|
|
||||||
// byte budget, and additionally drops any tail of s that is a
|
|
||||||
// proper prefix of a secret.
|
|
||||||
//
|
|
||||||
// The cut lands wherever the remote's padding puts it, so the
|
|
||||||
// remote chooses where inside the credential it falls. The
|
|
||||||
// severed prefix left behind equals no secret, so plain
|
|
||||||
// Redact would render it verbatim.
|
|
||||||
func (r Redactor) RedactCut(s string) string {
|
|
||||||
s = r.Redact(s)
|
|
||||||
|
|
||||||
if n := r.secretPrefixSuffix(s); n > 0 {
|
|
||||||
return s[:len(s)-n] + RedactionMarker
|
|
||||||
}
|
|
||||||
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
|
|
||||||
// secretPrefixSuffix returns the length of the longest suffix
|
|
||||||
// of s that is a proper prefix of one of the secrets, or 0
|
|
||||||
// when there is none.
|
|
||||||
func (r Redactor) secretPrefixSuffix(s string) int {
|
|
||||||
longest := 0
|
|
||||||
|
|
||||||
for _, secret := range r.secrets {
|
|
||||||
// Proper prefixes only: a whole secret at the tail was
|
|
||||||
// already replaced by Redact.
|
|
||||||
n := min(len(secret)-1, len(s))
|
|
||||||
for ; n > longest; n-- {
|
|
||||||
if strings.HasSuffix(s, secret[:n]) {
|
|
||||||
longest = n
|
|
||||||
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return longest
|
|
||||||
}
|
|
||||||
|
|
||||||
// targetSecrets returns the credential-bearing strings a
|
|
||||||
// target's configuration carries.
|
|
||||||
//
|
|
||||||
// The destination URL contributes. Its path, query and
|
|
||||||
// userinfo are the credential for both target types that have
|
|
||||||
// one — an incoming-webhook URL is a bearer token, which is
|
|
||||||
// why MaskURL elides exactly those parts — and they are the
|
|
||||||
// material this service actually sends, so a remote that
|
|
||||||
// echoes the request back echoes them.
|
|
||||||
//
|
|
||||||
// Configured request headers contribute their values, but
|
|
||||||
// only for the credential-shaped names isCredentialHeaderName
|
|
||||||
// picks out. That is the same class-based rule applied to
|
|
||||||
// URLs: an echoed Accept or User-Agent still renders, an
|
|
||||||
// echoed Authorization does not.
|
|
||||||
func targetSecrets(t *database.Target) []string {
|
|
||||||
if t == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
switch t.Type {
|
|
||||||
case database.TargetTypeSlack:
|
|
||||||
cfg, err := parseSlackConfig(t.Config)
|
|
||||||
if err != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return urlSecrets(cfg.WebhookURL)
|
|
||||||
case database.TargetTypeHTTP:
|
|
||||||
cfg, err := parseHTTPConfig(t.Config)
|
|
||||||
if err != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return append(
|
|
||||||
urlSecrets(cfg.URL),
|
|
||||||
headerSecrets(cfg.Headers)...,
|
|
||||||
)
|
|
||||||
case database.TargetTypeDatabase, database.TargetTypeLog:
|
|
||||||
// Neither has a destination URL, so neither has
|
|
||||||
// anything to redact.
|
|
||||||
return nil
|
|
||||||
default:
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// urlSecrets returns the substrings of a destination URL that
|
|
||||||
// must not survive into a rendered page: the whole URL, the
|
|
||||||
// parts of it MaskURL elides, and any userinfo.
|
|
||||||
//
|
|
||||||
// No length floor is applied to the path, and none to the
|
|
||||||
// userinfo. A short path or a four-byte username is treated as
|
|
||||||
// a credential exactly like a long one, because the field takes
|
|
||||||
// an arbitrary URL and no part of it can be assumed non-secret —
|
|
||||||
// the same rule MaskURL applies. headerSecrets does carry a
|
|
||||||
// floor, and the difference is deliberate: a header is picked
|
|
||||||
// out by a name-shaped guess and its value may be ordinary
|
|
||||||
// text, whereas a URL's path and userinfo are credential
|
|
||||||
// material by position.
|
|
||||||
func urlSecrets(raw string) []string {
|
|
||||||
raw = strings.TrimSpace(raw)
|
|
||||||
if raw == "" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
secrets := []string{raw}
|
|
||||||
|
|
||||||
parsed, err := url.Parse(raw)
|
|
||||||
if err != nil {
|
|
||||||
return secrets
|
|
||||||
}
|
|
||||||
|
|
||||||
if parsed.Path != "" && parsed.Path != "/" {
|
|
||||||
requestURI := parsed.RequestURI()
|
|
||||||
secrets = append(secrets, requestURI)
|
|
||||||
|
|
||||||
if escaped := parsed.EscapedPath(); escaped != requestURI {
|
|
||||||
secrets = append(secrets, escaped)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if parsed.User != nil {
|
|
||||||
secrets = append(secrets, parsed.User.String())
|
|
||||||
|
|
||||||
if pw, ok := parsed.User.Password(); ok && pw != "" {
|
|
||||||
secrets = append(secrets, pw)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return secrets
|
|
||||||
}
|
|
||||||
|
|
||||||
// minHeaderSecretBytes is the shortest header value treated as
|
|
||||||
// a credential. Unlike a URL path, a header value can be a
|
|
||||||
// couple of bytes long, and redacting those would scatter the
|
|
||||||
// marker through ordinary response text for no gain.
|
|
||||||
const minHeaderSecretBytes = 4
|
|
||||||
|
|
||||||
// headerSecrets returns the values of the configured headers
|
|
||||||
// whose names are credential-shaped.
|
|
||||||
func headerSecrets(headers map[string]string) []string {
|
|
||||||
var secrets []string
|
|
||||||
|
|
||||||
for name, value := range headers {
|
|
||||||
value = strings.TrimSpace(value)
|
|
||||||
if len(value) < minHeaderSecretBytes {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if isCredentialHeaderName(name) {
|
|
||||||
secrets = append(secrets, value)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return secrets
|
|
||||||
}
|
|
||||||
|
|
||||||
// isCredentialHeaderName classifies a header by its name. The
|
|
||||||
// value is never inspected, so the rule is the same
|
|
||||||
// class-based one MaskURL applies to a destination URL.
|
|
||||||
//
|
|
||||||
// The fragments are short on purpose, and match anywhere in
|
|
||||||
// the name, so abbreviations an operator might use are covered
|
|
||||||
// too: X-Sig, X-Pass, X-HMAC. That over-matches — a header
|
|
||||||
// named X-Design contains "sig" — and over-matching is the
|
|
||||||
// safe direction here: the cost is a marker where an echoed
|
|
||||||
// header value would have rendered.
|
|
||||||
func isCredentialHeaderName(name string) bool {
|
|
||||||
name = strings.ToLower(strings.TrimSpace(name))
|
|
||||||
|
|
||||||
// Names that carry a credential by definition.
|
|
||||||
switch name {
|
|
||||||
case "authorization", "proxy-authorization", "cookie":
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// What operators call their own credential headers:
|
|
||||||
// X-Api-Key, X-Hub-Signature, X-Auth-Token.
|
|
||||||
for _, fragment := range []string{
|
|
||||||
"auth",
|
|
||||||
"credential",
|
|
||||||
"hmac",
|
|
||||||
"key",
|
|
||||||
"pass",
|
|
||||||
"secret",
|
|
||||||
"sig",
|
|
||||||
"token",
|
|
||||||
} {
|
|
||||||
if strings.Contains(name, fragment) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
@@ -1,284 +0,0 @@
|
|||||||
package delivery_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/url"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The secret path segments of a Slack incoming webhook URL.
|
|
||||||
const (
|
|
||||||
redactSecretPath = "/services/T11111111/B11111111/" +
|
|
||||||
"YYYYYYYYYYYYYYYYYYYYYYYY"
|
|
||||||
redactWebhookURL = "https://hooks.slack.com" +
|
|
||||||
redactSecretPath
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestRedactor_RemovesSlackWebhookURL(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
r := delivery.NewRedactor(&database.Target{
|
|
||||||
Type: database.TargetTypeSlack,
|
|
||||||
Config: `{"webhookUrl":"` + redactWebhookURL + `"}`,
|
|
||||||
})
|
|
||||||
|
|
||||||
got := r.Redact("no_service for " + redactWebhookURL)
|
|
||||||
|
|
||||||
assert.NotContains(t, got, redactSecretPath)
|
|
||||||
assert.NotContains(t, got, "T11111111")
|
|
||||||
|
|
||||||
// One marker, not a marker with the host left in front of
|
|
||||||
// it: the whole URL is replaced before the path it
|
|
||||||
// contains, which is what sorting the secrets longest
|
|
||||||
// first buys.
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
"no_service for "+delivery.RedactionMarker,
|
|
||||||
got,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRedactor_RemovesSecretSeveredByACut covers the input the
|
|
||||||
// redactor exists for: text cut to a byte budget with the
|
|
||||||
// credential straddling the cut. The remote chooses the
|
|
||||||
// padding, so it chooses where the cut lands inside the
|
|
||||||
// credential, and the severed prefix that remains equals no
|
|
||||||
// secret.
|
|
||||||
func TestRedactor_RemovesSecretSeveredByACut(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
r := delivery.NewRedactor(&database.Target{
|
|
||||||
Type: database.TargetTypeSlack,
|
|
||||||
Config: `{"webhookUrl":"` + redactWebhookURL + `"}`,
|
|
||||||
})
|
|
||||||
|
|
||||||
// Every cut position inside the credential, not just a
|
|
||||||
// convenient one.
|
|
||||||
for n := 1; n < len(redactWebhookURL); n++ {
|
|
||||||
severed := redactWebhookURL[:n]
|
|
||||||
cut := "padding " + severed
|
|
||||||
|
|
||||||
got := r.RedactCut(cut)
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
"padding "+delivery.RedactionMarker,
|
|
||||||
got,
|
|
||||||
"cut after %d bytes of the credential", n,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRedactor_RedactsCredentialShapedHeaderValues pins the
|
|
||||||
// class-based header rule: a header whose name says credential
|
|
||||||
// has its value redacted, and a routine header does not, so
|
|
||||||
// ordinary response content survives.
|
|
||||||
func TestRedactor_RedactsCredentialShapedHeaderValues(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
r := delivery.NewRedactor(&database.Target{
|
|
||||||
Type: database.TargetTypeHTTP,
|
|
||||||
Config: `{"url":"https://example.com/in",` +
|
|
||||||
`"headers":{` +
|
|
||||||
`"Authorization":"Bearer AAAAAAAAAAAA",` +
|
|
||||||
`"Cookie":"session=BBBBBBBBBBBB",` +
|
|
||||||
`"X-Api-Key":"CCCCCCCCCCCC",` +
|
|
||||||
`"X-Hub-Signature":"sha256=DDDDDDDDDDDD",` +
|
|
||||||
`"X-Sig":"EEEEEEEEEEEE",` +
|
|
||||||
`"X-Pass":"FFFFFFFFFFFF",` +
|
|
||||||
`"X-HMAC":"GGGGGGGGGGGG",` +
|
|
||||||
`"X-Credential":"HHHHHHHHHHHH",` +
|
|
||||||
`"Accept":"application/json",` +
|
|
||||||
`"User-Agent":"webhooker/1.0"}}`,
|
|
||||||
})
|
|
||||||
|
|
||||||
for _, secret := range []string{
|
|
||||||
"Bearer AAAAAAAAAAAA",
|
|
||||||
"session=BBBBBBBBBBBB",
|
|
||||||
"CCCCCCCCCCCC",
|
|
||||||
"sha256=DDDDDDDDDDDD",
|
|
||||||
// Abbreviated names an operator might use.
|
|
||||||
"EEEEEEEEEEEE",
|
|
||||||
"FFFFFFFFFFFF",
|
|
||||||
"GGGGGGGGGGGG",
|
|
||||||
"HHHHHHHHHHHH",
|
|
||||||
} {
|
|
||||||
got := r.Redact("echo: " + secret)
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
"echo: "+delivery.RedactionMarker,
|
|
||||||
got,
|
|
||||||
secret,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
const routine = "Accept: application/json, " +
|
|
||||||
"User-Agent: webhooker/1.0"
|
|
||||||
|
|
||||||
assert.Equal(t, routine, r.Redact(routine))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRedactor_IgnoresVeryShortHeaderValues pins the floor
|
|
||||||
// under a header value. Redacting a two-byte value would put
|
|
||||||
// the marker through every response that happens to contain
|
|
||||||
// those bytes.
|
|
||||||
func TestRedactor_IgnoresVeryShortHeaderValues(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
r := delivery.NewRedactor(&database.Target{
|
|
||||||
Type: database.TargetTypeHTTP,
|
|
||||||
Config: `{"url":"https://example.com/in",` +
|
|
||||||
`"headers":{"X-Api-Key":"ab"}}`,
|
|
||||||
})
|
|
||||||
|
|
||||||
const response = "rabbit"
|
|
||||||
|
|
||||||
assert.Equal(t, response, r.Redact(response))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRedactor_RemovesBarePath covers a remote that echoes
|
|
||||||
// only the request path rather than the whole URL. The path
|
|
||||||
// segments are the credential on their own.
|
|
||||||
func TestRedactor_RemovesBarePath(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
r := delivery.NewRedactor(&database.Target{
|
|
||||||
Type: database.TargetTypeSlack,
|
|
||||||
Config: `{"webhookUrl":"` + redactWebhookURL + `"}`,
|
|
||||||
})
|
|
||||||
|
|
||||||
got := r.Redact("POST " + redactSecretPath + " 404")
|
|
||||||
|
|
||||||
assert.NotContains(t, got, redactSecretPath)
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
"POST "+delivery.RedactionMarker+" 404",
|
|
||||||
got,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRedactor_RemovesHTTPURLQueryAndUserinfo covers the HTTP
|
|
||||||
// target, whose destination is an arbitrary URL: the query
|
|
||||||
// string and the userinfo carry credentials as readily as the
|
|
||||||
// path does.
|
|
||||||
func TestRedactor_RemovesHTTPURLQueryAndUserinfo(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Assembled rather than written out, so the literal is
|
|
||||||
// not itself a credential-shaped string.
|
|
||||||
dest := url.URL{
|
|
||||||
Scheme: "https",
|
|
||||||
User: url.UserPassword("user", "hunter2"),
|
|
||||||
Host: "example.com",
|
|
||||||
Path: "/in",
|
|
||||||
RawQuery: "token=s3cr3t",
|
|
||||||
}
|
|
||||||
raw := dest.String()
|
|
||||||
|
|
||||||
r := delivery.NewRedactor(&database.Target{
|
|
||||||
Type: database.TargetTypeHTTP,
|
|
||||||
Config: `{"url":"` + raw + `"}`,
|
|
||||||
})
|
|
||||||
|
|
||||||
for _, echoed := range []string{
|
|
||||||
raw,
|
|
||||||
"/in?token=s3cr3t",
|
|
||||||
"hunter2",
|
|
||||||
} {
|
|
||||||
got := r.Redact("rejected: " + echoed)
|
|
||||||
|
|
||||||
assert.NotContains(t, got, "s3cr3t", echoed)
|
|
||||||
assert.NotContains(t, got, "hunter2", echoed)
|
|
||||||
assert.Contains(
|
|
||||||
t, got, delivery.RedactionMarker, echoed,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRedactor_LeavesUnrelatedTextAlone pins that the
|
|
||||||
// redactor matches literally: it does not guess at what a
|
|
||||||
// secret looks like, so ordinary response content survives.
|
|
||||||
func TestRedactor_LeavesUnrelatedTextAlone(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const response = "ok=false error=channel_not_found"
|
|
||||||
|
|
||||||
r := delivery.NewRedactor(&database.Target{
|
|
||||||
Type: database.TargetTypeSlack,
|
|
||||||
Config: `{"webhookUrl":"` + redactWebhookURL + `"}`,
|
|
||||||
})
|
|
||||||
|
|
||||||
assert.Equal(t, response, r.Redact(response))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRedactor_EmptyUserinfoDoesNotShredTheBody covers a
|
|
||||||
// destination URL written with a bare "@" and no userinfo:
|
|
||||||
// url.Parse returns a non-nil User whose String is empty. An
|
|
||||||
// empty secret in the list would make strings.ReplaceAll
|
|
||||||
// insert the marker at every byte boundary, destroying every
|
|
||||||
// body and error string the target renders.
|
|
||||||
func TestRedactor_EmptyUserinfoDoesNotShredTheBody(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const dest = "https://@example.com/in"
|
|
||||||
|
|
||||||
// The premise: this URL really does parse to a non-nil
|
|
||||||
// User contributing an empty string.
|
|
||||||
parsed, err := url.Parse(dest)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, parsed.User)
|
|
||||||
require.Empty(t, parsed.User.String())
|
|
||||||
|
|
||||||
r := delivery.NewRedactor(&database.Target{
|
|
||||||
Type: database.TargetTypeHTTP,
|
|
||||||
Config: `{"url":"` + dest + `"}`,
|
|
||||||
})
|
|
||||||
|
|
||||||
const body = "ok=false error=channel_not_found"
|
|
||||||
|
|
||||||
assert.Equal(t, body, r.Redact(body))
|
|
||||||
assert.Equal(t, body, r.RedactCut(body))
|
|
||||||
|
|
||||||
// The real credential material still goes, so filtering the
|
|
||||||
// empty string out did not disarm the redactor.
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
"POST "+delivery.RedactionMarker,
|
|
||||||
r.Redact("POST "+dest),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRedactor_ZeroValueAndConfiglessTargets pins that a
|
|
||||||
// caller with no target, an unparseable config, or a target
|
|
||||||
// type with no destination URL gets a redactor that changes
|
|
||||||
// nothing rather than one that panics.
|
|
||||||
func TestRedactor_ZeroValueAndConfiglessTargets(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const text = "some response body"
|
|
||||||
|
|
||||||
var zero delivery.Redactor
|
|
||||||
|
|
||||||
assert.Equal(t, text, zero.Redact(text))
|
|
||||||
assert.Equal(t, text, delivery.NewRedactor(nil).Redact(text))
|
|
||||||
|
|
||||||
for _, tgt := range []database.Target{
|
|
||||||
{Type: database.TargetTypeLog},
|
|
||||||
{Type: database.TargetTypeDatabase},
|
|
||||||
{Type: database.TargetTypeSlack, Config: "not json"},
|
|
||||||
{Type: database.TargetTypeHTTP, Config: ""},
|
|
||||||
} {
|
|
||||||
assert.Equal(
|
|
||||||
t, text,
|
|
||||||
delivery.NewRedactor(&tgt).Redact(text),
|
|
||||||
tgt.Type,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -101,8 +101,7 @@ func (t *slackTarget) failConfig(
|
|||||||
)
|
)
|
||||||
|
|
||||||
t.eng.updateDeliveryStatus(
|
t.eng.updateDeliveryStatus(
|
||||||
webhookDB, d, d.Target.Type,
|
webhookDB, d, database.DeliveryStatusFailed,
|
||||||
database.DeliveryStatusFailed,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -126,7 +125,7 @@ func (t *slackTarget) attempt(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return attemptResult{
|
return attemptResult{
|
||||||
success: false,
|
success: false,
|
||||||
errMsg: maskURLError(err).Error(),
|
errMsg: err.Error(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,61 +0,0 @@
|
|||||||
package delivery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"net/url"
|
|
||||||
)
|
|
||||||
|
|
||||||
// urlPathElision stands in for a URL's elided path.
|
|
||||||
const urlPathElision = "/..."
|
|
||||||
|
|
||||||
// MaskURL renders a URL as scheme plus host with everything
|
|
||||||
// that can carry a secret removed. A delivery target URL is
|
|
||||||
// itself a credential — a Slack incoming webhook URL is a
|
|
||||||
// bearer token — so the path, query and userinfo are never
|
|
||||||
// reproduced, in a page, a log line or a stored error. A URL
|
|
||||||
// that does not parse into a scheme and host yields the
|
|
||||||
// neutral placeholder, never the raw string.
|
|
||||||
func MaskURL(raw string) string {
|
|
||||||
parsed, err := url.Parse(raw)
|
|
||||||
if err != nil || parsed.Scheme == "" ||
|
|
||||||
parsed.Host == "" {
|
|
||||||
return configUnavailable
|
|
||||||
}
|
|
||||||
|
|
||||||
masked := parsed.Scheme + "://" + parsed.Host
|
|
||||||
|
|
||||||
if parsed.Path != "" && parsed.Path != "/" {
|
|
||||||
masked += urlPathElision
|
|
||||||
}
|
|
||||||
|
|
||||||
return masked
|
|
||||||
}
|
|
||||||
|
|
||||||
// maskURLError strips the credential from an error raised
|
|
||||||
// against a request URL. The net/http and net/url packages
|
|
||||||
// embed the full request URL in every *url.Error they return,
|
|
||||||
// so an unmodified transport error persisted into
|
|
||||||
// DeliveryResult.Error writes the credential to disk.
|
|
||||||
//
|
|
||||||
// The masked error keeps the operation and the wrapped cause,
|
|
||||||
// so a DNS failure still reads differently from a refused
|
|
||||||
// connection, a TLS handshake failure or a timeout, and Is,
|
|
||||||
// As, Timeout and Temporary keep working on it. Only the
|
|
||||||
// path, query and userinfo of the URL are dropped. Errors
|
|
||||||
// that carry no URL are returned unchanged.
|
|
||||||
//
|
|
||||||
// Call it where the error is raised, before any wrapping: it
|
|
||||||
// replaces the *url.Error itself, so any context wrapped
|
|
||||||
// around it first would be discarded.
|
|
||||||
func maskURLError(err error) error {
|
|
||||||
var urlErr *url.Error
|
|
||||||
if !errors.As(err, &urlErr) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return &url.Error{
|
|
||||||
Op: urlErr.Op,
|
|
||||||
URL: MaskURL(urlErr.URL),
|
|
||||||
Err: urlErr.Err,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,196 +0,0 @@
|
|||||||
package delivery_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The path of a Slack incoming webhook URL is the credential:
|
|
||||||
// whoever holds these segments can post to the channel
|
|
||||||
// forever. None of them may reach a stored delivery error,
|
|
||||||
// which lives on disk in the per-webhook database and is
|
|
||||||
// serialized by the JSON tag on DeliveryResult.Error.
|
|
||||||
const (
|
|
||||||
maskSecretPath = "/services/T00000000/B00000000/" +
|
|
||||||
"XXXXXXXXXXXXXXXXXXXXXXXX"
|
|
||||||
)
|
|
||||||
|
|
||||||
// assertNoCredential fails if the whole path or any single
|
|
||||||
// segment of it survived into the message, so a partial leak
|
|
||||||
// fails the test too.
|
|
||||||
func assertNoCredential(t *testing.T, msg string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
segments := []string{
|
|
||||||
maskSecretPath,
|
|
||||||
"services",
|
|
||||||
"T00000000",
|
|
||||||
"B00000000",
|
|
||||||
"XXXXXXXXXXXXXXXXXXXXXXXX",
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, segment := range segments {
|
|
||||||
assert.NotContains(t, msg, segment)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// storedDeliveryError returns the error string persisted for a
|
|
||||||
// delivery, which is what an operator and any future API read.
|
|
||||||
func storedDeliveryError(
|
|
||||||
t *testing.T, db *gorm.DB, deliveryID string,
|
|
||||||
) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var result database.DeliveryResult
|
|
||||||
|
|
||||||
require.NoError(t, db.Where(
|
|
||||||
"delivery_id = ?", deliveryID,
|
|
||||||
).First(&result).Error)
|
|
||||||
|
|
||||||
return result.Error
|
|
||||||
}
|
|
||||||
|
|
||||||
// deliverSlackTo runs a Slack delivery against webhookURL and
|
|
||||||
// returns the error string it persisted.
|
|
||||||
func deliverSlackTo(
|
|
||||||
t *testing.T, webhookURL string,
|
|
||||||
) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
db := testWebhookDB(t)
|
|
||||||
e := testEngine(t, 1)
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
|
|
||||||
slackCfg, err := json.Marshal(
|
|
||||||
delivery.SlackTargetConfig{
|
|
||||||
WebhookURL: webhookURL,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
event := seedEvent(t, db, `{"test":true}`)
|
|
||||||
|
|
||||||
dlv := seedDelivery(
|
|
||||||
t, db, event.ID, targetID,
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
|
|
||||||
d := buildSlackDelivery(
|
|
||||||
dlv, event, targetID,
|
|
||||||
"test-slack-mask", string(slackCfg),
|
|
||||||
)
|
|
||||||
|
|
||||||
e.ExportDeliverSlack(context.TODO(), db, d)
|
|
||||||
|
|
||||||
assertDeliveryStatus(t, db, dlv.ID,
|
|
||||||
database.DeliveryStatusFailed,
|
|
||||||
)
|
|
||||||
|
|
||||||
return storedDeliveryError(t, db, dlv.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDeliverSlack_TransportErrorMasksWebhookURL is the
|
|
||||||
// load-bearing regression test: a transport failure must not
|
|
||||||
// persist the webhook URL's credential into the database, and
|
|
||||||
// must still say what went wrong and where.
|
|
||||||
func TestDeliverSlack_TransportErrorMasksWebhookURL(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// A server closed before use gives a deterministic
|
|
||||||
// transport failure against a known host.
|
|
||||||
ts := httptest.NewServer(http.NewServeMux())
|
|
||||||
host := ts.URL
|
|
||||||
|
|
||||||
ts.Close()
|
|
||||||
|
|
||||||
errMsg := deliverSlackTo(t, host+maskSecretPath)
|
|
||||||
|
|
||||||
require.NotEmpty(t, errMsg)
|
|
||||||
assertNoCredential(t, errMsg)
|
|
||||||
|
|
||||||
// The diagnostic value survives: the operation, the host
|
|
||||||
// and the transport failure are all still reported, and
|
|
||||||
// only the path is elided.
|
|
||||||
assert.Contains(t, errMsg, "sending request")
|
|
||||||
assert.Contains(t, errMsg, "Post")
|
|
||||||
assert.Contains(t, errMsg, host+"/...")
|
|
||||||
assert.Contains(t, errMsg, "connection refused")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDeliverSlack_UnparsableURLMasksWebhookURL covers the
|
|
||||||
// other error path out of a Slack attempt: url.Parse also
|
|
||||||
// embeds the whole URL in the error it returns.
|
|
||||||
func TestDeliverSlack_UnparsableURLMasksWebhookURL(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
errMsg := deliverSlackTo(
|
|
||||||
t,
|
|
||||||
"https://hooks.slack.com"+maskSecretPath+"\n",
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NotEmpty(t, errMsg)
|
|
||||||
assertNoCredential(t, errMsg)
|
|
||||||
assert.Contains(t, errMsg, "invalid control character")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDoHTTPRequest_TransportErrorMasksURL proves the HTTP
|
|
||||||
// target's transport errors are masked too; its destination
|
|
||||||
// URL can carry a token in a query string.
|
|
||||||
func TestDoHTTPRequest_TransportErrorMasksURL(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ts := httptest.NewServer(http.NewServeMux())
|
|
||||||
host := ts.URL
|
|
||||||
|
|
||||||
ts.Close()
|
|
||||||
|
|
||||||
e := testEngine(t, 1)
|
|
||||||
|
|
||||||
cfg, err := e.ExportParseHTTPConfig(
|
|
||||||
newHTTPTargetConfig(host + maskSecretPath),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
statusCode, _, _, reqErr := e.ExportDoHTTPRequest(
|
|
||||||
context.TODO(), cfg,
|
|
||||||
&database.Event{Body: `{"test":true}`},
|
|
||||||
)
|
|
||||||
require.Error(t, reqErr)
|
|
||||||
assert.Zero(t, statusCode)
|
|
||||||
|
|
||||||
assertNoCredential(t, reqErr.Error())
|
|
||||||
assert.Contains(t, reqErr.Error(), host+"/...")
|
|
||||||
assert.Contains(
|
|
||||||
t, reqErr.Error(), "connection refused",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestValidateTargetURL_UnparsableURLIsMasked proves the SSRF
|
|
||||||
// validator's error does not carry the submitted URL, which
|
|
||||||
// the handler both logs and shows.
|
|
||||||
func TestValidateTargetURL_UnparsableURLIsMasked(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
err := delivery.NewTestGuard().ValidateTargetURL(
|
|
||||||
context.TODO(),
|
|
||||||
"https://hooks.slack.com"+maskSecretPath+"\n",
|
|
||||||
)
|
|
||||||
require.Error(t, err)
|
|
||||||
|
|
||||||
assertNoCredential(t, err.Error())
|
|
||||||
assert.Contains(t, err.Error(), "invalid URL")
|
|
||||||
}
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
package gormlog
|
|
||||||
|
|
||||||
import (
|
|
||||||
"log/slog"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ExportNewWithSlowThreshold builds a Logger whose slow-statement
|
|
||||||
// threshold is d rather than DefaultSlowThreshold, so a test can pin
|
|
||||||
// which arm of Trace it is exercising instead of racing the clock on a
|
|
||||||
// loaded machine. The threshold is set at construction, like every
|
|
||||||
// other field, so the type's concurrency guarantee still holds.
|
|
||||||
func ExportNewWithSlowThreshold(
|
|
||||||
log *slog.Logger, d time.Duration,
|
|
||||||
) *Logger {
|
|
||||||
return &Logger{log: log, slowThreshold: d}
|
|
||||||
}
|
|
||||||
@@ -1,229 +0,0 @@
|
|||||||
package gormlog_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"go.uber.org/fx"
|
|
||||||
"go.uber.org/fx/fxtest"
|
|
||||||
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/globals"
|
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
|
||||||
)
|
|
||||||
|
|
||||||
// argon2Prefix opens every encoded Argon2id hash this service
|
|
||||||
// produces. It is asserted on separately from the hash itself, so that
|
|
||||||
// a change to the password encoding cannot quietly turn the hash
|
|
||||||
// assertion into a comparison against a string the log never held.
|
|
||||||
const argon2Prefix = "$argon2id$"
|
|
||||||
|
|
||||||
// settingsInsert and usersInsert are the two statements a first boot
|
|
||||||
// runs that carry a secret. The sqlite dialector quotes identifiers
|
|
||||||
// with backticks.
|
|
||||||
const (
|
|
||||||
settingsInsert = "INSERT INTO `settings`"
|
|
||||||
usersInsert = "INSERT INTO `users`"
|
|
||||||
)
|
|
||||||
|
|
||||||
// captureStdoutToFile redirects os.Stdout into a file for the rest of
|
|
||||||
// the test and returns a function that reads back everything written
|
|
||||||
// to it.
|
|
||||||
//
|
|
||||||
// A file rather than a pipe: internal/logger writes synchronously to
|
|
||||||
// whatever os.Stdout is when it builds its handler, so once fx's start
|
|
||||||
// returns, every byte the boot produced is already in the file and no
|
|
||||||
// draining goroutine is needed to prove it. Redirecting the variable
|
|
||||||
// before the application is built is what puts the service logger —
|
|
||||||
// and therefore the GORM adapter, which writes through it — into the
|
|
||||||
// capture.
|
|
||||||
//
|
|
||||||
// The redirect also decides the handler: a regular file is not a
|
|
||||||
// character device, so internal/logger installs its JSON handler, the
|
|
||||||
// one it installs in production under a log collector.
|
|
||||||
func captureStdoutToFile(t *testing.T) func() string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "stdout.log")
|
|
||||||
|
|
||||||
//nolint:gosec // The path is this test's own t.TempDir().
|
|
||||||
f, err := os.Create(path)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
orig := os.Stdout
|
|
||||||
os.Stdout = f
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
os.Stdout = orig
|
|
||||||
_ = f.Close()
|
|
||||||
})
|
|
||||||
|
|
||||||
return func() string {
|
|
||||||
require.NoError(t, f.Sync())
|
|
||||||
|
|
||||||
//nolint:gosec // As above.
|
|
||||||
b, readErr := os.ReadFile(path)
|
|
||||||
require.NoError(t, readErr)
|
|
||||||
|
|
||||||
return string(b)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// firstBootSecrets are the two values a first boot generates and
|
|
||||||
// stores, read back out of the database.
|
|
||||||
type firstBootSecrets struct {
|
|
||||||
sessionKey string
|
|
||||||
passwordHash string
|
|
||||||
}
|
|
||||||
|
|
||||||
// readFirstBootSecrets reads those two secrets straight out of the
|
|
||||||
// SQLite file with database/sql rather than through GORM, so that
|
|
||||||
// reading them cannot itself add a line to the log under test.
|
|
||||||
func readFirstBootSecrets(
|
|
||||||
t *testing.T, dataDir string,
|
|
||||||
) firstBootSecrets {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
db, err := sql.Open("sqlite", filepath.Join(
|
|
||||||
dataDir, "webhooker.db",
|
|
||||||
))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, db.Close()) }()
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
var got firstBootSecrets
|
|
||||||
|
|
||||||
require.NoError(t, db.QueryRowContext(
|
|
||||||
ctx, `SELECT value FROM settings WHERE key = 'session_key'`,
|
|
||||||
).Scan(&got.sessionKey))
|
|
||||||
|
|
||||||
require.NoError(t, db.QueryRowContext(
|
|
||||||
ctx, `SELECT password FROM users WHERE username = 'admin'`,
|
|
||||||
).Scan(&got.passwordHash))
|
|
||||||
|
|
||||||
require.NotEmpty(t, got.sessionKey)
|
|
||||||
require.Contains(t, got.passwordHash, argon2Prefix)
|
|
||||||
|
|
||||||
return got
|
|
||||||
}
|
|
||||||
|
|
||||||
// bootAtDebug starts and stops the real application graph against
|
|
||||||
// dataDir with DEBUG=true, and returns everything it wrote to standard
|
|
||||||
// output.
|
|
||||||
//
|
|
||||||
// config.New reads DEBUG from the environment exactly as the binary
|
|
||||||
// does, internal/logger builds the handler it builds in production,
|
|
||||||
// database.New runs the migrations and creates the admin user, and
|
|
||||||
// session.New takes the session key. Those four are the whole of the
|
|
||||||
// path that writes either secret.
|
|
||||||
func bootAtDebug(t *testing.T, dataDir string) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
t.Setenv("DEBUG", "true")
|
|
||||||
t.Setenv("DATA_DIR", dataDir)
|
|
||||||
|
|
||||||
read := captureStdoutToFile(t)
|
|
||||||
|
|
||||||
var sess *session.Session
|
|
||||||
|
|
||||||
app := fxtest.New(
|
|
||||||
t,
|
|
||||||
fx.Provide(
|
|
||||||
globals.New,
|
|
||||||
logger.New,
|
|
||||||
config.New,
|
|
||||||
database.New,
|
|
||||||
session.New,
|
|
||||||
),
|
|
||||||
fx.Populate(&sess),
|
|
||||||
)
|
|
||||||
|
|
||||||
app.RequireStart()
|
|
||||||
app.RequireStop()
|
|
||||||
|
|
||||||
return read()
|
|
||||||
}
|
|
||||||
|
|
||||||
// requireFirstBootWasLogged is the non-vacuity half of the test below.
|
|
||||||
// Without it a build that logged no SQL at all, or that never reached
|
|
||||||
// DEBUG, would satisfy every absence assertion.
|
|
||||||
func requireFirstBootWasLogged(t *testing.T, out string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
require.Contains(
|
|
||||||
t, out, `"level":"DEBUG"`,
|
|
||||||
"DEBUG=true did not reach the logger",
|
|
||||||
)
|
|
||||||
require.Contains(
|
|
||||||
t, out, settingsInsert,
|
|
||||||
"the session key INSERT was not logged, so its absence "+
|
|
||||||
"proves nothing",
|
|
||||||
)
|
|
||||||
require.Contains(
|
|
||||||
t, out, usersInsert,
|
|
||||||
"the admin user INSERT was not logged, so its absence "+
|
|
||||||
"proves nothing",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFirstBootAtDebug_LogsNeitherSecret is the definition of done.
|
|
||||||
//
|
|
||||||
// A first boot is the only boot that writes either secret. The
|
|
||||||
// settings INSERT carries the base64 session encryption key, which is
|
|
||||||
// the whole of the session security model: anyone holding it can forge
|
|
||||||
// an authenticated session cookie. The users INSERT carries the admin
|
|
||||||
// account's Argon2id hash. Under interpolated statement logging both
|
|
||||||
// landed in the log an operator diagnosing a startup problem pastes
|
|
||||||
// into an issue.
|
|
||||||
//
|
|
||||||
// The secrets are read back out of the database file afterwards, so
|
|
||||||
// the assertions are made against the values this boot actually
|
|
||||||
// generated rather than against a pattern that might not match them.
|
|
||||||
//
|
|
||||||
// What this test does not cover: the initial admin password itself,
|
|
||||||
// which internal/database logs once in the clear, on purpose, because
|
|
||||||
// that line is the only place an operator ever sees it. That is a
|
|
||||||
// separate decision from the SQL log, and it is documented in the
|
|
||||||
// README rather than asserted here.
|
|
||||||
//
|
|
||||||
// Not parallel: it redirects os.Stdout and sets environment variables,
|
|
||||||
// both process-global.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // Deliberately sequential; see above.
|
|
||||||
func TestFirstBootAtDebug_LogsNeitherSecret(t *testing.T) {
|
|
||||||
dataDir := t.TempDir()
|
|
||||||
|
|
||||||
// An empty DATA_DIR is what makes this a first boot: with a
|
|
||||||
// database already in place neither INSERT runs.
|
|
||||||
entries, err := os.ReadDir(dataDir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Empty(t, entries, "DATA_DIR was not empty")
|
|
||||||
|
|
||||||
out := bootAtDebug(t, dataDir)
|
|
||||||
|
|
||||||
requireFirstBootWasLogged(t, out)
|
|
||||||
|
|
||||||
secrets := readFirstBootSecrets(t, dataDir)
|
|
||||||
|
|
||||||
assert.NotContains(
|
|
||||||
t, out, secrets.sessionKey,
|
|
||||||
"the session encryption key reached the debug log",
|
|
||||||
)
|
|
||||||
assert.NotContains(
|
|
||||||
t, out, secrets.passwordHash,
|
|
||||||
"the admin password hash reached the debug log",
|
|
||||||
)
|
|
||||||
assert.NotContains(
|
|
||||||
t, out, argon2Prefix,
|
|
||||||
"an encoded Argon2id hash reached the debug log",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -1,219 +0,0 @@
|
|||||||
// Package gormlog adapts GORM's logger onto the service's slog
|
|
||||||
// logger.
|
|
||||||
//
|
|
||||||
// GORM's own default logger is not usable here. It is built at package
|
|
||||||
// init with log.New(os.Stdout, ...) at LogLevel Warn with
|
|
||||||
// IgnoreRecordNotFoundError false, so it writes the fully interpolated
|
|
||||||
// SQL — parameters and all — for every statement that returns an
|
|
||||||
// error, including gorm.ErrRecordNotFound. Two of this service's
|
|
||||||
// lookups miss by design on unauthenticated routes: the entrypoint
|
|
||||||
// lookup on /webhook/{uuid}, whose path segment the client picks
|
|
||||||
// outright, and the user lookup behind the login form, whose username
|
|
||||||
// the client picks outright. Under the default logger each of those
|
|
||||||
// misses printed an unbounded, attacker-chosen string, at no level the
|
|
||||||
// operator can turn down, past every handler internal/logger installs.
|
|
||||||
//
|
|
||||||
// This adapter fixes all three properties at once: the lines get a
|
|
||||||
// level the operator controls, they are shaped by whichever handler
|
|
||||||
// internal/logger selected, and every value a client can influence is
|
|
||||||
// spent through logfield.Truncate.
|
|
||||||
//
|
|
||||||
// It also logs no bound value at all. See ParamsFilter: the statement
|
|
||||||
// is written with its placeholders intact, at every level, so the
|
|
||||||
// values a statement carries never reach the log in the first place.
|
|
||||||
package gormlog
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"log/slog"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"gorm.io/gorm"
|
|
||||||
gormlogger "gorm.io/gorm/logger"
|
|
||||||
"sneak.berlin/go/webhooker/internal/logfield"
|
|
||||||
)
|
|
||||||
|
|
||||||
// DefaultSlowThreshold is the duration at or above which a statement
|
|
||||||
// is logged as slow. It is GORM's own default, kept deliberately: slow
|
|
||||||
// SQL is the one thing GORM's logger reports that nothing else in this
|
|
||||||
// service does, so silencing the logger outright would have cost real
|
|
||||||
// observability to fix a log-volume defect.
|
|
||||||
const DefaultSlowThreshold = 200 * time.Millisecond
|
|
||||||
|
|
||||||
// Logger implements gormlogger.Interface on top of an *slog.Logger.
|
|
||||||
//
|
|
||||||
// It is safe for concurrent use: every field is set at construction
|
|
||||||
// and never written again.
|
|
||||||
type Logger struct {
|
|
||||||
log *slog.Logger
|
|
||||||
slowThreshold time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// Interface compliance is asserted here rather than discovered at the
|
|
||||||
// gorm.Open call sites. gorm.ParamsFilter is the optional half: GORM
|
|
||||||
// type-asserts for it and silently keeps interpolating if it is
|
|
||||||
// missing, so losing it would cost no build error and no test that
|
|
||||||
// does not look at the emitted SQL.
|
|
||||||
var (
|
|
||||||
_ gormlogger.Interface = (*Logger)(nil)
|
|
||||||
_ gorm.ParamsFilter = (*Logger)(nil)
|
|
||||||
)
|
|
||||||
|
|
||||||
// New returns a GORM logger that writes through log.
|
|
||||||
func New(log *slog.Logger) *Logger {
|
|
||||||
return &Logger{
|
|
||||||
log: log,
|
|
||||||
slowThreshold: DefaultSlowThreshold,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// LogMode returns the logger unchanged.
|
|
||||||
//
|
|
||||||
// GORM's LogLevel is deliberately not honoured. Level is the operator's
|
|
||||||
// decision and it is expressed once, through LOG_LEVEL and the
|
|
||||||
// slog.LevelVar internal/logger holds; a second level knob inside the
|
|
||||||
// database layer could only disagree with it. The mapping from GORM's
|
|
||||||
// four categories onto slog levels is fixed in Trace below.
|
|
||||||
//
|
|
||||||
//nolint:ireturn // The interface return is GORM's signature, not a choice.
|
|
||||||
func (l *Logger) LogMode(gormlogger.LogLevel) gormlogger.Interface {
|
|
||||||
return l
|
|
||||||
}
|
|
||||||
|
|
||||||
// ParamsFilter drops every bound value before GORM renders a statement
|
|
||||||
// for the log, so what is logged is the statement's shape — its
|
|
||||||
// placeholders — and never the values in it.
|
|
||||||
//
|
|
||||||
// GORM builds the string it hands to Trace by calling
|
|
||||||
// Dialector.Explain(sql, vars...), which substitutes each value into
|
|
||||||
// the statement. Discarding vars here leaves the '?' placeholders in
|
|
||||||
// place, because ExplainSQL only substitutes while it still has a
|
|
||||||
// value for the next one. That happens before Trace is reached, so it
|
|
||||||
// holds on all three of its arms: the failed statement, the slow one,
|
|
||||||
// and the routine one an operator sees at DEBUG.
|
|
||||||
//
|
|
||||||
// This is the whole of the fix, and it is deliberately unconditional
|
|
||||||
// rather than a list of tables to redact. At first boot the two
|
|
||||||
// statements that carry a secret are the INSERT into settings holding
|
|
||||||
// the base64 session key — which is the entire session security model,
|
|
||||||
// since anyone with it can mint a valid cookie — and the INSERT into
|
|
||||||
// users holding the Argon2id hash. A denylist would have had to be
|
|
||||||
// extended by hand for every table added afterwards, and the cost of
|
|
||||||
// missing one is a credential in a log that gets pasted into issues.
|
|
||||||
//
|
|
||||||
// What is given up is the ability to read a value out of the log. The
|
|
||||||
// statement, the table, the error and the row count are all still
|
|
||||||
// there, which is what identifies a failing statement; reproducing it
|
|
||||||
// needs the values, and those an operator now gets from the database
|
|
||||||
// rather than from the log.
|
|
||||||
//
|
|
||||||
// One GORM path does not consult this: (*gorm.DB).Scan records the
|
|
||||||
// statement through gorm's own traceRecorder, which does not implement
|
|
||||||
// this interface. No production code path calls it; its one caller is
|
|
||||||
// internal/database/database_test.go:91, whose SELECT 1 binds nothing.
|
|
||||||
// scan_guard_test.go fails if a non-test file calls it.
|
|
||||||
// (*gorm.DB).Pluck, Row and Raw all run through the normal callback
|
|
||||||
// processor and are filtered.
|
|
||||||
func (l *Logger) ParamsFilter(
|
|
||||||
_ context.Context, sql string, _ ...any,
|
|
||||||
) (string, []any) {
|
|
||||||
return sql, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Info logs one of GORM's own informational messages.
|
|
||||||
func (l *Logger) Info(
|
|
||||||
ctx context.Context, msg string, data ...any,
|
|
||||||
) {
|
|
||||||
l.log.InfoContext(ctx, "gorm", "message", format(msg, data...))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Warn logs one of GORM's own warnings.
|
|
||||||
func (l *Logger) Warn(
|
|
||||||
ctx context.Context, msg string, data ...any,
|
|
||||||
) {
|
|
||||||
l.log.WarnContext(ctx, "gorm", "message", format(msg, data...))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Error logs one of GORM's own errors.
|
|
||||||
func (l *Logger) Error(
|
|
||||||
ctx context.Context, msg string, data ...any,
|
|
||||||
) {
|
|
||||||
l.log.ErrorContext(ctx, "gorm", "message", format(msg, data...))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Trace reports the outcome of a single statement. GORM calls it for
|
|
||||||
// every statement it runs, so the cheap paths stay cheap: fc() renders
|
|
||||||
// the statement — with placeholders, per ParamsFilter — and is called
|
|
||||||
// only on a branch that will actually emit.
|
|
||||||
//
|
|
||||||
// The arms are ordered exactly as GORM's own Trace orders them —
|
|
||||||
// non-record-not-found error, then slow, then the routine case — so
|
|
||||||
// that a statement which both misses and runs slow is still reported
|
|
||||||
// as slow. A miss is the likeliest statement to be slow, since it is
|
|
||||||
// the one that scans without finding a row, and ordering the drop
|
|
||||||
// ahead of the slow arm would have made this adapter less observant
|
|
||||||
// than the IgnoreRecordNotFoundError option it was chosen over.
|
|
||||||
func (l *Logger) Trace(
|
|
||||||
ctx context.Context,
|
|
||||||
begin time.Time,
|
|
||||||
fc func() (string, int64),
|
|
||||||
err error,
|
|
||||||
) {
|
|
||||||
elapsed := time.Since(begin)
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case err != nil && !errors.Is(err, gormlogger.ErrRecordNotFound):
|
|
||||||
sql, rows := fc()
|
|
||||||
l.log.ErrorContext(ctx, "sql statement failed",
|
|
||||||
"error", logfield.Truncate(err.Error(), logfield.MaxBytes),
|
|
||||||
"sql", logfield.Truncate(sql, logfield.MaxBytes),
|
|
||||||
"rows", rows,
|
|
||||||
"elapsed_ms", elapsed.Milliseconds(),
|
|
||||||
)
|
|
||||||
|
|
||||||
case l.slowThreshold > 0 && elapsed >= l.slowThreshold:
|
|
||||||
sql, rows := fc()
|
|
||||||
l.log.WarnContext(ctx, "slow sql statement",
|
|
||||||
"sql", logfield.Truncate(sql, logfield.MaxBytes),
|
|
||||||
"rows", rows,
|
|
||||||
"elapsed_ms", elapsed.Milliseconds(),
|
|
||||||
"threshold_ms", l.slowThreshold.Milliseconds(),
|
|
||||||
)
|
|
||||||
|
|
||||||
case err != nil:
|
|
||||||
// gorm.ErrRecordNotFound is not an error on the paths that
|
|
||||||
// produce it here: an invented entrypoint UUID and an unknown
|
|
||||||
// username are the expected outcome of an unauthenticated
|
|
||||||
// request, not a fault. This is the IgnoreRecordNotFoundError
|
|
||||||
// behaviour, and it is unconditional rather than configurable
|
|
||||||
// because no caller in this service wants the other one — the
|
|
||||||
// two handlers that care already record the miss themselves,
|
|
||||||
// at DEBUG, without the SQL. A miss that ran slow has already
|
|
||||||
// been reported by the arm above.
|
|
||||||
return
|
|
||||||
|
|
||||||
case l.log.Enabled(ctx, slog.LevelDebug):
|
|
||||||
sql, rows := fc()
|
|
||||||
l.log.DebugContext(ctx, "sql statement",
|
|
||||||
"sql", logfield.Truncate(sql, logfield.MaxBytes),
|
|
||||||
"rows", rows,
|
|
||||||
"elapsed_ms", elapsed.Milliseconds(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// format renders one of GORM's printf-style internal messages and
|
|
||||||
// bounds it. GORM builds these itself, but they can quote a value the
|
|
||||||
// statement carried, so they are spent through the same budget as
|
|
||||||
// everything else rather than trusted.
|
|
||||||
func format(msg string, data ...any) string {
|
|
||||||
if len(data) == 0 {
|
|
||||||
return logfield.Truncate(msg, logfield.MaxBytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
return logfield.Truncate(
|
|
||||||
fmt.Sprintf(msg, data...), logfield.MaxBytes,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -1,438 +0,0 @@
|
|||||||
package gormlog_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"fmt"
|
|
||||||
"log/slog"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/driver/sqlite"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
|
||||||
"sneak.berlin/go/webhooker/internal/middleware"
|
|
||||||
)
|
|
||||||
|
|
||||||
// fillBytes is how much client-chosen text each case drives into the
|
|
||||||
// statement. It is well past every budget in play, so a value that
|
|
||||||
// arrives short arrived short because something cut it.
|
|
||||||
const fillBytes = 8 << 10
|
|
||||||
|
|
||||||
// tailMarker sits at the far end of every generated value. A line that
|
|
||||||
// contains it carried the whole value, which means nothing cut it — so
|
|
||||||
// a value that merely happened to be short cannot pass for a truncated
|
|
||||||
// one.
|
|
||||||
const tailMarker = "ENDOFCLIENTVALUE"
|
|
||||||
|
|
||||||
// fills are the characters a client can drive into a SQL parameter,
|
|
||||||
// chosen for what the log handlers charge for them rather than for
|
|
||||||
// looking dangerous.
|
|
||||||
//
|
|
||||||
// The C0 control is the one that matters. Both handlers spell U+0001
|
|
||||||
// as a six-byte escape for the single byte it costs a client to send,
|
|
||||||
// which is the widest multiplier available in the basic multilingual
|
|
||||||
// plane and the case a raw-byte budget breaks on first. The astral
|
|
||||||
// non-printable costs ten under the text handler, four more than the
|
|
||||||
// JSON handler ever spends.
|
|
||||||
func fills() []struct {
|
|
||||||
name string
|
|
||||||
fill string
|
|
||||||
} {
|
|
||||||
return []struct {
|
|
||||||
name string
|
|
||||||
fill string
|
|
||||||
}{
|
|
||||||
{"plain", "x"},
|
|
||||||
{"quote", `"`},
|
|
||||||
{"backslash", `\`},
|
|
||||||
{"tab", "\t"},
|
|
||||||
{"newline", "\n"},
|
|
||||||
{"c0_control", "\x01"},
|
|
||||||
{"astral_nonprintable", "\U0001000C"},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// clientValue builds a value of at least fillBytes raw bytes out of
|
|
||||||
// fill, ending in tailMarker.
|
|
||||||
func clientValue(fill string) string {
|
|
||||||
var b strings.Builder
|
|
||||||
|
|
||||||
for b.Len() < fillBytes {
|
|
||||||
b.WriteString(fill)
|
|
||||||
}
|
|
||||||
|
|
||||||
b.WriteString(tailMarker)
|
|
||||||
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// handlers are the two slog handlers internal/logger can install. The
|
|
||||||
// ceiling is quoted to operators unqualified, so every case is
|
|
||||||
// asserted under both.
|
|
||||||
func handlers() []struct {
|
|
||||||
name string
|
|
||||||
make func(*bytes.Buffer) slog.Handler
|
|
||||||
} {
|
|
||||||
opts := &slog.HandlerOptions{Level: slog.LevelDebug}
|
|
||||||
|
|
||||||
return []struct {
|
|
||||||
name string
|
|
||||||
make func(*bytes.Buffer) slog.Handler
|
|
||||||
}{
|
|
||||||
{"json", func(b *bytes.Buffer) slog.Handler {
|
|
||||||
return slog.NewJSONHandler(b, opts)
|
|
||||||
}},
|
|
||||||
{"text", func(b *bytes.Buffer) slog.Handler {
|
|
||||||
return slog.NewTextHandler(b, opts)
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type thing struct {
|
|
||||||
ID string `gorm:"primaryKey"`
|
|
||||||
Name string
|
|
||||||
}
|
|
||||||
|
|
||||||
// neverSlow is a slow-statement threshold no statement in this file
|
|
||||||
// can reach. Cases that are about a non-slow arm of Trace set it, so
|
|
||||||
// that a machine under load cannot turn a miss into a slow report and
|
|
||||||
// decide the outcome for them.
|
|
||||||
const neverSlow = time.Hour
|
|
||||||
|
|
||||||
// alwaysSlow makes every statement count as slow, so the slow arm is
|
|
||||||
// reached without the test waiting for it.
|
|
||||||
const alwaysSlow = time.Nanosecond
|
|
||||||
|
|
||||||
// openDB opens a real SQLite database behind the adapter under test,
|
|
||||||
// so every assertion below is made against SQL that GORM actually
|
|
||||||
// rendered rather than against a string a test wrote by hand. slow is
|
|
||||||
// the adapter's slow-statement threshold.
|
|
||||||
func openDB(
|
|
||||||
t *testing.T, buf *bytes.Buffer, h slog.Handler, slow time.Duration,
|
|
||||||
) *gorm.DB {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
sqlDB, err := sql.Open("sqlite", fmt.Sprintf(
|
|
||||||
"file:%s?mode=rwc",
|
|
||||||
filepath.Join(t.TempDir(), "gormlog.db"),
|
|
||||||
))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
|
||||||
|
|
||||||
gl := gormlog.ExportNewWithSlowThreshold(slog.New(h), slow)
|
|
||||||
|
|
||||||
gdb, err := gorm.Open(
|
|
||||||
sqlite.Dialector{Conn: sqlDB},
|
|
||||||
&gorm.Config{Logger: gl},
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
require.NoError(t, gdb.AutoMigrate(&thing{}))
|
|
||||||
|
|
||||||
// Migration chatter is not what any of these cases is about.
|
|
||||||
buf.Reset()
|
|
||||||
|
|
||||||
return gdb
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertBounded holds every line the adapter wrote to the stated
|
|
||||||
// ceiling and proves each was cut rather than merely short.
|
|
||||||
func assertBounded(t *testing.T, out string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
assert.NotContains(
|
|
||||||
t, out, tailMarker,
|
|
||||||
"the far end of the client value reached the log, so "+
|
|
||||||
"nothing truncated it",
|
|
||||||
)
|
|
||||||
|
|
||||||
for line := range strings.SplitSeq(
|
|
||||||
strings.TrimRight(out, "\n"), "\n",
|
|
||||||
) {
|
|
||||||
if line == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.LessOrEqual(
|
|
||||||
t, len(line), middleware.MaxAccessLogLineBytes,
|
|
||||||
"log line exceeded its bound: %s",
|
|
||||||
line[:min(len(line), 300)],
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRecordNotFound_WritesNothing is the defect itself. GORM's own
|
|
||||||
// default logger prints the fully interpolated SELECT on every
|
|
||||||
// ErrRecordNotFound, and on this service's two unauthenticated
|
|
||||||
// lookups the interpolated parameter is whatever the client sent.
|
|
||||||
func TestRecordNotFound_WritesNothing(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, h := range handlers() {
|
|
||||||
for _, f := range fills() {
|
|
||||||
t.Run(h.name+"/"+f.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
gdb := openDB(t, &buf, h.make(&buf), neverSlow)
|
|
||||||
|
|
||||||
var got thing
|
|
||||||
|
|
||||||
err := gdb.Where(
|
|
||||||
"id = ?", clientValue(f.fill),
|
|
||||||
).First(&got).Error
|
|
||||||
require.ErrorIs(t, err, gorm.ErrRecordNotFound)
|
|
||||||
|
|
||||||
assert.Empty(
|
|
||||||
t, buf.String(),
|
|
||||||
"a miss on a client-chosen key must not "+
|
|
||||||
"write a log line",
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSlowRecordNotFound_IsStillReportedSlow pins the arm ordering in
|
|
||||||
// Trace against the drop above.
|
|
||||||
//
|
|
||||||
// GORM's own Trace orders its cases error-that-is-not-a-miss, then
|
|
||||||
// slow, then routine, so IgnoreRecordNotFoundError: true — the cheap
|
|
||||||
// option this adapter was chosen over — still reports a miss that ran
|
|
||||||
// slow. An adapter that dropped the miss first would be strictly less
|
|
||||||
// observant than the option it replaced, on exactly the two lookups
|
|
||||||
// this package exists for. A miss is also the statement most likely to
|
|
||||||
// be slow, since it is the one that scans without finding a row.
|
|
||||||
func TestSlowRecordNotFound_IsStillReportedSlow(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, h := range handlers() {
|
|
||||||
for _, f := range fills() {
|
|
||||||
t.Run(h.name+"/"+f.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
gdb := openDB(t, &buf, h.make(&buf), alwaysSlow)
|
|
||||||
|
|
||||||
var got thing
|
|
||||||
|
|
||||||
err := gdb.Where(
|
|
||||||
"id = ?", clientValue(f.fill),
|
|
||||||
).First(&got).Error
|
|
||||||
require.ErrorIs(t, err, gorm.ErrRecordNotFound)
|
|
||||||
|
|
||||||
assert.Contains(
|
|
||||||
t, buf.String(), slowLine,
|
|
||||||
"a slow statement that missed was not "+
|
|
||||||
"reported as slow",
|
|
||||||
)
|
|
||||||
assertBounded(t, buf.String())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRecordNotFoundFlood_DoesNotGrowWithInput states the definition
|
|
||||||
// of done directly: a flood of misses at two input sizes 64 times
|
|
||||||
// apart must cost the same number of bytes of log.
|
|
||||||
func TestRecordNotFoundFlood_DoesNotGrowWithInput(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const requests = 50
|
|
||||||
|
|
||||||
flood := func(t *testing.T, size int) int {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
gdb := openDB(
|
|
||||||
t, &buf,
|
|
||||||
slog.NewJSONHandler(&buf, &slog.HandlerOptions{
|
|
||||||
Level: slog.LevelDebug,
|
|
||||||
}),
|
|
||||||
neverSlow,
|
|
||||||
)
|
|
||||||
|
|
||||||
value := strings.Repeat("\x01", size)
|
|
||||||
|
|
||||||
for range requests {
|
|
||||||
var got thing
|
|
||||||
|
|
||||||
_ = gdb.Where("id = ?", value).First(&got).Error
|
|
||||||
}
|
|
||||||
|
|
||||||
return buf.Len()
|
|
||||||
}
|
|
||||||
|
|
||||||
small := flood(t, 128)
|
|
||||||
big := flood(t, 128*64)
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t, small, big,
|
|
||||||
"log volume tracked the size of the client's input",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStatementError_LineIsBounded covers the branch that does log.
|
|
||||||
// A driver error is not ErrRecordNotFound, so the statement is
|
|
||||||
// written, and the driver's own error text can quote what the client
|
|
||||||
// supplied. The statement's parameters are no longer part of that —
|
|
||||||
// see TestBoundValues_NeverReachTheLog — but the budget is what holds
|
|
||||||
// the line when the statement itself, or the error, is the long part.
|
|
||||||
func TestStatementError_LineIsBounded(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, h := range handlers() {
|
|
||||||
for _, f := range fills() {
|
|
||||||
t.Run(h.name+"/"+f.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
gdb := openDB(t, &buf, h.make(&buf), neverSlow)
|
|
||||||
|
|
||||||
row := thing{ID: clientValue(f.fill), Name: "a"}
|
|
||||||
|
|
||||||
require.NoError(t, gdb.Create(&row).Error)
|
|
||||||
|
|
||||||
buf.Reset()
|
|
||||||
|
|
||||||
// The same primary key a second time: a UNIQUE
|
|
||||||
// constraint failure, which is an error GORM logs.
|
|
||||||
err := gdb.Create(&thing{
|
|
||||||
ID: row.ID, Name: "b",
|
|
||||||
}).Error
|
|
||||||
require.Error(t, err)
|
|
||||||
|
|
||||||
assert.Contains(
|
|
||||||
t, buf.String(), errorLine,
|
|
||||||
)
|
|
||||||
assertBounded(t, buf.String())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSucceedingStatement_LineIsBoundedOnEitherArm covers the two
|
|
||||||
// arms a statement that returns no error can take, over the same
|
|
||||||
// query, so neither can be bounded by accident of the other.
|
|
||||||
//
|
|
||||||
// - slow. Silencing GORM outright would have been the cheaper fix
|
|
||||||
// and would have cost this report, which is the one thing GORM's
|
|
||||||
// logger gave an operator that nothing else in this service does.
|
|
||||||
// - routine. The branch an operator reaches by turning the level
|
|
||||||
// down to DEBUG: every statement is reported, so every statement
|
|
||||||
// has to be bounded too.
|
|
||||||
func TestSucceedingStatement_LineIsBoundedOnEitherArm(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// routineLine is a substring of slowLine, so the routine arm
|
|
||||||
// carries notWant as well: Contains alone cannot tell the two arms
|
|
||||||
// apart in that direction.
|
|
||||||
arms := []struct {
|
|
||||||
name string
|
|
||||||
slow time.Duration
|
|
||||||
want string
|
|
||||||
notWant string
|
|
||||||
}{
|
|
||||||
{"slow", alwaysSlow, slowLine, ""},
|
|
||||||
{"routine", neverSlow, routineLine, slowLine},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, a := range arms {
|
|
||||||
for _, h := range handlers() {
|
|
||||||
for _, f := range fills() {
|
|
||||||
name := a.name + "/" + h.name + "/" + f.name
|
|
||||||
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
gdb := openDB(t, &buf, h.make(&buf), a.slow)
|
|
||||||
|
|
||||||
var got []thing
|
|
||||||
|
|
||||||
require.NoError(t, gdb.Where(
|
|
||||||
"name = ?", clientValue(f.fill),
|
|
||||||
).Find(&got).Error)
|
|
||||||
|
|
||||||
assert.Contains(t, buf.String(), a.want)
|
|
||||||
|
|
||||||
if a.notWant != "" {
|
|
||||||
assert.NotContains(
|
|
||||||
t, buf.String(), a.notWant,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
assertBounded(t, buf.String())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGORMOwnMessages_AreBounded covers the three printf-style
|
|
||||||
// entry points. GORM builds these itself, but nothing stops one of
|
|
||||||
// them quoting a value the statement carried.
|
|
||||||
func TestGORMOwnMessages_AreBounded(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, h := range handlers() {
|
|
||||||
for _, f := range fills() {
|
|
||||||
t.Run(h.name+"/"+f.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
gl := gormlog.New(slog.New(h.make(&buf)))
|
|
||||||
ctx := context.Background()
|
|
||||||
value := clientValue(f.fill)
|
|
||||||
|
|
||||||
gl.Info(ctx, "%s", value)
|
|
||||||
gl.Warn(ctx, "%s", value)
|
|
||||||
gl.Error(ctx, "%s", value)
|
|
||||||
|
|
||||||
// The no-argument form, which is how GORM reports
|
|
||||||
// most of its own conditions. Reached through a
|
|
||||||
// function value so the vet printf check does not
|
|
||||||
// read the message as a format string — which is
|
|
||||||
// also why the adapter does not.
|
|
||||||
noArgs := func(
|
|
||||||
f func(context.Context, string, ...any),
|
|
||||||
msg string,
|
|
||||||
) {
|
|
||||||
f(ctx, msg)
|
|
||||||
}
|
|
||||||
noArgs(gl.Info, value)
|
|
||||||
|
|
||||||
assertBounded(t, buf.String())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLogMode_KeepsTheOperatorsLevel records that GORM's own level
|
|
||||||
// knob is deliberately inert: level belongs to LOG_LEVEL, and a
|
|
||||||
// second one inside the database layer could only disagree with it.
|
|
||||||
func TestLogMode_KeepsTheOperatorsLevel(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
gl := gormlog.New(slog.New(slog.NewJSONHandler(
|
|
||||||
&buf, &slog.HandlerOptions{Level: slog.LevelDebug},
|
|
||||||
)))
|
|
||||||
|
|
||||||
assert.Same(t, gl, gl.LogMode(0))
|
|
||||||
}
|
|
||||||
@@ -1,260 +0,0 @@
|
|||||||
package gormlog_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"go/ast"
|
|
||||||
"go/parser"
|
|
||||||
"go/token"
|
|
||||||
"io/fs"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
// minNonTestFiles guards the walk below against passing because it
|
|
||||||
// found nothing to look at. The tree held 60 non-test .go files when
|
|
||||||
// this was written.
|
|
||||||
const minNonTestFiles = 40
|
|
||||||
|
|
||||||
// isRowProducer reports whether name is a method that returns a
|
|
||||||
// database/sql row handle. GORM's Row and Rows return *sql.Row and
|
|
||||||
// *sql.Rows, so Scan on the result of one of them is database/sql's
|
|
||||||
// Scan and never (*gorm.DB).Scan.
|
|
||||||
func isRowProducer(name string) bool {
|
|
||||||
switch name {
|
|
||||||
case "Row", "Rows", "QueryRow", "QueryRowContext":
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// receiverIsRowHandle reports whether x is syntactically a call to a
|
|
||||||
// row producer, which is the only receiver form this check accepts for
|
|
||||||
// a Scan.
|
|
||||||
func receiverIsRowHandle(x ast.Expr) bool {
|
|
||||||
call, ok := x.(*ast.CallExpr)
|
|
||||||
if !ok {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
sel, ok := call.Fun.(*ast.SelectorExpr)
|
|
||||||
if !ok {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
return isRowProducer(sel.Sel.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
// unguardedScans returns the position of every Scan call in file whose
|
|
||||||
// receiver is not a row handle. It fails closed: a receiver it cannot
|
|
||||||
// resolve syntactically — a local variable, a struct field — is
|
|
||||||
// reported rather than assumed safe.
|
|
||||||
func unguardedScans(
|
|
||||||
fset *token.FileSet, file *ast.File,
|
|
||||||
) []token.Position {
|
|
||||||
var found []token.Position
|
|
||||||
|
|
||||||
ast.Inspect(file, func(n ast.Node) bool {
|
|
||||||
call, ok := n.(*ast.CallExpr)
|
|
||||||
if !ok {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
sel, ok := call.Fun.(*ast.SelectorExpr)
|
|
||||||
if !ok || sel.Sel.Name != "Scan" {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
if !receiverIsRowHandle(sel.X) {
|
|
||||||
found = append(found, fset.Position(sel.Sel.Pos()))
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
})
|
|
||||||
|
|
||||||
return found
|
|
||||||
}
|
|
||||||
|
|
||||||
// moduleRoot walks up from the working directory to the directory
|
|
||||||
// holding go.mod.
|
|
||||||
func moduleRoot(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
dir, err := os.Getwd()
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
for {
|
|
||||||
_, statErr := os.Stat(filepath.Join(dir, "go.mod"))
|
|
||||||
if statErr == nil {
|
|
||||||
return dir
|
|
||||||
}
|
|
||||||
|
|
||||||
parent := filepath.Dir(dir)
|
|
||||||
require.NotEqual(t, parent, dir, "no go.mod above %s", dir)
|
|
||||||
|
|
||||||
dir = parent
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// skipDir reports whether a directory holds no source this check
|
|
||||||
// governs.
|
|
||||||
func skipDir(name string) bool {
|
|
||||||
switch name {
|
|
||||||
case ".git", "bin", "node_modules", "testdata":
|
|
||||||
return true
|
|
||||||
default:
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// walkNonTestGo parses every non-test .go file under root and returns
|
|
||||||
// how many it parsed along with every unguarded Scan it found.
|
|
||||||
func walkNonTestGo(t *testing.T, root string) (int, []string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var (
|
|
||||||
parsed int
|
|
||||||
hits []string
|
|
||||||
)
|
|
||||||
|
|
||||||
fset := token.NewFileSet()
|
|
||||||
|
|
||||||
require.NoError(t, filepath.WalkDir(
|
|
||||||
root,
|
|
||||||
func(path string, d fs.DirEntry, err error) error {
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if d.IsDir() {
|
|
||||||
if skipDir(d.Name()) {
|
|
||||||
return fs.SkipDir
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if !isNonTestGo(d.Name()) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
file, err := parser.ParseFile(fset, path, nil, 0)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
parsed++
|
|
||||||
|
|
||||||
for _, pos := range unguardedScans(fset, file) {
|
|
||||||
hits = append(hits, relPosition(root, pos))
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
))
|
|
||||||
|
|
||||||
return parsed, hits
|
|
||||||
}
|
|
||||||
|
|
||||||
// isNonTestGo reports whether a file name is Go source this check
|
|
||||||
// governs.
|
|
||||||
func isNonTestGo(name string) bool {
|
|
||||||
return strings.HasSuffix(name, ".go") &&
|
|
||||||
!strings.HasSuffix(name, "_test.go")
|
|
||||||
}
|
|
||||||
|
|
||||||
// relPosition renders pos with its path relative to root, so a failure
|
|
||||||
// names the file the way the repository does.
|
|
||||||
func relPosition(root string, pos token.Position) string {
|
|
||||||
name := pos.Filename
|
|
||||||
|
|
||||||
rel, err := filepath.Rel(root, name)
|
|
||||||
if err == nil {
|
|
||||||
name = rel
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Sprintf("%s:%d:%d", name, pos.Line, pos.Column)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGormScanIsNeverCalledOutsideTests keeps (*gorm.DB).Scan out of
|
|
||||||
// non-test code.
|
|
||||||
//
|
|
||||||
// It is the one statement path (*Logger).ParamsFilter does not reach:
|
|
||||||
// Scan swaps GORM's own trace recorder in for the adapter, and that
|
|
||||||
// recorder does not implement gorm.ParamsFilter, so the statement is
|
|
||||||
// logged with its values interpolated. The package comment states the
|
|
||||||
// limit; this fails when someone adds a call site anyway.
|
|
||||||
//
|
|
||||||
// The current tree has one caller, internal/database/database_test.go,
|
|
||||||
// which this check does not govern: it is test-only and its SELECT 1
|
|
||||||
// binds nothing.
|
|
||||||
func TestGormScanIsNeverCalledOutsideTests(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
parsed, offenders := walkNonTestGo(t, moduleRoot(t))
|
|
||||||
|
|
||||||
require.GreaterOrEqual(
|
|
||||||
t, parsed, minNonTestFiles,
|
|
||||||
"parsed %d non-test .go files, so this check found "+
|
|
||||||
"nothing to look at", parsed,
|
|
||||||
)
|
|
||||||
require.Empty(
|
|
||||||
t, offenders,
|
|
||||||
"Scan called on a receiver this check cannot show is a "+
|
|
||||||
"database/sql row handle. (*gorm.DB).Scan logs the "+
|
|
||||||
"statement with its bound values interpolated — use "+
|
|
||||||
"Find, Pluck, or Raw(...).Row().Scan instead. A "+
|
|
||||||
"database/sql Scan reached through a variable is "+
|
|
||||||
"reported too; write it as <producer>().Scan rather "+
|
|
||||||
"than widening this check.",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// scanGuardCase is one planted snippet and whether the check above
|
|
||||||
// should report it.
|
|
||||||
type scanGuardCase struct {
|
|
||||||
name string
|
|
||||||
body string
|
|
||||||
want int
|
|
||||||
}
|
|
||||||
|
|
||||||
func scanGuardCases() []scanGuardCase {
|
|
||||||
return []scanGuardCase{
|
|
||||||
{"gorm chain", `db.DB().Raw("SELECT 1").Scan(&v)`, 1},
|
|
||||||
{"gorm receiver", `gdb.Scan(&v)`, 1},
|
|
||||||
{"gorm via variable", "q := gdb.Raw(\"x\")\nq.Scan(&v)", 1},
|
|
||||||
{"gorm model chain", `gdb.Model(&x).Scan(&v)`, 1},
|
|
||||||
{"sql row", `gdb.Raw("SELECT 1").Row().Scan(&v)`, 0},
|
|
||||||
{"sql rows", `gdb.Raw("SELECT 1").Rows().Scan(&v)`, 0},
|
|
||||||
{"unrelated call", `gdb.Find(&v)`, 0},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestScanGuard_ReportsPlantedCalls proves the check fires. Without it
|
|
||||||
// a detector that matched nothing would satisfy the walk above no
|
|
||||||
// matter what the tree contained.
|
|
||||||
func TestScanGuard_ReportsPlantedCalls(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range scanGuardCases() {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
fset := token.NewFileSet()
|
|
||||||
src := fmt.Sprintf(
|
|
||||||
"package p\n\nfunc f() {\n\t%s\n}\n", tc.body,
|
|
||||||
)
|
|
||||||
|
|
||||||
file, err := parser.ParseFile(
|
|
||||||
fset, tc.name+".go", src, 0,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
require.Len(t, unguardedScans(fset, file), tc.want)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,185 +0,0 @@
|
|||||||
package gormlog_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
)
|
|
||||||
|
|
||||||
// secretValue is bound as a parameter by every case below. Nothing
|
|
||||||
// else in this package writes it, so finding it in captured output
|
|
||||||
// means a bound value was rendered into the log.
|
|
||||||
const secretValue = "QQBOUNDVALUEMARKERQQ"
|
|
||||||
|
|
||||||
// The three messages Trace emits under, one per arm.
|
|
||||||
//
|
|
||||||
// routineLine is a substring of slowLine, so a case that wants the
|
|
||||||
// routine arm has to rule the slow one out as well rather than rely on
|
|
||||||
// Contains alone.
|
|
||||||
const (
|
|
||||||
routineLine = "sql statement"
|
|
||||||
slowLine = "slow sql statement"
|
|
||||||
errorLine = "sql statement failed"
|
|
||||||
)
|
|
||||||
|
|
||||||
// boundValueCase is one arm of Trace, driven by a statement that binds
|
|
||||||
// secretValue.
|
|
||||||
type boundValueCase struct {
|
|
||||||
name string
|
|
||||||
slow time.Duration
|
|
||||||
want string
|
|
||||||
drive func(t *testing.T, gdb *gorm.DB)
|
|
||||||
}
|
|
||||||
|
|
||||||
// insertSecret returns a driver that inserts one row whose Name is the
|
|
||||||
// secret.
|
|
||||||
func insertSecret(id string) func(*testing.T, *gorm.DB) {
|
|
||||||
return func(t *testing.T, gdb *gorm.DB) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
require.NoError(t, gdb.Create(&thing{
|
|
||||||
ID: id, Name: secretValue,
|
|
||||||
}).Error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// insertSecretTwice drives the error arm: the same primary key a
|
|
||||||
// second time is a UNIQUE constraint failure, which is an error GORM
|
|
||||||
// logs with the statement.
|
|
||||||
func insertSecretTwice(t *testing.T, gdb *gorm.DB) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
require.NoError(t, gdb.Create(&thing{
|
|
||||||
ID: secretValue, Name: secretValue,
|
|
||||||
}).Error)
|
|
||||||
require.Error(t, gdb.Create(&thing{
|
|
||||||
ID: secretValue, Name: "other",
|
|
||||||
}).Error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// selectSecret drives a query whose WHERE clause binds the secret,
|
|
||||||
// covering the read side as well as the write side.
|
|
||||||
func selectSecret(t *testing.T, gdb *gorm.DB) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var got []thing
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t, gdb.Where("name = ?", secretValue).Find(&got).Error,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func boundValueCases() []boundValueCase {
|
|
||||||
return []boundValueCase{
|
|
||||||
{
|
|
||||||
name: "routine", slow: neverSlow,
|
|
||||||
want: routineLine, drive: insertSecret("routine"),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "slow", slow: alwaysSlow,
|
|
||||||
want: slowLine, drive: insertSecret("slow"),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "error", slow: neverSlow,
|
|
||||||
want: errorLine, drive: insertSecretTwice,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "select", slow: neverSlow,
|
|
||||||
want: routineLine, drive: selectSecret,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestBoundValues_NeverReachTheLog states the values-off property
|
|
||||||
// directly, on each arm of Trace that emits.
|
|
||||||
//
|
|
||||||
// Truncation is not what is being asserted. A bounded secret is still
|
|
||||||
// a secret: the session key is 44 base64 characters and an Argon2id
|
|
||||||
// hash under 100, so both fit inside every budget this package
|
|
||||||
// applies. What keeps them out is that the adapter logs the
|
|
||||||
// statement's shape and discards its parameters — see
|
|
||||||
// (*Logger).ParamsFilter — and that has to hold at DEBUG as much as on
|
|
||||||
// an error, because DEBUG is the level at which a successful INSERT is
|
|
||||||
// written at all.
|
|
||||||
//
|
|
||||||
// Each case also requires a placeholder in the logged statement.
|
|
||||||
// Without that, the absence of the value would be satisfied by a
|
|
||||||
// logger that wrote nothing useful.
|
|
||||||
func TestBoundValues_NeverReachTheLog(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range boundValueCases() {
|
|
||||||
for _, h := range handlers() {
|
|
||||||
t.Run(tc.name+"/"+h.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
gdb := openDB(t, &buf, h.make(&buf), tc.slow)
|
|
||||||
|
|
||||||
tc.drive(t, gdb)
|
|
||||||
|
|
||||||
assertNoBoundValue(t, buf.String(), tc.want)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertNoBoundValue holds one captured arm to the property: it wrote
|
|
||||||
// the line it was supposed to write, that line kept its placeholders,
|
|
||||||
// and it carried no bound value.
|
|
||||||
func assertNoBoundValue(t *testing.T, out, want string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
require.Contains(
|
|
||||||
t, out, want,
|
|
||||||
"the arm under test wrote nothing, so the assertions "+
|
|
||||||
"below are vacuous",
|
|
||||||
)
|
|
||||||
assert.NotContains(
|
|
||||||
t, out, secretValue,
|
|
||||||
"a bound parameter was rendered into the log",
|
|
||||||
)
|
|
||||||
assert.Contains(
|
|
||||||
t, out, "?",
|
|
||||||
"the statement was logged without its placeholders",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestInsert_KeepsOnePlaceholderPerBoundValue pins the shape of the
|
|
||||||
// INSERT specifically, since that is the statement that carries both
|
|
||||||
// first-boot secrets. A statement that dropped one value and kept the
|
|
||||||
// other would satisfy the assertions above.
|
|
||||||
func TestInsert_KeepsOnePlaceholderPerBoundValue(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
gdb := openDB(t, &buf, handlers()[0].make(&buf), neverSlow)
|
|
||||||
|
|
||||||
require.NoError(t, gdb.Create(&thing{
|
|
||||||
ID: "m", Name: secretValue,
|
|
||||||
}).Error)
|
|
||||||
|
|
||||||
out := buf.String()
|
|
||||||
|
|
||||||
require.Contains(t, out, "INSERT INTO")
|
|
||||||
assert.NotContains(t, out, secretValue)
|
|
||||||
|
|
||||||
for line := range strings.SplitSeq(out, "\n") {
|
|
||||||
if !strings.Contains(line, "INSERT INTO") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.GreaterOrEqual(
|
|
||||||
t, strings.Count(line, "?"), 2,
|
|
||||||
"insert logged fewer placeholders than it bound "+
|
|
||||||
"values: %s", line,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -2,10 +2,8 @@ package handlers
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/logfield"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// HandleLoginPage returns a handler for the login page (GET)
|
// HandleLoginPage returns a handler for the login page (GET)
|
||||||
@@ -31,8 +29,10 @@ func (h *Handlers) HandleLoginPage() http.HandlerFunc {
|
|||||||
// HandleLoginSubmit handles the login form submission (POST)
|
// HandleLoginSubmit handles the login form submission (POST)
|
||||||
func (h *Handlers) HandleLoginSubmit() http.HandlerFunc {
|
func (h *Handlers) HandleLoginSubmit() http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
// The body size cap is enforced by the MaxBodySize
|
// Limit request body to prevent memory exhaustion
|
||||||
// middleware, which runs before CSRF parses the form.
|
r.Body = http.MaxBytesReader(w, r.Body, 1<<maxBodyShift)
|
||||||
|
|
||||||
|
// Parse form data
|
||||||
err := r.ParseForm()
|
err := r.ParseForm()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.log.Error("failed to parse form", "error", err)
|
h.log.Error("failed to parse form", "error", err)
|
||||||
@@ -41,10 +41,8 @@ func (h *Handlers) HandleLoginSubmit() http.HandlerFunc {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// PostFormValue, not FormValue: the credential must come
|
username := r.FormValue("username")
|
||||||
// from the body, never from the query string.
|
password := r.FormValue("password")
|
||||||
username := r.PostFormValue("username")
|
|
||||||
password := r.PostFormValue("password")
|
|
||||||
|
|
||||||
// Validate input
|
// Validate input
|
||||||
if username == "" || password == "" {
|
if username == "" || password == "" {
|
||||||
@@ -71,9 +69,7 @@ func (h *Handlers) HandleLoginSubmit() http.HandlerFunc {
|
|||||||
|
|
||||||
h.log.Info(
|
h.log.Info(
|
||||||
"user logged in",
|
"user logged in",
|
||||||
"username", logfield.Truncate(
|
"username", username,
|
||||||
username, logfield.MaxBytes,
|
|
||||||
),
|
|
||||||
"user_id", user.ID,
|
"user_id", user.ID,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -99,16 +95,6 @@ func (h *Handlers) renderLoginError(
|
|||||||
|
|
||||||
// authenticateUser looks up and verifies a user's credentials.
|
// authenticateUser looks up and verifies a user's credentials.
|
||||||
// On failure it writes an HTTP response and returns an error.
|
// On failure it writes an HTTP response and returns an error.
|
||||||
//
|
|
||||||
// The credential check runs BEFORE any rate-limit budget is
|
|
||||||
// consulted, and only a failed check spends budget. That is what
|
|
||||||
// keeps the single administrative path reachable: behind the reverse
|
|
||||||
// proxy this deployment requires, with TRUSTED_PROXIES unset, every
|
|
||||||
// client shares one bucket, so a limiter spent on arrival lets any
|
|
||||||
// stranger deny the operator's own correct password indefinitely.
|
|
||||||
//
|
|
||||||
// Verifying first means every login POST costs an Argon2id hash, so
|
|
||||||
// the work is taken under a bounded number of verification slots.
|
|
||||||
func (h *Handlers) authenticateUser(
|
func (h *Handlers) authenticateUser(
|
||||||
w http.ResponseWriter,
|
w http.ResponseWriter,
|
||||||
r *http.Request,
|
r *http.Request,
|
||||||
@@ -116,49 +102,16 @@ func (h *Handlers) authenticateUser(
|
|||||||
) (database.User, error) {
|
) (database.User, error) {
|
||||||
var user database.User
|
var user database.User
|
||||||
|
|
||||||
release, ok := h.mw.BeginPasswordVerification(r.Context())
|
|
||||||
if !ok {
|
|
||||||
h.log.Warn(
|
|
||||||
"password verification capacity exhausted",
|
|
||||||
"path", logfield.Truncate(
|
|
||||||
r.URL.Path, logfield.MaxBytes,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
h.renderLoginError(
|
|
||||||
w, r,
|
|
||||||
"The server is busy verifying credentials. "+
|
|
||||||
"Please try again.",
|
|
||||||
http.StatusServiceUnavailable,
|
|
||||||
)
|
|
||||||
|
|
||||||
return user, errVerificationBusy
|
|
||||||
}
|
|
||||||
|
|
||||||
defer release()
|
|
||||||
|
|
||||||
err := h.db.DB().Where(
|
err := h.db.DB().Where(
|
||||||
"username = ?", username,
|
"username = ?", username,
|
||||||
).First(&user).Error
|
).First(&user).Error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// A username that does not exist is charged the same work
|
h.log.Debug("user not found", "username", username)
|
||||||
// as one that does. Skipping the hash here would answer in
|
h.renderLoginError(
|
||||||
// microseconds where a real account takes tens of
|
w, r,
|
||||||
// milliseconds, handing every client a username oracle.
|
"Invalid username or password",
|
||||||
h.dummyVerifications.Add(1)
|
http.StatusUnauthorized,
|
||||||
database.VerifyDummyPassword(password)
|
|
||||||
|
|
||||||
// Login is unauthenticated, and the submitted username is
|
|
||||||
// a form field the client fills to any length the 1 MB
|
|
||||||
// body cap allows. On this branch it matched no row, so
|
|
||||||
// nothing else bounds it. The rate limiter caps how often
|
|
||||||
// the line is written, not how wide it is.
|
|
||||||
h.log.Debug(
|
|
||||||
"user not found",
|
|
||||||
"username", logfield.Truncate(
|
|
||||||
username, logfield.MaxBytes,
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
h.rejectLogin(w, r, username)
|
|
||||||
|
|
||||||
return user, err
|
return user, err
|
||||||
}
|
}
|
||||||
@@ -175,60 +128,17 @@ func (h *Handlers) authenticateUser(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if !valid {
|
if !valid {
|
||||||
// Reached only once the username matched a stored row, so
|
h.log.Debug("invalid password", "username", username)
|
||||||
// it is bounded by the operator's own data. Capped anyway,
|
|
||||||
// so that every username this unauthenticated endpoint
|
|
||||||
// logs is capped and no reader has to work out which
|
|
||||||
// branch narrowed it.
|
|
||||||
h.log.Debug(
|
|
||||||
"invalid password",
|
|
||||||
"username", logfield.Truncate(
|
|
||||||
username, logfield.MaxBytes,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
h.rejectLogin(w, r, username)
|
|
||||||
|
|
||||||
return user, errInvalidPassword
|
|
||||||
}
|
|
||||||
|
|
||||||
// The password was correct, so forgive whatever failures this
|
|
||||||
// client accumulated: an operator who mistypes a few times and
|
|
||||||
// then gets it right must not stay throttled afterwards.
|
|
||||||
h.mw.ForgiveLoginFailures(r, username)
|
|
||||||
|
|
||||||
return user, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// rejectLogin counts one failed credential verification and answers
|
|
||||||
// it: 401 while this client still has failure budget against the
|
|
||||||
// submitted username, 429 with a Retry-After once it is spent.
|
|
||||||
//
|
|
||||||
// The 429 throttles wrong passwords only. A correct one never
|
|
||||||
// reaches here, so no amount of failure — from this client or any
|
|
||||||
// other sharing its bucket — can keep the operator out.
|
|
||||||
func (h *Handlers) rejectLogin(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
r *http.Request,
|
|
||||||
username string,
|
|
||||||
) {
|
|
||||||
if !h.mw.RecordLoginFailure(r, username) {
|
|
||||||
h.renderLoginError(
|
h.renderLoginError(
|
||||||
w, r,
|
w, r,
|
||||||
"Invalid username or password",
|
"Invalid username or password",
|
||||||
http.StatusUnauthorized,
|
http.StatusUnauthorized,
|
||||||
)
|
)
|
||||||
|
|
||||||
return
|
return user, errInvalidPassword
|
||||||
}
|
}
|
||||||
|
|
||||||
w.Header().Set("Retry-After", strconv.Itoa(int(
|
return user, nil
|
||||||
h.mw.LoginFailureInterval().Seconds(),
|
|
||||||
)))
|
|
||||||
h.renderLoginError(
|
|
||||||
w, r,
|
|
||||||
"Too many failed login attempts. Please try again later.",
|
|
||||||
http.StatusTooManyRequests,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// createAuthenticatedSession regenerates the session and stores
|
// createAuthenticatedSession regenerates the session and stores
|
||||||
|
|||||||
@@ -1,455 +0,0 @@
|
|||||||
package handlers_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/url"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// operatorUser and operatorPassword are the single admin account
|
|
||||||
// these tests defend.
|
|
||||||
operatorUser = "admin"
|
|
||||||
operatorPassword = "correct horse battery staple"
|
|
||||||
|
|
||||||
// sharedProxyPeer is the whole point of this file. Production is
|
|
||||||
// required to run behind a TLS-terminating reverse proxy, and
|
|
||||||
// TRUSTED_PROXIES defaults to empty, so every client — attacker
|
|
||||||
// and operator alike — reaches the process from the proxy's
|
|
||||||
// address and shares one rate-limit bucket. Both parties in
|
|
||||||
// these tests therefore use the same RemoteAddr.
|
|
||||||
sharedProxyPeer = "10.0.0.1:44444"
|
|
||||||
|
|
||||||
// loginFailureLimit is the failure budget one client has against
|
|
||||||
// one submitted username. Restated here rather than imported
|
|
||||||
// from the middleware package, so that changing the production
|
|
||||||
// limit fails these tests instead of silently moving with them.
|
|
||||||
loginFailureLimit = 5
|
|
||||||
)
|
|
||||||
|
|
||||||
// seedOperator gives the bootstrapped admin account a password these
|
|
||||||
// tests know. The account itself is created at startup with a random
|
|
||||||
// password, which is exactly why its username is predictable to an
|
|
||||||
// attacker and why keying failures by username alone does not fix
|
|
||||||
// this issue.
|
|
||||||
func seedOperator(t *testing.T, db *database.Database) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
hash, err := database.HashPassword(operatorPassword)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
result := db.DB().Model(&database.User{}).
|
|
||||||
Where("username = ?", operatorUser).
|
|
||||||
Update("password", hash)
|
|
||||||
|
|
||||||
require.NoError(t, result.Error)
|
|
||||||
require.EqualValues(
|
|
||||||
t, 1, result.RowsAffected,
|
|
||||||
"the bootstrap admin account must exist",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// loginPost builds a login form POST arriving from peer.
|
|
||||||
func loginPost(peer, username, password string) *http.Request {
|
|
||||||
form := url.Values{}
|
|
||||||
form.Set("username", username)
|
|
||||||
form.Set("password", password)
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(
|
|
||||||
context.Background(),
|
|
||||||
http.MethodPost,
|
|
||||||
"/pages/login",
|
|
||||||
strings.NewReader(form.Encode()),
|
|
||||||
)
|
|
||||||
req.Header.Set(
|
|
||||||
"Content-Type", "application/x-www-form-urlencoded",
|
|
||||||
)
|
|
||||||
req.RemoteAddr = peer
|
|
||||||
|
|
||||||
return req
|
|
||||||
}
|
|
||||||
|
|
||||||
// submitLogin drives one login POST through the handler.
|
|
||||||
func submitLogin(
|
|
||||||
h *handlers.Handlers, peer, username, password string,
|
|
||||||
) *httptest.ResponseRecorder {
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
h.HandleLoginSubmit().ServeHTTP(w, loginPost(
|
|
||||||
peer, username, password,
|
|
||||||
))
|
|
||||||
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
|
|
||||||
// floodFailures sends attempts wrong-password logins for username
|
|
||||||
// from peer, which is what an attacker does.
|
|
||||||
func floodFailures(
|
|
||||||
t *testing.T,
|
|
||||||
h *handlers.Handlers,
|
|
||||||
peer, username string,
|
|
||||||
attempts int,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for i := range attempts {
|
|
||||||
w := submitLogin(h, peer, username, fmt.Sprintf("guess-%d", i))
|
|
||||||
require.NotEqual(
|
|
||||||
t, http.StatusSeeOther, w.Code,
|
|
||||||
"attempt %d must not authenticate", i,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLogin_StrangersFloodCannotLockOutTheOperator is the
|
|
||||||
// done-criterion of https://git.eeqj.de/sneak/webhooker/issues/150.
|
|
||||||
//
|
|
||||||
// The attacker and the operator share one rate-limit bucket, because
|
|
||||||
// behind the mandated reverse proxy with TRUSTED_PROXIES unset every
|
|
||||||
// client keys on the proxy's address. The attacker floods the
|
|
||||||
// operator's own username — a single-admin product has a predictable
|
|
||||||
// one — far past the failure limit. The operator must still be able
|
|
||||||
// to log in with the correct password.
|
|
||||||
//
|
|
||||||
// This fails if credentials stop being verified ahead of the limiter.
|
|
||||||
func TestLogin_StrangersFloodCannotLockOutTheOperator(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
db *database.Database
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &db)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
seedOperator(t, db)
|
|
||||||
|
|
||||||
// Well past the limit, and from the same bucket the operator
|
|
||||||
// will arrive in.
|
|
||||||
floodFailures(
|
|
||||||
t, h, sharedProxyPeer, operatorUser,
|
|
||||||
loginFailureLimit*2,
|
|
||||||
)
|
|
||||||
|
|
||||||
w := submitLogin(
|
|
||||||
h, sharedProxyPeer, operatorUser, operatorPassword,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t, http.StatusSeeOther, w.Code,
|
|
||||||
"a correct password must never be throttled: the operator "+
|
|
||||||
"has no second administrative path",
|
|
||||||
)
|
|
||||||
assert.Equal(t, "/", w.Header().Get("Location"))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLogin_StrangersFloodCannotDenyAnotherAccount is the
|
|
||||||
// cross-account half: flooding one username must not spend another
|
|
||||||
// account's budget, even from the same shared bucket.
|
|
||||||
func TestLogin_StrangersFloodCannotDenyAnotherAccount(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
db *database.Database
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &db)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
seedOperator(t, db)
|
|
||||||
|
|
||||||
floodFailures(
|
|
||||||
t, h, sharedProxyPeer, "someone-else",
|
|
||||||
loginFailureLimit*2,
|
|
||||||
)
|
|
||||||
|
|
||||||
w := submitLogin(h, sharedProxyPeer, operatorUser, "wrong")
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t, http.StatusUnauthorized, w.Code,
|
|
||||||
"a flood against one username must not spend another "+
|
|
||||||
"account's failure budget",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLogin_RepeatedWrongPasswordsAreThrottled is the brute-force
|
|
||||||
// half. Verifying before counting must not remove the throttle:
|
|
||||||
// repeated wrong passwords for one username from one client key run
|
|
||||||
// out of budget and are answered 429 with a Retry-After.
|
|
||||||
func TestLogin_RepeatedWrongPasswordsAreThrottled(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
db *database.Database
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &db)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
seedOperator(t, db)
|
|
||||||
|
|
||||||
for i := range loginFailureLimit - 1 {
|
|
||||||
w := submitLogin(
|
|
||||||
h, sharedProxyPeer, operatorUser,
|
|
||||||
fmt.Sprintf("guess-%d", i),
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t, http.StatusUnauthorized, w.Code,
|
|
||||||
"attempt %d is still inside the budget", i,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
w := submitLogin(h, sharedProxyPeer, operatorUser, "guess-last")
|
|
||||||
assert.Equal(
|
|
||||||
t, http.StatusTooManyRequests, w.Code,
|
|
||||||
"wrong passwords must still run out of budget",
|
|
||||||
)
|
|
||||||
assert.NotEmpty(
|
|
||||||
t, w.Header().Get("Retry-After"),
|
|
||||||
"a throttled login must say when to come back",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLogin_SuccessForgivesEarlierMistakes covers the operator who
|
|
||||||
// mistypes several times and then gets it right: the successful
|
|
||||||
// attempt clears the counter, so the next mistake is answered 401
|
|
||||||
// rather than 429.
|
|
||||||
func TestLogin_SuccessForgivesEarlierMistakes(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
db *database.Database
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &db)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
seedOperator(t, db)
|
|
||||||
|
|
||||||
floodFailures(
|
|
||||||
t, h, sharedProxyPeer, operatorUser,
|
|
||||||
loginFailureLimit,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.Equal(
|
|
||||||
t, http.StatusSeeOther,
|
|
||||||
submitLogin(
|
|
||||||
h, sharedProxyPeer, operatorUser, operatorPassword,
|
|
||||||
).Code,
|
|
||||||
)
|
|
||||||
|
|
||||||
w := submitLogin(h, sharedProxyPeer, operatorUser, "typo")
|
|
||||||
assert.Equal(
|
|
||||||
t, http.StatusUnauthorized, w.Code,
|
|
||||||
"a success must forgive the failures before it",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLogin_UnknownUsernameCostsTheSameVerification is the
|
|
||||||
// username-enumeration guard. Verifying credentials before the
|
|
||||||
// limiter means response time is observable per attempt, so an
|
|
||||||
// unknown username must be charged an equivalent-cost verification
|
|
||||||
// against a dummy hash rather than returning early.
|
|
||||||
//
|
|
||||||
// The assertion is on the code path, not on wall-clock time: timing
|
|
||||||
// assertions are flaky, and what actually has to hold is that the
|
|
||||||
// hash is computed.
|
|
||||||
func TestLogin_UnknownUsernameCostsTheSameVerification(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
db *database.Database
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &db)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
seedOperator(t, db)
|
|
||||||
|
|
||||||
require.Zero(t, h.DummyVerificationsForTest())
|
|
||||||
|
|
||||||
// A username that exists, with the wrong password: a real
|
|
||||||
// Argon2id verification runs, and no dummy is needed.
|
|
||||||
require.Equal(
|
|
||||||
t, http.StatusUnauthorized,
|
|
||||||
submitLogin(h, sharedProxyPeer, operatorUser, "wrong").Code,
|
|
||||||
)
|
|
||||||
assert.Zero(
|
|
||||||
t, h.DummyVerificationsForTest(),
|
|
||||||
"a known username verifies against its own hash",
|
|
||||||
)
|
|
||||||
|
|
||||||
// A username that does not exist: indistinguishable response,
|
|
||||||
// and the equivalent-cost verification must have run.
|
|
||||||
require.Equal(
|
|
||||||
t, http.StatusUnauthorized,
|
|
||||||
submitLogin(h, sharedProxyPeer, "nosuchuser", "wrong").Code,
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t, uint64(1), h.DummyVerificationsForTest(),
|
|
||||||
"an unknown username must still pay for a hash, or the "+
|
|
||||||
"response time says whether the account exists",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLogin_ConcurrentLoginsAreAllAnswered covers the login path
|
|
||||||
// under the verification bound. The bound itself is pinned in the
|
|
||||||
// middleware package; what matters here is that funnelling every
|
|
||||||
// login through two slots does not lose or wedge a request — each one
|
|
||||||
// is answered, whether it got a slot or was shed with 503.
|
|
||||||
func TestLogin_ConcurrentLoginsAreAllAnswered(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const workers = 4
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
db *database.Database
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &db)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
seedOperator(t, db)
|
|
||||||
|
|
||||||
var (
|
|
||||||
wg sync.WaitGroup
|
|
||||||
mu sync.Mutex
|
|
||||||
answers = map[int]int{}
|
|
||||||
)
|
|
||||||
|
|
||||||
for i := range workers {
|
|
||||||
wg.Go(func() {
|
|
||||||
w := submitLogin(
|
|
||||||
h, fmt.Sprintf("203.0.113.%d:5000", i),
|
|
||||||
operatorUser, fmt.Sprintf("guess-%d", i),
|
|
||||||
)
|
|
||||||
|
|
||||||
mu.Lock()
|
|
||||||
answers[w.Code]++
|
|
||||||
mu.Unlock()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
wg.Wait()
|
|
||||||
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
|
|
||||||
assert.Zero(
|
|
||||||
t, answers[http.StatusInternalServerError],
|
|
||||||
"concurrent logins must not error",
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t, workers,
|
|
||||||
answers[http.StatusUnauthorized]+
|
|
||||||
answers[http.StatusTooManyRequests]+
|
|
||||||
answers[http.StatusServiceUnavailable],
|
|
||||||
"every concurrent login must be answered, whether it got "+
|
|
||||||
"a verification slot or was shed with 503",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLogin_MissingCredentialsRejectedBeforeAnyHash pins that the
|
|
||||||
// empty-field check still runs ahead of the verification slot, so a
|
|
||||||
// client sending nothing cannot occupy one.
|
|
||||||
func TestLogin_MissingCredentialsRejectedBeforeAnyHash(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
db *database.Database
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &db)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
seedOperator(t, db)
|
|
||||||
|
|
||||||
w := submitLogin(h, sharedProxyPeer, "", "")
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusBadRequest, w.Code)
|
|
||||||
assert.Zero(
|
|
||||||
t, h.DummyVerificationsForTest(),
|
|
||||||
"an empty submission must not cost a hash",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLogin_SuccessCreatesSession is the control for the tests above:
|
|
||||||
// the success path they assert on really does authenticate.
|
|
||||||
func TestLogin_SuccessCreatesSession(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
db *database.Database
|
|
||||||
sess *session.Session
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &db, &sess)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
seedOperator(t, db)
|
|
||||||
|
|
||||||
w := submitLogin(
|
|
||||||
h, sharedProxyPeer, operatorUser, operatorPassword,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
|
||||||
require.NotEmpty(
|
|
||||||
t, w.Result().Cookies(), "a session cookie must be issued",
|
|
||||||
)
|
|
||||||
|
|
||||||
next := httptest.NewRequestWithContext(
|
|
||||||
context.Background(), http.MethodGet, "/", nil,
|
|
||||||
)
|
|
||||||
|
|
||||||
// Login regenerates the session, so the response carries two
|
|
||||||
// Set-Cookie headers under the same name: one expiring the
|
|
||||||
// pre-login cookie and one issuing the new one. A browser keeps
|
|
||||||
// only the second, so replay only the one that is not an
|
|
||||||
// expiry.
|
|
||||||
for _, c := range w.Result().Cookies() {
|
|
||||||
if c.MaxAge >= 0 {
|
|
||||||
next.AddCookie(c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
s, err := sess.Get(next)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.True(
|
|
||||||
t, sess.IsAuthenticated(s),
|
|
||||||
"the issued cookie must carry an authenticated session",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -1,378 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"strconv"
|
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"gorm.io/gorm/clause"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// replayOutcomeParam is the query parameter the replay POST redirects
|
|
||||||
// with and the event log page reads its banner from.
|
|
||||||
const replayOutcomeParam = "replay"
|
|
||||||
|
|
||||||
// replayOutcomeCode is the outcome of a replay POST. The redirect
|
|
||||||
// carries one of these fixed codes rather than a message, so nothing a
|
|
||||||
// client submits can reach the rendered page through it.
|
|
||||||
type replayOutcomeCode string
|
|
||||||
|
|
||||||
const (
|
|
||||||
// replayQueued reports that a new delivery was created and handed
|
|
||||||
// to the delivery engine.
|
|
||||||
replayQueued replayOutcomeCode = "queued"
|
|
||||||
|
|
||||||
// replayTargetDeleted reports a target that once existed and has
|
|
||||||
// since been deleted. Deletes are soft and deliveries carry no
|
|
||||||
// foreign key to the target row, so the history survives its
|
|
||||||
// target and this is the ordinary case for an old event.
|
|
||||||
replayTargetDeleted replayOutcomeCode = "target-deleted"
|
|
||||||
|
|
||||||
// replayTargetMissing reports a target id that names no row at
|
|
||||||
// all, deleted or otherwise.
|
|
||||||
replayTargetMissing replayOutcomeCode = "target-missing"
|
|
||||||
|
|
||||||
// replayTargetInactive reports a target the operator has
|
|
||||||
// deactivated. A deactivated target receives no new deliveries, so
|
|
||||||
// a replay to it would be a delivery they switched off.
|
|
||||||
replayTargetInactive replayOutcomeCode = "target-inactive"
|
|
||||||
|
|
||||||
// replayNotTerminal reports a delivery the engine has not finished
|
|
||||||
// with.
|
|
||||||
replayNotTerminal replayOutcomeCode = "not-terminal"
|
|
||||||
|
|
||||||
// replayInFlight reports that an earlier replay of this event to
|
|
||||||
// this target is still running.
|
|
||||||
replayInFlight replayOutcomeCode = "in-flight"
|
|
||||||
)
|
|
||||||
|
|
||||||
// replayOutcome returns the banner the event log page shows for an
|
|
||||||
// outcome code, and whether the replay was queued. An unrecognised
|
|
||||||
// code yields no banner.
|
|
||||||
func replayOutcome(code string) (string, bool) {
|
|
||||||
switch replayOutcomeCode(code) {
|
|
||||||
case replayQueued:
|
|
||||||
return "Replay queued: a new delivery was created against " +
|
|
||||||
"the target's current configuration.", true
|
|
||||||
case replayTargetDeleted:
|
|
||||||
return "Not replayed: the target this delivery was for has " +
|
|
||||||
"been deleted. Recreate the target, then replay.", false
|
|
||||||
case replayTargetMissing:
|
|
||||||
return "Not replayed: the target this delivery was for no " +
|
|
||||||
"longer exists.", false
|
|
||||||
case replayTargetInactive:
|
|
||||||
return "Not replayed: the target this delivery was for is " +
|
|
||||||
"deactivated. Activate it, then replay.", false
|
|
||||||
case replayNotTerminal:
|
|
||||||
return "Not replayed: this delivery has not finished yet.",
|
|
||||||
false
|
|
||||||
case replayInFlight:
|
|
||||||
return "Not replayed: a delivery of this event to this " +
|
|
||||||
"target is already in flight.", false
|
|
||||||
default:
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleDeliveryReplay re-sends a finished delivery's event to its
|
|
||||||
// target.
|
|
||||||
//
|
|
||||||
// A replay never touches the delivery it repeats. It creates a NEW
|
|
||||||
// pending delivery row for the same event and target and hands it to
|
|
||||||
// the delivery engine through the same Notifier the receiver uses, so
|
|
||||||
// the original's status, attempts and timestamps stand as the record
|
|
||||||
// of what actually happened, and the replay is retried, SSRF-guarded
|
|
||||||
// and circuit-broken exactly as a first attempt is.
|
|
||||||
//
|
|
||||||
// What is re-sent is the stored EVENT body, never the response the
|
|
||||||
// original delivery received.
|
|
||||||
//
|
|
||||||
// The target's configuration is read now rather than as it stood when
|
|
||||||
// the original ran: a replay exists to deliver where the operator
|
|
||||||
// currently wants the event to go. That is also why a deleted target
|
|
||||||
// is refused rather than delivered to from stale configuration.
|
|
||||||
func (h *Handlers) HandleDeliveryReplay() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
webhook, ok := h.ownedWebhook(w, r)
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// The body size cap is enforced by the MaxBodySize
|
|
||||||
// middleware, which runs before CSRF parses the form.
|
|
||||||
err := r.ParseForm()
|
|
||||||
if err != nil {
|
|
||||||
http.Error(
|
|
||||||
w, "Bad request", http.StatusBadRequest,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.replayDelivery(w, r, webhook)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// replayDelivery performs the replay for a webhook the caller has
|
|
||||||
// already established the session's user owns.
|
|
||||||
func (h *Handlers) replayDelivery(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
r *http.Request,
|
|
||||||
webhook database.Webhook,
|
|
||||||
) {
|
|
||||||
if !h.dbMgr.DBExists(webhook.ID) {
|
|
||||||
http.NotFound(w, r)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, "failed to get webhook database", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
original, ok := h.loadReplaySource(w, r, webhookDB)
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if !original.Status.Terminal() {
|
|
||||||
h.finishReplay(w, r, webhook, replayNotTerminal)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
target, code := h.replayTarget(webhook.ID, original.TargetID)
|
|
||||||
if target == nil {
|
|
||||||
h.finishReplay(w, r, webhook, code)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.queueReplay(w, r, webhookDB, webhook, original, target)
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadReplaySource loads the delivery to be replayed, selecting only
|
|
||||||
// the columns the replay needs so no association is populated. A
|
|
||||||
// delivery id that names no row in this webhook's database is a 404.
|
|
||||||
func (h *Handlers) loadReplaySource(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
r *http.Request,
|
|
||||||
webhookDB *gorm.DB,
|
|
||||||
) (*database.Delivery, bool) {
|
|
||||||
var original database.Delivery
|
|
||||||
|
|
||||||
err := webhookDB.
|
|
||||||
Select("id", "event_id", "target_id", "status").
|
|
||||||
First(
|
|
||||||
&original, "id = ?", chi.URLParam(r, "deliveryID"),
|
|
||||||
).Error
|
|
||||||
if err != nil {
|
|
||||||
http.NotFound(w, r)
|
|
||||||
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
return &original, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// queueReplay writes the new delivery and hands it to the engine.
|
|
||||||
func (h *Handlers) queueReplay(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
r *http.Request,
|
|
||||||
webhookDB *gorm.DB,
|
|
||||||
webhook database.Webhook,
|
|
||||||
original *database.Delivery,
|
|
||||||
target *database.Target,
|
|
||||||
) {
|
|
||||||
inFlight, err := countInFlightDeliveries(
|
|
||||||
webhookDB, original.EventID, target.ID,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(
|
|
||||||
w, "failed to count in-flight deliveries", err,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if inFlight > 0 {
|
|
||||||
h.finishReplay(w, r, webhook, replayInFlight)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
var event database.Event
|
|
||||||
|
|
||||||
err = webhookDB.
|
|
||||||
First(&event, "id = ?", original.EventID).Error
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, "failed to load event for replay", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
task, err := createReplayDelivery(
|
|
||||||
webhookDB, webhook.ID, &event, target,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(
|
|
||||||
w, "failed to create replay delivery", err,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.mtr.DeliveryReplayed(target.Type)
|
|
||||||
h.notifier.Notify([]delivery.Task{task})
|
|
||||||
|
|
||||||
h.log.Info(
|
|
||||||
"delivery replay queued",
|
|
||||||
"webhook_id", webhook.ID,
|
|
||||||
"event_id", event.ID,
|
|
||||||
"target_id", target.ID,
|
|
||||||
"replayed_delivery_id", original.ID,
|
|
||||||
"delivery_id", task.DeliveryID,
|
|
||||||
)
|
|
||||||
|
|
||||||
h.finishReplay(w, r, webhook, replayQueued)
|
|
||||||
}
|
|
||||||
|
|
||||||
// replayTarget loads the delivery's target as it stands now.
|
|
||||||
//
|
|
||||||
// The load is Unscoped so that a soft-deleted row is still found:
|
|
||||||
// deletes are soft and a delivery carries no foreign key to its
|
|
||||||
// target, so a target's history outlives it, and without the deleted
|
|
||||||
// row there is no way to tell "you deleted this target" from "this id
|
|
||||||
// never named anything". A nil target means the replay is refused,
|
|
||||||
// with the returned code saying why.
|
|
||||||
func (h *Handlers) replayTarget(
|
|
||||||
webhookID, targetID string,
|
|
||||||
) (*database.Target, replayOutcomeCode) {
|
|
||||||
var target database.Target
|
|
||||||
|
|
||||||
err := h.db.DB().Unscoped().Where(
|
|
||||||
"id = ? AND webhook_id = ?", targetID, webhookID,
|
|
||||||
).First(&target).Error
|
|
||||||
if err != nil {
|
|
||||||
return nil, replayTargetMissing
|
|
||||||
}
|
|
||||||
|
|
||||||
if target.DeletedAt.Valid {
|
|
||||||
return nil, replayTargetDeleted
|
|
||||||
}
|
|
||||||
|
|
||||||
if !target.Active {
|
|
||||||
return nil, replayTargetInactive
|
|
||||||
}
|
|
||||||
|
|
||||||
return &target, replayQueued
|
|
||||||
}
|
|
||||||
|
|
||||||
// countInFlightDeliveries reports how many deliveries of this event to
|
|
||||||
// this target the engine has not finished.
|
|
||||||
//
|
|
||||||
// It is the replay-storm guard: a replay is refused while an earlier
|
|
||||||
// one is still pending or retrying, so a held-down button or a scripted
|
|
||||||
// loop cannot stack copies of work already queued. It is a check and
|
|
||||||
// not a lock, so two simultaneous POSTs can still both pass it; the
|
|
||||||
// per-client rate limit on the route is what bounds that.
|
|
||||||
func countInFlightDeliveries(
|
|
||||||
webhookDB *gorm.DB, eventID, targetID string,
|
|
||||||
) (int64, error) {
|
|
||||||
var count int64
|
|
||||||
|
|
||||||
err := webhookDB.Model(&database.Delivery{}).Where(
|
|
||||||
"event_id = ? AND target_id = ? AND status IN ?",
|
|
||||||
eventID, targetID,
|
|
||||||
[]database.DeliveryStatus{
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
database.DeliveryStatusRetrying,
|
|
||||||
},
|
|
||||||
).Count(&count).Error
|
|
||||||
|
|
||||||
return count, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// createReplayDelivery writes the new pending delivery row and returns
|
|
||||||
// the task that carries it to the delivery engine.
|
|
||||||
//
|
|
||||||
// The row is written with associations omitted, and neither Event nor
|
|
||||||
// Target is populated on it: GORM's SaveBeforeAssociations would
|
|
||||||
// otherwise upsert the whole target row — plaintext config, which for a
|
|
||||||
// Slack target is the credential — into the per-webhook event database.
|
|
||||||
// See https://git.eeqj.de/sneak/webhooker/issues/206.
|
|
||||||
func createReplayDelivery(
|
|
||||||
webhookDB *gorm.DB,
|
|
||||||
webhookID string,
|
|
||||||
event *database.Event,
|
|
||||||
target *database.Target,
|
|
||||||
) (delivery.Task, error) {
|
|
||||||
dlv := &database.Delivery{
|
|
||||||
EventID: event.ID,
|
|
||||||
TargetID: target.ID,
|
|
||||||
Status: database.DeliveryStatusPending,
|
|
||||||
}
|
|
||||||
|
|
||||||
err := webhookDB.Omit(clause.Associations).Create(dlv).Error
|
|
||||||
if err != nil {
|
|
||||||
return delivery.Task{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return delivery.Task{
|
|
||||||
DeliveryID: dlv.ID,
|
|
||||||
EventID: event.ID,
|
|
||||||
WebhookID: webhookID,
|
|
||||||
EntrypointID: event.EntrypointID,
|
|
||||||
TargetID: target.ID,
|
|
||||||
TargetName: target.Name,
|
|
||||||
TargetType: target.Type,
|
|
||||||
TargetConfig: target.Config,
|
|
||||||
MaxRetries: target.MaxRetries,
|
|
||||||
Method: event.Method,
|
|
||||||
Headers: event.Headers,
|
|
||||||
ContentType: event.ContentType,
|
|
||||||
Body: replayBody(event.Body),
|
|
||||||
AttemptNum: 1,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// replayBody returns the stored event body for a replay task to carry
|
|
||||||
// inline, or nil when it is large enough that the engine should fetch
|
|
||||||
// it from the per-webhook database instead.
|
|
||||||
func replayBody(body string) *string {
|
|
||||||
if len(body) >= delivery.MaxInlineBodySize {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return &body
|
|
||||||
}
|
|
||||||
|
|
||||||
// finishReplay redirects back to the event log the replay was
|
|
||||||
// triggered from, carrying the outcome code the page turns into a
|
|
||||||
// banner and the page number the form submitted.
|
|
||||||
func (h *Handlers) finishReplay(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
r *http.Request,
|
|
||||||
webhook database.Webhook,
|
|
||||||
code replayOutcomeCode,
|
|
||||||
) {
|
|
||||||
dest := "/source/" + webhook.ID + "/logs?" +
|
|
||||||
replayOutcomeParam + "=" + string(code)
|
|
||||||
|
|
||||||
// The page is read from the form rather than the query string:
|
|
||||||
// this is a POST, and its query string is what logs and Referer
|
|
||||||
// headers record.
|
|
||||||
if page := parseNonNegativeInt(
|
|
||||||
r.PostFormValue("page"),
|
|
||||||
); page > 1 {
|
|
||||||
dest += "&page=" + strconv.Itoa(page)
|
|
||||||
}
|
|
||||||
|
|
||||||
http.Redirect(w, r, dest, http.StatusSeeOther)
|
|
||||||
}
|
|
||||||
@@ -1,526 +0,0 @@
|
|||||||
package handlers_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"gorm.io/gorm/clause"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
|
||||||
)
|
|
||||||
|
|
||||||
// paramDeliveryID is the chi URL parameter name the replay handler
|
|
||||||
// reads.
|
|
||||||
const paramDeliveryID = "deliveryID"
|
|
||||||
|
|
||||||
// replayTargetURL is a public destination, so a target configured with
|
|
||||||
// it is one the SSRF guard would accept. Nothing in these tests
|
|
||||||
// dispatches to it: the notifier is recorded, not run.
|
|
||||||
const replayTargetURL = "http://93.184.216.34/hook"
|
|
||||||
|
|
||||||
// seedFailedDelivery records an event, a terminally failed delivery of
|
|
||||||
// it to the given target, and the attempt that failed.
|
|
||||||
func seedFailedDelivery(
|
|
||||||
t *testing.T,
|
|
||||||
dbMgr *database.WebhookDBManager,
|
|
||||||
webhookID, targetID string,
|
|
||||||
) (*database.Event, *database.Delivery) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
webhookDB, err := dbMgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
event := &database.Event{
|
|
||||||
WebhookID: webhookID,
|
|
||||||
EntrypointID: "entrypoint-" + webhookID,
|
|
||||||
Method: http.MethodPost,
|
|
||||||
Headers: `{"X-Test":["yes"]}`,
|
|
||||||
Body: `{"replay":"me"}`,
|
|
||||||
ContentType: contentTypeJSON,
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, webhookDB.Omit(
|
|
||||||
clause.Associations,
|
|
||||||
).Create(event).Error)
|
|
||||||
|
|
||||||
dlv := &database.Delivery{
|
|
||||||
EventID: event.ID,
|
|
||||||
TargetID: targetID,
|
|
||||||
Status: database.DeliveryStatusFailed,
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, webhookDB.Omit(
|
|
||||||
clause.Associations,
|
|
||||||
).Create(dlv).Error)
|
|
||||||
|
|
||||||
result := &database.DeliveryResult{
|
|
||||||
DeliveryID: dlv.ID,
|
|
||||||
AttemptNum: 1,
|
|
||||||
Success: false,
|
|
||||||
StatusCode: http.StatusBadGateway,
|
|
||||||
Error: "connection refused",
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, webhookDB.Omit(
|
|
||||||
clause.Associations,
|
|
||||||
).Create(result).Error)
|
|
||||||
|
|
||||||
return event, dlv
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadDelivery reads a delivery back out of a webhook's database.
|
|
||||||
func loadDelivery(
|
|
||||||
t *testing.T, webhookDB *gorm.DB, deliveryID string,
|
|
||||||
) database.Delivery {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var dlv database.Delivery
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t,
|
|
||||||
webhookDB.First(&dlv, "id = ?", deliveryID).Error,
|
|
||||||
)
|
|
||||||
|
|
||||||
return dlv
|
|
||||||
}
|
|
||||||
|
|
||||||
// listDeliveries reads every delivery of an event.
|
|
||||||
func listDeliveries(
|
|
||||||
t *testing.T, webhookDB *gorm.DB, eventID string,
|
|
||||||
) []database.Delivery {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var deliveries []database.Delivery
|
|
||||||
|
|
||||||
require.NoError(t, webhookDB.Where(
|
|
||||||
"event_id = ?", eventID,
|
|
||||||
).Find(&deliveries).Error)
|
|
||||||
|
|
||||||
return deliveries
|
|
||||||
}
|
|
||||||
|
|
||||||
// theOtherDelivery returns the one delivery in the slice that is not
|
|
||||||
// excludeID. Identity is used rather than an ordering because the rows
|
|
||||||
// are minted milliseconds apart and their ids are random.
|
|
||||||
func theOtherDelivery(
|
|
||||||
t *testing.T,
|
|
||||||
deliveries []database.Delivery,
|
|
||||||
excludeID string,
|
|
||||||
) database.Delivery {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var found []database.Delivery
|
|
||||||
|
|
||||||
for _, d := range deliveries {
|
|
||||||
if d.ID != excludeID {
|
|
||||||
found = append(found, d)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
require.Len(t, found, 1)
|
|
||||||
|
|
||||||
return found[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
// postReplay runs the real replay handler for one delivery.
|
|
||||||
func postReplay(
|
|
||||||
t *testing.T,
|
|
||||||
h *handlers.Handlers,
|
|
||||||
sess *session.Session,
|
|
||||||
webhookID, deliveryID string,
|
|
||||||
) *httptest.ResponseRecorder {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
req := postRequest(
|
|
||||||
"/source/"+webhookID+"/deliveries/"+
|
|
||||||
deliveryID+"/replay",
|
|
||||||
authenticatedCookies(
|
|
||||||
t, sess, deleteTestUserID, deleteTestUsername,
|
|
||||||
),
|
|
||||||
map[string]string{
|
|
||||||
paramSourceID: webhookID,
|
|
||||||
paramDeliveryID: deliveryID,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
h.HandleDeliveryReplay().ServeHTTP(w, req)
|
|
||||||
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleDeliveryReplay_AppendsDeliveryAndLeavesOriginal is the
|
|
||||||
// core requirement: replaying a failed delivery succeeds, appends a
|
|
||||||
// new delivery, and leaves the original row and its recorded attempt
|
|
||||||
// exactly as they were.
|
|
||||||
//
|
|
||||||
// It also pins the two things a replay would be wrong to get from the
|
|
||||||
// original: the task carries the target's CURRENT configuration, which
|
|
||||||
// this test changes between the failure and the replay, and it carries
|
|
||||||
// the stored EVENT body rather than anything the failed attempt
|
|
||||||
// received back.
|
|
||||||
func TestHandleDeliveryReplay_AppendsDeliveryAndLeavesOriginal(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
notif *recordingNotifier
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr, ¬if)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
tgt := seedConfiguredTarget(
|
|
||||||
t, db, wh.ID, database.TargetTypeHTTP,
|
|
||||||
`{"url":"`+replayTargetURL+`"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
event, original := seedFailedDelivery(
|
|
||||||
t, dbMgr, wh.ID, tgt.ID,
|
|
||||||
)
|
|
||||||
|
|
||||||
webhookDB, err := dbMgr.GetDB(wh.ID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
before := loadDelivery(t, webhookDB, original.ID)
|
|
||||||
|
|
||||||
// The operator fixes the destination, which is the whole reason
|
|
||||||
// to replay. The replay must use this, not the config the
|
|
||||||
// original delivery ran against.
|
|
||||||
const fixedConfig = `{"url":"http://93.184.216.34/fixed"}`
|
|
||||||
|
|
||||||
require.NoError(t, db.DB().Model(&database.Target{}).
|
|
||||||
Where("id = ?", tgt.ID).
|
|
||||||
Update("config", fixedConfig).Error)
|
|
||||||
|
|
||||||
w := postReplay(t, h, sess, wh.ID, original.ID)
|
|
||||||
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
"/source/"+wh.ID+"/logs?replay=queued",
|
|
||||||
w.Header().Get("Location"),
|
|
||||||
)
|
|
||||||
|
|
||||||
deliveries := listDeliveries(t, webhookDB, event.ID)
|
|
||||||
require.Len(
|
|
||||||
t, deliveries, 2,
|
|
||||||
"replay must append a delivery, not reuse one",
|
|
||||||
)
|
|
||||||
|
|
||||||
replayed := theOtherDelivery(t, deliveries, original.ID)
|
|
||||||
assert.Equal(t, tgt.ID, replayed.TargetID)
|
|
||||||
assert.Equal(t, event.ID, replayed.EventID)
|
|
||||||
assert.Equal(
|
|
||||||
t, database.DeliveryStatusPending, replayed.Status,
|
|
||||||
)
|
|
||||||
|
|
||||||
assertDeliveryUntouched(t, webhookDB, before)
|
|
||||||
|
|
||||||
tasks := notif.Tasks()
|
|
||||||
require.Len(t, tasks, 1)
|
|
||||||
assertReplayTask(
|
|
||||||
t, tasks[0], wh.ID, event, tgt, replayed.ID, fixedConfig,
|
|
||||||
)
|
|
||||||
assertNoLeakedTarget(t, webhookDB)
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertDeliveryUntouched proves a delivery row is exactly as it was
|
|
||||||
// read before: same terminal status, same timestamps, and the same
|
|
||||||
// recorded attempts.
|
|
||||||
func assertDeliveryUntouched(
|
|
||||||
t *testing.T,
|
|
||||||
webhookDB *gorm.DB,
|
|
||||||
before database.Delivery,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
after := loadDelivery(t, webhookDB, before.ID)
|
|
||||||
assert.Equal(
|
|
||||||
t, before.Status, after.Status,
|
|
||||||
"replay must not resurrect the original delivery",
|
|
||||||
)
|
|
||||||
assert.Equal(t, before.UpdatedAt, after.UpdatedAt)
|
|
||||||
assert.Equal(t, before.CreatedAt, after.CreatedAt)
|
|
||||||
|
|
||||||
var attempts int64
|
|
||||||
|
|
||||||
require.NoError(t, webhookDB.
|
|
||||||
Model(&database.DeliveryResult{}).
|
|
||||||
Where("delivery_id = ?", before.ID).
|
|
||||||
Count(&attempts).Error)
|
|
||||||
assert.Equal(
|
|
||||||
t, int64(1), attempts,
|
|
||||||
"the original delivery's attempt history must stand",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertReplayTask proves the task handed to the delivery engine is
|
|
||||||
// the one the receiver would build for this event and this target, and
|
|
||||||
// that it carries wantConfig — the target's configuration as it stands
|
|
||||||
// now rather than as the original delivery ran against it.
|
|
||||||
func assertReplayTask(
|
|
||||||
t *testing.T,
|
|
||||||
task delivery.Task,
|
|
||||||
webhookID string,
|
|
||||||
event *database.Event,
|
|
||||||
target *database.Target,
|
|
||||||
wantDeliveryID, wantConfig string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
assert.Equal(t, wantDeliveryID, task.DeliveryID)
|
|
||||||
assert.Equal(t, event.ID, task.EventID)
|
|
||||||
assert.Equal(t, webhookID, task.WebhookID)
|
|
||||||
assert.Equal(t, event.EntrypointID, task.EntrypointID)
|
|
||||||
assert.Equal(t, target.ID, task.TargetID)
|
|
||||||
assert.Equal(t, target.Type, task.TargetType)
|
|
||||||
assert.JSONEq(
|
|
||||||
t, wantConfig, task.TargetConfig,
|
|
||||||
"replay must use the target's current configuration",
|
|
||||||
)
|
|
||||||
assert.Equal(t, event.Method, task.Method)
|
|
||||||
assert.Equal(t, event.Headers, task.Headers)
|
|
||||||
assert.Equal(t, event.ContentType, task.ContentType)
|
|
||||||
assert.Equal(t, 1, task.AttemptNum)
|
|
||||||
require.NotNil(t, task.Body)
|
|
||||||
assert.Equal(
|
|
||||||
t, event.Body, *task.Body,
|
|
||||||
"replay re-sends the stored event body",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertNoLeakedTarget proves the per-webhook database holds no target
|
|
||||||
// rows. AutoMigrate creates the table there because Delivery declares
|
|
||||||
// the relation, so it is a ROW that signals a leak: an association
|
|
||||||
// write would have upserted the whole target, plaintext config and
|
|
||||||
// all, into the event database. See
|
|
||||||
// https://git.eeqj.de/sneak/webhooker/issues/206.
|
|
||||||
func assertNoLeakedTarget(t *testing.T, webhookDB *gorm.DB) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var leaked int64
|
|
||||||
|
|
||||||
require.NoError(t, webhookDB.Unscoped().
|
|
||||||
Model(&database.Target{}).Count(&leaked).Error)
|
|
||||||
assert.Zero(
|
|
||||||
t, leaked,
|
|
||||||
"replay must not write the target into the event database",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleDeliveryReplay_RefusesDeletedTarget proves the required
|
|
||||||
// refusal: a target deleted since the delivery ran is reported as
|
|
||||||
// deleted rather than erroring, and nothing is created or queued.
|
|
||||||
func TestHandleDeliveryReplay_RefusesDeletedTarget(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
notif *recordingNotifier
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr, ¬if)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
tgt := seedConfiguredTarget(
|
|
||||||
t, db, wh.ID, database.TargetTypeHTTP,
|
|
||||||
`{"url":"`+replayTargetURL+`"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
event, original := seedFailedDelivery(
|
|
||||||
t, dbMgr, wh.ID, tgt.ID,
|
|
||||||
)
|
|
||||||
|
|
||||||
// Deletes are soft, so the delivery history outlives the target.
|
|
||||||
require.NoError(t, db.DB().Where(
|
|
||||||
"id = ?", tgt.ID,
|
|
||||||
).Delete(&database.Target{}).Error)
|
|
||||||
|
|
||||||
w := postReplay(t, h, sess, wh.ID, original.ID)
|
|
||||||
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
"/source/"+wh.ID+"/logs?replay=target-deleted",
|
|
||||||
w.Header().Get("Location"),
|
|
||||||
)
|
|
||||||
|
|
||||||
webhookDB, err := dbMgr.GetDB(wh.ID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
assert.Len(
|
|
||||||
t, listDeliveries(t, webhookDB, event.ID), 1,
|
|
||||||
"a refused replay must create no delivery",
|
|
||||||
)
|
|
||||||
assert.Empty(
|
|
||||||
t, notif.Tasks(),
|
|
||||||
"a refused replay must queue nothing",
|
|
||||||
)
|
|
||||||
|
|
||||||
// The refusal is specific, which is why the target is looked up
|
|
||||||
// including soft-deleted rows: an id that never named a target
|
|
||||||
// is a different outcome, and a different message, from one the
|
|
||||||
// operator deleted.
|
|
||||||
_, orphan := seedFailedDelivery(
|
|
||||||
t, dbMgr, wh.ID, "target-that-never-existed",
|
|
||||||
)
|
|
||||||
|
|
||||||
missing := postReplay(t, h, sess, wh.ID, orphan.ID)
|
|
||||||
require.Equal(t, http.StatusSeeOther, missing.Code)
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
"/source/"+wh.ID+"/logs?replay=target-missing",
|
|
||||||
missing.Header().Get("Location"),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleDeliveryReplay_RefusesWhileEarlierReplayInFlight proves
|
|
||||||
// the replay-storm guard: a second replay of the same event to the
|
|
||||||
// same target is refused while the first is still queued, so repeated
|
|
||||||
// submissions cannot stack copies of work the engine has not done.
|
|
||||||
func TestHandleDeliveryReplay_RefusesWhileEarlierReplayInFlight(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
notif *recordingNotifier
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr, ¬if)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
tgt := seedConfiguredTarget(
|
|
||||||
t, db, wh.ID, database.TargetTypeHTTP,
|
|
||||||
`{"url":"`+replayTargetURL+`"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
event, original := seedFailedDelivery(
|
|
||||||
t, dbMgr, wh.ID, tgt.ID,
|
|
||||||
)
|
|
||||||
|
|
||||||
first := postReplay(t, h, sess, wh.ID, original.ID)
|
|
||||||
require.Equal(t, http.StatusSeeOther, first.Code)
|
|
||||||
require.Equal(
|
|
||||||
t,
|
|
||||||
"/source/"+wh.ID+"/logs?replay=queued",
|
|
||||||
first.Header().Get("Location"),
|
|
||||||
)
|
|
||||||
|
|
||||||
second := postReplay(t, h, sess, wh.ID, original.ID)
|
|
||||||
require.Equal(t, http.StatusSeeOther, second.Code)
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
"/source/"+wh.ID+"/logs?replay=in-flight",
|
|
||||||
second.Header().Get("Location"),
|
|
||||||
)
|
|
||||||
|
|
||||||
webhookDB, err := dbMgr.GetDB(wh.ID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
assert.Len(
|
|
||||||
t, listDeliveries(t, webhookDB, event.ID), 2,
|
|
||||||
"the refused second replay must add nothing",
|
|
||||||
)
|
|
||||||
assert.Len(
|
|
||||||
t, notif.Tasks(), 1,
|
|
||||||
"only the first replay reaches the delivery engine",
|
|
||||||
)
|
|
||||||
|
|
||||||
// A delivery the engine has not finished is not replayable
|
|
||||||
// either, which is the same rule seen from the other side.
|
|
||||||
queued := theOtherDelivery(
|
|
||||||
t, listDeliveries(t, webhookDB, event.ID), original.ID,
|
|
||||||
)
|
|
||||||
|
|
||||||
pending := postReplay(t, h, sess, wh.ID, queued.ID)
|
|
||||||
require.Equal(t, http.StatusSeeOther, pending.Code)
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
"/source/"+wh.ID+"/logs?replay=not-terminal",
|
|
||||||
pending.Header().Get("Location"),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceLogs_RendersReplayControlAndBanner proves the action
|
|
||||||
// reaches the page it belongs on: a finished delivery renders a POST
|
|
||||||
// form carrying a CSRF token, and the outcome code a refusal redirects
|
|
||||||
// with becomes a readable message.
|
|
||||||
func TestHandleSourceLogs_RendersReplayControlAndBanner(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
tgt := seedConfiguredTarget(
|
|
||||||
t, db, wh.ID, database.TargetTypeHTTP,
|
|
||||||
`{"url":"`+replayTargetURL+`"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
_, original := seedFailedDelivery(t, dbMgr, wh.ID, tgt.ID)
|
|
||||||
|
|
||||||
body := renderSourceLogsPage(t, h, sess, wh.ID)
|
|
||||||
|
|
||||||
assert.Contains(
|
|
||||||
t, body,
|
|
||||||
`action="/source/`+wh.ID+`/deliveries/`+
|
|
||||||
original.ID+`/replay"`,
|
|
||||||
)
|
|
||||||
assert.Contains(t, body, `method="POST"`)
|
|
||||||
assert.Contains(t, body, `name="csrf_token"`)
|
|
||||||
assert.Contains(t, body, ">Replay<")
|
|
||||||
|
|
||||||
refused := renderSourceLogsPageWithQuery(
|
|
||||||
t, h, sess, wh.ID, "?replay=target-deleted",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Contains(t, refused, "alert-error")
|
|
||||||
assert.Contains(t, refused, "has been deleted")
|
|
||||||
|
|
||||||
// An outcome code nobody issued renders no banner at all.
|
|
||||||
unknown := renderSourceLogsPageWithQuery(
|
|
||||||
t, h, sess, wh.ID, "?replay=made-up",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.NotContains(t, unknown, "alert-error")
|
|
||||||
assert.NotContains(t, unknown, "alert-success")
|
|
||||||
assert.NotContains(t, unknown, "made-up")
|
|
||||||
}
|
|
||||||
@@ -1,169 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// maxRenderedResponseBytes caps how many bytes of one stored
|
|
||||||
// delivery response body reach the event log page.
|
|
||||||
//
|
|
||||||
// The bound is enforced here and in SQL, because this page's
|
|
||||||
// memory profile must not depend on a constant in another
|
|
||||||
// package staying where it is, and because rows predating the
|
|
||||||
// delivery engine's own cap or restored from an archive are
|
|
||||||
// not covered by it at all.
|
|
||||||
//
|
|
||||||
// It happens to equal that engine cap, so a row written by the
|
|
||||||
// current engine reaches this one exactly and is never cut
|
|
||||||
// twice. Nothing here may assume the two differ: see view.
|
|
||||||
const maxRenderedResponseBytes = 4096
|
|
||||||
|
|
||||||
// deliveryResultColumns is the delivery attempt projection.
|
|
||||||
// The casts to blob are load-bearing for the same reason they
|
|
||||||
// are in eventLogColumns: they make substr and length count
|
|
||||||
// bytes rather than characters, and they make SQLite do the
|
|
||||||
// cut, so an oversized stored response never becomes a Go
|
|
||||||
// string at all.
|
|
||||||
const deliveryResultColumns = "delivery_id, attempt_num, success, " +
|
|
||||||
"status_code, error, duration, " +
|
|
||||||
"substr(cast(response_body as blob), 1, ?) AS response_body, " +
|
|
||||||
"length(cast(response_body as blob)) AS response_bytes"
|
|
||||||
|
|
||||||
// DeliveryResultView is the display-safe projection of one
|
|
||||||
// delivery attempt for the event log page. It carries a
|
|
||||||
// capped response body plus the true stored size, so the page
|
|
||||||
// can mark a response as truncated without holding the whole
|
|
||||||
// thing.
|
|
||||||
//
|
|
||||||
// Both Error and ResponseBody have been through the target's
|
|
||||||
// Redactor. The engine already masks the URL out of the
|
|
||||||
// errors it stores, so for errors this is a second line
|
|
||||||
// covering rows written before it did; for response bodies it
|
|
||||||
// is the only line, and its reach is what
|
|
||||||
// delivery.Redactor documents.
|
|
||||||
type DeliveryResultView struct {
|
|
||||||
AttemptNum int
|
|
||||||
Success bool
|
|
||||||
|
|
||||||
// StatusCode is 0 when the attempt never got a response,
|
|
||||||
// which is why the page asks HasStatusCode rather than
|
|
||||||
// printing the number.
|
|
||||||
StatusCode int
|
|
||||||
|
|
||||||
// Error is the stored failure message, redacted.
|
|
||||||
Error string
|
|
||||||
|
|
||||||
// DurationMS is how long the attempt took.
|
|
||||||
DurationMS int64
|
|
||||||
|
|
||||||
// ResponseBody holds at most maxRenderedResponseBytes
|
|
||||||
// bytes of the stored response, redacted. It is remote
|
|
||||||
// content and must only ever be rendered escaped.
|
|
||||||
ResponseBody string
|
|
||||||
|
|
||||||
// ResponseBytes is the size of the stored response body,
|
|
||||||
// before the cut and before redaction. It is what the
|
|
||||||
// remote sent only when ResponseSizeKnown is set.
|
|
||||||
ResponseBytes int64
|
|
||||||
|
|
||||||
// ResponseShownBytes is how much of that the page is
|
|
||||||
// showing. It is the size of the cut, taken before
|
|
||||||
// redaction, so the truncation marker reports what SQLite
|
|
||||||
// returned rather than how much the marker substitution
|
|
||||||
// then changed the length.
|
|
||||||
ResponseShownBytes int
|
|
||||||
|
|
||||||
// ResponseTruncated reports that the body shown may be
|
|
||||||
// incomplete, so the page owes the reader a marker. Every
|
|
||||||
// body that reaches the cap counts, because one the
|
|
||||||
// delivery engine cut at its own equal cap is
|
|
||||||
// indistinguishable from a complete one.
|
|
||||||
ResponseTruncated bool
|
|
||||||
|
|
||||||
// ResponseSizeKnown reports that ResponseBytes is the whole
|
|
||||||
// response the remote sent, so the marker may quote it. It
|
|
||||||
// is false for a body that only reaches the cap, where how
|
|
||||||
// much came after it was never recorded.
|
|
||||||
ResponseSizeKnown bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// HasStatusCode reports whether the attempt got as far as an
|
|
||||||
// HTTP response. A transport failure stores no status code,
|
|
||||||
// and rendering that as "0" would read as a real status.
|
|
||||||
func (v DeliveryResultView) HasStatusCode() bool {
|
|
||||||
return v.StatusCode != 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// deliveryResultRow is one row of the delivery attempt
|
|
||||||
// projection. Its response body arrives already cut to the
|
|
||||||
// cap by SQLite, with the true size beside it.
|
|
||||||
type deliveryResultRow struct {
|
|
||||||
DeliveryID string
|
|
||||||
AttemptNum int
|
|
||||||
Success bool
|
|
||||||
StatusCode int
|
|
||||||
Error string
|
|
||||||
Duration int64
|
|
||||||
ResponseBody []byte
|
|
||||||
ResponseBytes int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// view projects a loaded row for rendering, stripping the
|
|
||||||
// target's own credential out of the two fields a remote peer
|
|
||||||
// gets to influence.
|
|
||||||
func (r *deliveryResultRow) view(
|
|
||||||
redactor delivery.Redactor,
|
|
||||||
) DeliveryResultView {
|
|
||||||
body := r.ResponseBody
|
|
||||||
|
|
||||||
// Two different cuts can have shortened this body, and the
|
|
||||||
// row records only one of them. SQLite cuts here, whenever
|
|
||||||
// the stored value is larger than the cap. The delivery
|
|
||||||
// engine cut earlier, whenever the remote sent more than
|
|
||||||
// its own maxBodyLog — which is this same number, so such a
|
|
||||||
// row stores the cut length as its whole length and nothing
|
|
||||||
// in it separates a response that ended at the cap from one
|
|
||||||
// severed there.
|
|
||||||
//
|
|
||||||
// So a body that reaches the cap is treated as cut either
|
|
||||||
// way. Gating on ResponseBytes alone would assume the two
|
|
||||||
// caps differ, and they do not: under the current engine
|
|
||||||
// that gate never opens.
|
|
||||||
cut := r.ResponseBytes > int64(len(body)) ||
|
|
||||||
len(body) >= maxRenderedResponseBytes
|
|
||||||
|
|
||||||
// The row holds more than the page shows only in the first
|
|
||||||
// of those cases. In the second the stored row is all there
|
|
||||||
// is, and its size is a floor rather than the true one.
|
|
||||||
sizeKnown := r.ResponseBytes > int64(len(body))
|
|
||||||
|
|
||||||
// Only a cut response can have been left mid-sequence,
|
|
||||||
// exactly as with an event body.
|
|
||||||
if cut {
|
|
||||||
body = trimPartialRune(body)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A cut body goes through RedactCut: the remote controls
|
|
||||||
// the padding ahead of a credential it echoes, so it
|
|
||||||
// controls where the cut falls inside that credential, and
|
|
||||||
// the severed prefix left behind matches no secret whole.
|
|
||||||
rendered := string(body)
|
|
||||||
if cut {
|
|
||||||
rendered = redactor.RedactCut(rendered)
|
|
||||||
} else {
|
|
||||||
rendered = redactor.Redact(rendered)
|
|
||||||
}
|
|
||||||
|
|
||||||
return DeliveryResultView{
|
|
||||||
AttemptNum: r.AttemptNum,
|
|
||||||
Success: r.Success,
|
|
||||||
StatusCode: r.StatusCode,
|
|
||||||
Error: redactor.Redact(r.Error),
|
|
||||||
DurationMS: r.Duration,
|
|
||||||
ResponseBody: rendered,
|
|
||||||
ResponseBytes: r.ResponseBytes,
|
|
||||||
ResponseShownBytes: len(body),
|
|
||||||
ResponseTruncated: cut,
|
|
||||||
ResponseSizeKnown: sizeKnown,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,515 +0,0 @@
|
|||||||
package handlers_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/gorm/clause"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
|
||||||
)
|
|
||||||
|
|
||||||
// responseCap is the number of response bytes the event log
|
|
||||||
// page is allowed to render for one delivery attempt.
|
|
||||||
const responseCap = handlers.MaxRenderedResponseBytesForTest
|
|
||||||
|
|
||||||
// failedAttempt describes the failed delivery every test in
|
|
||||||
// this file seeds. The values are distinctive so that finding
|
|
||||||
// them in the rendered page cannot be a coincidence.
|
|
||||||
const (
|
|
||||||
attemptStatusCode = 502
|
|
||||||
attemptDurationMS = 1234
|
|
||||||
attemptNumber = 3
|
|
||||||
attemptError = "upstream returned 502 Bad Gateway"
|
|
||||||
)
|
|
||||||
|
|
||||||
// seedFailedDeliveryWithResponse records an event, a failed
|
|
||||||
// delivery against targetID, and one delivery result carrying
|
|
||||||
// the given response body. It returns the delivery.
|
|
||||||
//
|
|
||||||
// Distinct from seedFailedDelivery in delivery_replay_test.go,
|
|
||||||
// which seeds an attempt with no response body and returns the
|
|
||||||
// event as well; these tests need the recorded response.
|
|
||||||
func seedFailedDeliveryWithResponse(
|
|
||||||
t *testing.T,
|
|
||||||
dbMgr *database.WebhookDBManager,
|
|
||||||
webhookID, targetID, responseBody string,
|
|
||||||
) *database.Delivery {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
webhookDB, err := dbMgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
event := &database.Event{
|
|
||||||
WebhookID: webhookID,
|
|
||||||
Method: http.MethodPost,
|
|
||||||
Body: `{"test":true}`,
|
|
||||||
ContentType: contentTypeJSON,
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, webhookDB.Omit(
|
|
||||||
clause.Associations,
|
|
||||||
).Create(event).Error)
|
|
||||||
|
|
||||||
dlv := &database.Delivery{
|
|
||||||
EventID: event.ID,
|
|
||||||
TargetID: targetID,
|
|
||||||
Status: database.DeliveryStatusFailed,
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, webhookDB.Omit(
|
|
||||||
clause.Associations,
|
|
||||||
).Create(dlv).Error)
|
|
||||||
|
|
||||||
result := &database.DeliveryResult{
|
|
||||||
DeliveryID: dlv.ID,
|
|
||||||
AttemptNum: attemptNumber,
|
|
||||||
Success: false,
|
|
||||||
StatusCode: attemptStatusCode,
|
|
||||||
ResponseBody: responseBody,
|
|
||||||
Error: attemptError,
|
|
||||||
Duration: attemptDurationMS,
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, webhookDB.Omit(
|
|
||||||
clause.Associations,
|
|
||||||
).Create(result).Error)
|
|
||||||
|
|
||||||
return dlv
|
|
||||||
}
|
|
||||||
|
|
||||||
// seedFailureAndRender seeds a failed delivery against a
|
|
||||||
// target of the given type and config, and returns the
|
|
||||||
// rendered event log page.
|
|
||||||
func seedFailureAndRender(
|
|
||||||
t *testing.T,
|
|
||||||
targetType database.TargetType,
|
|
||||||
config, responseBody string,
|
|
||||||
) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
tgt := seedConfiguredTarget(
|
|
||||||
t, db, wh.ID, targetType, config,
|
|
||||||
)
|
|
||||||
|
|
||||||
seedFailedDeliveryWithResponse(
|
|
||||||
t, dbMgr, wh.ID, tgt.ID, responseBody,
|
|
||||||
)
|
|
||||||
|
|
||||||
return renderSourceLogsPage(t, h, sess, wh.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceLogs_RendersFailedAttempt is the regression
|
|
||||||
// test for the reported gap: a failed delivery used to render
|
|
||||||
// as the status word alone, so diagnosing it meant opening the
|
|
||||||
// per-webhook SQLite file by hand.
|
|
||||||
func TestHandleSourceLogs_RendersFailedAttempt(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
body := seedFailureAndRender(
|
|
||||||
t,
|
|
||||||
database.TargetTypeHTTP,
|
|
||||||
`{"url":"https://example.com/hook/abc"}`,
|
|
||||||
"upstream exploded",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Contains(
|
|
||||||
t, body, strconv.Itoa(attemptStatusCode),
|
|
||||||
"the attempt's status code must reach the page",
|
|
||||||
)
|
|
||||||
assert.Contains(
|
|
||||||
t, body, attemptError,
|
|
||||||
"the attempt's error must reach the page",
|
|
||||||
)
|
|
||||||
assert.Contains(
|
|
||||||
t, body, strconv.Itoa(attemptDurationMS),
|
|
||||||
"the attempt's duration must reach the page",
|
|
||||||
)
|
|
||||||
assert.Contains(
|
|
||||||
t, body, "Attempt "+strconv.Itoa(attemptNumber),
|
|
||||||
"the attempt number must reach the page",
|
|
||||||
)
|
|
||||||
assert.Contains(
|
|
||||||
t, body, "upstream exploded",
|
|
||||||
"the attempt's response body must reach the page",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceLogs_EscapesResponseBody proves the
|
|
||||||
// response body is treated as the untrusted remote content it
|
|
||||||
// is. The remote chooses these bytes and the page is rendered
|
|
||||||
// inside the operator's authenticated origin, where the
|
|
||||||
// application's own CSP allows inline script from 'self'.
|
|
||||||
func TestHandleSourceLogs_EscapesResponseBody(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const payload = `<script>alert("xss")</script>`
|
|
||||||
|
|
||||||
body := seedFailureAndRender(
|
|
||||||
t,
|
|
||||||
database.TargetTypeHTTP,
|
|
||||||
`{"url":"https://example.com/hook/abc"}`,
|
|
||||||
payload,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.NotContains(t, body, payload)
|
|
||||||
assert.NotContains(t, body, "<script>alert")
|
|
||||||
assert.Contains(t, body, "alert")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceLogs_RedactsCredentialEchoedInResponse
|
|
||||||
// covers the case that makes rendering a response body a
|
|
||||||
// disclosure question at all: the remote echoes back the
|
|
||||||
// credential the request carried, and the page would then put
|
|
||||||
// it on the operator's screen.
|
|
||||||
func TestHandleSourceLogs_RedactsCredentialEchoedInResponse(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
body := seedFailureAndRender(
|
|
||||||
t,
|
|
||||||
database.TargetTypeSlack,
|
|
||||||
`{"webhookUrl":"`+slackWebhookURL+`"}`,
|
|
||||||
"no_service: "+slackWebhookURL,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.NotContains(t, body, slackSecretPath)
|
|
||||||
assert.NotContains(t, body, "T00000000")
|
|
||||||
assert.NotContains(t, body, "B00000000")
|
|
||||||
assert.Contains(t, body, delivery.RedactionMarker)
|
|
||||||
|
|
||||||
// The rest of the response is still shown, or the
|
|
||||||
// redaction would have cost the operator the diagnosis.
|
|
||||||
assert.Contains(t, body, "no_service")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceLogs_RedactsCredentialEchoedInError covers
|
|
||||||
// the same disclosure through the error field. The delivery
|
|
||||||
// engine masks the URL out of the errors it stores, so this
|
|
||||||
// holds the read path to the rows written before it did.
|
|
||||||
func TestHandleSourceLogs_RedactsCredentialEchoedInError(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
tgt := seedConfiguredTarget(
|
|
||||||
t, db, wh.ID,
|
|
||||||
database.TargetTypeSlack,
|
|
||||||
`{"webhookUrl":"`+slackWebhookURL+`"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
dlv := seedFailedDeliveryWithResponse(t, dbMgr, wh.ID, tgt.ID, "")
|
|
||||||
|
|
||||||
webhookDB, err := dbMgr.GetDB(wh.ID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// An unmasked transport error, exactly as Go's HTTP
|
|
||||||
// client renders one.
|
|
||||||
require.NoError(t, webhookDB.Model(
|
|
||||||
&database.DeliveryResult{},
|
|
||||||
).Where(
|
|
||||||
"delivery_id = ?", dlv.ID,
|
|
||||||
).Update(
|
|
||||||
"error",
|
|
||||||
`Post "`+slackWebhookURL+`": dial tcp: i/o timeout`,
|
|
||||||
).Error)
|
|
||||||
|
|
||||||
body := renderSourceLogsPage(t, h, sess, wh.ID)
|
|
||||||
|
|
||||||
assert.NotContains(t, body, slackSecretPath)
|
|
||||||
assert.Contains(t, body, delivery.RedactionMarker)
|
|
||||||
assert.Contains(t, body, "i/o timeout")
|
|
||||||
}
|
|
||||||
|
|
||||||
// severedPadding is the filler that puts the end of an echoed
|
|
||||||
// webhook URL five bytes past a cut at the response cap, so
|
|
||||||
// the cut leaves the workspace ID, the bot ID and all but the
|
|
||||||
// last few token characters behind.
|
|
||||||
func severedPadding() string {
|
|
||||||
const severedTail = 5
|
|
||||||
|
|
||||||
return strings.Repeat(
|
|
||||||
"A", responseCap-len(slackWebhookURL)+severedTail,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceLogs_RedactsCredentialSeveredByTheEngineCut
|
|
||||||
// is the regression test for a redactor gated on the SQL cut
|
|
||||||
// alone. The delivery engine stops reading a response at its
|
|
||||||
// own cap, which is the same number of bytes this page
|
|
||||||
// renders, so a row the engine cut is byte-for-byte
|
|
||||||
// indistinguishable from a complete response and that gate
|
|
||||||
// never opened on anything the engine writes.
|
|
||||||
//
|
|
||||||
// The seeded body is what the engine stores for any remote
|
|
||||||
// that sends at least that much: exactly responseCap bytes,
|
|
||||||
// ending in a severed webhook URL.
|
|
||||||
// TestDeliverHTTP_CutsStoredResponseAtMaxBodyLog in
|
|
||||||
// internal/delivery pins that this is the size it produces.
|
|
||||||
func TestHandleSourceLogs_RedactsCredentialSeveredByTheEngineCut(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
sent := severedPadding() + slackWebhookURL +
|
|
||||||
strings.Repeat("Z", 128)
|
|
||||||
stored := sent[:responseCap]
|
|
||||||
|
|
||||||
require.Len(
|
|
||||||
t, stored, responseCap,
|
|
||||||
"the engine stores exactly the cap, never more",
|
|
||||||
)
|
|
||||||
require.Contains(
|
|
||||||
t, stored, "T00000000",
|
|
||||||
"the severed credential must be in what is seeded",
|
|
||||||
)
|
|
||||||
|
|
||||||
body := seedFailureAndRender(
|
|
||||||
t,
|
|
||||||
database.TargetTypeSlack,
|
|
||||||
`{"webhookUrl":"`+slackWebhookURL+`"}`,
|
|
||||||
stored,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.NotContains(t, body, "T00000000")
|
|
||||||
assert.NotContains(t, body, "B00000000")
|
|
||||||
assert.Contains(t, body, delivery.RedactionMarker)
|
|
||||||
assert.Contains(
|
|
||||||
t, body, "reached the recording limit",
|
|
||||||
"a body the engine cut must not be shown as complete",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceLogs_RedactsCredentialSeveredBySQLCut covers
|
|
||||||
// the same severing for a row larger than the cap, which is
|
|
||||||
// SQLite's cut rather than the engine's. The current engine
|
|
||||||
// writes no such row; rows predating its cap or restored from
|
|
||||||
// an archive are not bounded by it, which is why the page cuts
|
|
||||||
// again in SQL and has to redact that cut too.
|
|
||||||
func TestHandleSourceLogs_RedactsCredentialSeveredBySQLCut(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
stored := severedPadding() + slackWebhookURL +
|
|
||||||
strings.Repeat("Z", 128)
|
|
||||||
|
|
||||||
require.Greater(
|
|
||||||
t, len(stored), responseCap,
|
|
||||||
"the stored body must exceed the cap or nothing is cut",
|
|
||||||
)
|
|
||||||
|
|
||||||
body := seedFailureAndRender(
|
|
||||||
t,
|
|
||||||
database.TargetTypeSlack,
|
|
||||||
`{"webhookUrl":"`+slackWebhookURL+`"}`,
|
|
||||||
stored,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.NotContains(t, body, "T00000000")
|
|
||||||
assert.NotContains(t, body, "B00000000")
|
|
||||||
assert.NotContains(
|
|
||||||
t, body, slackWebhookURL[:len(slackWebhookURL)-10],
|
|
||||||
)
|
|
||||||
assert.Contains(t, body, delivery.RedactionMarker)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceLogs_RedactsForSoftDeletedTarget covers a
|
|
||||||
// target an operator has deleted. The row is only soft deleted
|
|
||||||
// and its deliveries survive in the per-webhook database, so
|
|
||||||
// its redactor has to survive with it or every response body
|
|
||||||
// it ever recorded renders unredacted.
|
|
||||||
func TestHandleSourceLogs_RedactsForSoftDeletedTarget(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
tgt := seedConfiguredTarget(
|
|
||||||
t, db, wh.ID,
|
|
||||||
database.TargetTypeSlack,
|
|
||||||
`{"webhookUrl":"`+slackWebhookURL+`"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
seedFailedDeliveryWithResponse(
|
|
||||||
t, dbMgr, wh.ID, tgt.ID,
|
|
||||||
"no_service: "+slackWebhookURL,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, db.DB().Delete(tgt).Error)
|
|
||||||
|
|
||||||
body := renderSourceLogsPage(t, h, sess, wh.ID)
|
|
||||||
|
|
||||||
assert.NotContains(t, body, slackSecretPath)
|
|
||||||
assert.NotContains(t, body, "T00000000")
|
|
||||||
assert.Contains(t, body, delivery.RedactionMarker)
|
|
||||||
assert.Contains(t, body, "no_service")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceLogs_BoundsRenderedAttempts pins the ceiling
|
|
||||||
// on how many of one delivery's attempts reach the page, and
|
|
||||||
// that what it drops is counted rather than hidden.
|
|
||||||
func TestHandleSourceLogs_BoundsRenderedAttempts(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const extraAttempts = 7
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
tgt := seedConfiguredTarget(
|
|
||||||
t, db, wh.ID, database.TargetTypeLog, "",
|
|
||||||
)
|
|
||||||
|
|
||||||
dlv := seedFailedDeliveryWithResponse(t, dbMgr, wh.ID, tgt.ID, "")
|
|
||||||
|
|
||||||
webhookDB, err := dbMgr.GetDB(wh.ID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
total := handlers.MaxRenderedAttemptsForTest + extraAttempts
|
|
||||||
|
|
||||||
// seedFailedDeliveryWithResponse already recorded one attempt.
|
|
||||||
for i := range total - 1 {
|
|
||||||
require.NoError(t, webhookDB.Omit(
|
|
||||||
clause.Associations,
|
|
||||||
).Create(&database.DeliveryResult{
|
|
||||||
DeliveryID: dlv.ID,
|
|
||||||
AttemptNum: attemptNumber + 1 + i,
|
|
||||||
Error: attemptError,
|
|
||||||
}).Error)
|
|
||||||
}
|
|
||||||
|
|
||||||
views := h.LoadEventLogViewsForTest(
|
|
||||||
httptest.NewRecorder(), *wh, 1,
|
|
||||||
)
|
|
||||||
require.Len(t, views, 1)
|
|
||||||
require.Len(t, views[0].Deliveries, 1)
|
|
||||||
|
|
||||||
dv := views[0].Deliveries[0]
|
|
||||||
|
|
||||||
assert.Equal(t, total, dv.AttemptCount)
|
|
||||||
assert.Len(
|
|
||||||
t, dv.Results, handlers.MaxRenderedAttemptsForTest,
|
|
||||||
)
|
|
||||||
assert.Equal(t, extraAttempts, dv.AttemptsOmitted)
|
|
||||||
|
|
||||||
page := renderSourceLogsPage(t, h, sess, wh.ID)
|
|
||||||
|
|
||||||
assert.Contains(t, page, "attempts omitted")
|
|
||||||
assert.Contains(
|
|
||||||
t, page, strconv.Itoa(total)+" attempts",
|
|
||||||
"the header must count every recorded attempt",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceLogs_BoundsOversizeResponse proves the
|
|
||||||
// rendered page is bounded by the response cap rather than by
|
|
||||||
// the stored response size. The cut happens in SQLite, so the
|
|
||||||
// oversized value never becomes a Go string; this asserts the
|
|
||||||
// observable consequence, that neither the page nor the
|
|
||||||
// projection carries the tail.
|
|
||||||
func TestHandleSourceLogs_BoundsOversizeResponse(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const tail = "QQRESPONSETAILQQ"
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
tgt := seedConfiguredTarget(
|
|
||||||
t, db, wh.ID, database.TargetTypeLog, "",
|
|
||||||
)
|
|
||||||
|
|
||||||
stored := strings.Repeat("A", responseCap*4) + tail
|
|
||||||
seedFailedDeliveryWithResponse(t, dbMgr, wh.ID, tgt.ID, stored)
|
|
||||||
|
|
||||||
views := h.LoadEventLogViewsForTest(
|
|
||||||
httptest.NewRecorder(), *wh, 1,
|
|
||||||
)
|
|
||||||
require.Len(t, views, 1)
|
|
||||||
require.Len(t, views[0].Deliveries, 1)
|
|
||||||
require.Len(t, views[0].Deliveries[0].Results, 1)
|
|
||||||
|
|
||||||
attempt := views[0].Deliveries[0].Results[0]
|
|
||||||
|
|
||||||
assert.LessOrEqual(
|
|
||||||
t, len(attempt.ResponseBody), responseCap,
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t, int64(len(stored)), attempt.ResponseBytes,
|
|
||||||
)
|
|
||||||
assert.True(t, attempt.ResponseTruncated)
|
|
||||||
|
|
||||||
page := renderSourceLogsPage(t, h, sess, wh.ID)
|
|
||||||
|
|
||||||
assert.NotContains(t, page, tail)
|
|
||||||
assert.Contains(
|
|
||||||
t, page, "Response truncated for display",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -1,342 +0,0 @@
|
|||||||
package handlers_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/url"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
|
||||||
)
|
|
||||||
|
|
||||||
// submitEntrypointSecret posts the signature configuration form for
|
|
||||||
// an entrypoint and returns the recorder.
|
|
||||||
func submitEntrypointSecret(
|
|
||||||
t *testing.T,
|
|
||||||
h *handlers.Handlers,
|
|
||||||
cookies []*http.Cookie,
|
|
||||||
webhookID, entrypointID, scheme, secret string,
|
|
||||||
) *httptest.ResponseRecorder {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
form := url.Values{}
|
|
||||||
form.Set("signature_scheme", scheme)
|
|
||||||
form.Set("secret", secret)
|
|
||||||
|
|
||||||
req := formRequest(
|
|
||||||
"/source/"+webhookID+"/entrypoints/"+
|
|
||||||
entrypointID+"/secret",
|
|
||||||
cookies,
|
|
||||||
form,
|
|
||||||
map[string]string{
|
|
||||||
paramSourceID: webhookID,
|
|
||||||
entrypointIDParam: entrypointID,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
|
|
||||||
h.HandleEntrypointSecret().ServeHTTP(w, req)
|
|
||||||
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
|
|
||||||
// reloadEntrypoint reads an entrypoint back from the database,
|
|
||||||
// including the columns the model keeps out of JSON.
|
|
||||||
func reloadEntrypoint(
|
|
||||||
t *testing.T,
|
|
||||||
db *database.Database,
|
|
||||||
id string,
|
|
||||||
) database.Entrypoint {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var ep database.Entrypoint
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t, db.DB().Where("id = ?", id).First(&ep).Error,
|
|
||||||
)
|
|
||||||
|
|
||||||
return ep
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEntrypointSecretSetRotateAndRemove walks the whole lifecycle
|
|
||||||
// the UI has to support: turning verification on, rotating the secret
|
|
||||||
// to a new value, and turning it back off.
|
|
||||||
func TestEntrypointSecretSetRotateAndRemove(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
cookies := authenticatedCookies(
|
|
||||||
t, sess, deleteTestUserID, deleteTestUsername,
|
|
||||||
)
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
ep := seedSignedEntrypoint(
|
|
||||||
t, db, wh.ID, database.SignatureSchemeNone, "",
|
|
||||||
)
|
|
||||||
|
|
||||||
// Set.
|
|
||||||
w := submitEntrypointSecret(
|
|
||||||
t, h, cookies, wh.ID, ep.ID, "github", inboundSecret,
|
|
||||||
)
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
|
||||||
|
|
||||||
stored := reloadEntrypoint(t, db, ep.ID)
|
|
||||||
assert.Equal(
|
|
||||||
t, database.SignatureSchemeGitHub, stored.SignatureScheme,
|
|
||||||
)
|
|
||||||
assert.Equal(t, inboundSecret, stored.SignatureSecret)
|
|
||||||
assert.True(t, stored.SignatureConfigured())
|
|
||||||
|
|
||||||
// Rotate: a new secret and a different scheme in one submission.
|
|
||||||
// The new value is submitted with surrounding whitespace, the way
|
|
||||||
// a secret pasted out of a password manager arrives; storing that
|
|
||||||
// verbatim would make every later request fail verification with
|
|
||||||
// nothing visible on either side to explain it.
|
|
||||||
const rotated = "QQROTATEDSECRETQQ"
|
|
||||||
|
|
||||||
w = submitEntrypointSecret(
|
|
||||||
t, h, cookies, wh.ID, ep.ID, "gitlab", " "+rotated+"\t",
|
|
||||||
)
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
|
||||||
|
|
||||||
stored = reloadEntrypoint(t, db, ep.ID)
|
|
||||||
assert.Equal(
|
|
||||||
t, database.SignatureSchemeGitLab, stored.SignatureScheme,
|
|
||||||
)
|
|
||||||
assert.Equal(t, rotated, stored.SignatureSecret)
|
|
||||||
|
|
||||||
// Remove. The secret has to go with the scheme: a stored
|
|
||||||
// credential nothing reads is one more copy to leak.
|
|
||||||
w = submitEntrypointSecret(t, h, cookies, wh.ID, ep.ID, "", "")
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
|
||||||
|
|
||||||
stored = reloadEntrypoint(t, db, ep.ID)
|
|
||||||
assert.Equal(
|
|
||||||
t, database.SignatureSchemeNone, stored.SignatureScheme,
|
|
||||||
)
|
|
||||||
assert.Empty(t, stored.SignatureSecret)
|
|
||||||
assert.False(t, stored.SignatureConfigured())
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEntrypointSecretRejectsBadInput proves the form cannot create a
|
|
||||||
// row the receiver would later have to refuse. Both rejections leave
|
|
||||||
// the stored configuration untouched rather than half-applied.
|
|
||||||
func TestEntrypointSecretRejectsBadInput(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
scheme string
|
|
||||||
secret string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "unsupported scheme",
|
|
||||||
scheme: "stripe",
|
|
||||||
secret: inboundSecret,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "scheme with no secret",
|
|
||||||
scheme: "github",
|
|
||||||
secret: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// Whitespace is stripped, so a secret of spaces is an
|
|
||||||
// empty one.
|
|
||||||
name: "scheme with blank secret",
|
|
||||||
scheme: "github",
|
|
||||||
secret: " ",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
cookies := authenticatedCookies(
|
|
||||||
t, sess, deleteTestUserID, deleteTestUsername,
|
|
||||||
)
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
ep := seedSignedEntrypoint(
|
|
||||||
t, db, wh.ID,
|
|
||||||
database.SignatureSchemeGitLab, inboundSecret,
|
|
||||||
)
|
|
||||||
|
|
||||||
w := submitEntrypointSecret(
|
|
||||||
t, h, cookies, wh.ID, ep.ID, tc.scheme, tc.secret,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t, http.StatusBadRequest, w.Code, "case %s", tc.name,
|
|
||||||
)
|
|
||||||
|
|
||||||
stored := reloadEntrypoint(t, db, ep.ID)
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
database.SignatureSchemeGitLab,
|
|
||||||
stored.SignatureScheme,
|
|
||||||
"case %s", tc.name,
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t, inboundSecret, stored.SignatureSecret,
|
|
||||||
"case %s", tc.name,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEntrypointSecretRequiresOwnership proves the configuration
|
|
||||||
// endpoint is bound by the same ownership check as the rest of the
|
|
||||||
// webhook's pages: another user's entrypoint is a 404, and the secret
|
|
||||||
// is not touched.
|
|
||||||
func TestEntrypointSecretRequiresOwnership(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
ep := seedSignedEntrypoint(
|
|
||||||
t, db, wh.ID,
|
|
||||||
database.SignatureSchemeGitLab, inboundSecret,
|
|
||||||
)
|
|
||||||
|
|
||||||
stranger := authenticatedCookies(
|
|
||||||
t, sess, "someone-else", "someoneelse",
|
|
||||||
)
|
|
||||||
|
|
||||||
w := submitEntrypointSecret(
|
|
||||||
t, h, stranger, wh.ID, ep.ID, "github", "hijacked",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusNotFound, w.Code)
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
inboundSecret,
|
|
||||||
reloadEntrypoint(t, db, ep.ID).SignatureSecret,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceDetail_MasksEntrypointSecret is the regression test
|
|
||||||
// for the credential on the entrypoint: the page has to say that
|
|
||||||
// verification is configured and which header carries it, without the
|
|
||||||
// secret itself ever reaching the rendered HTML.
|
|
||||||
func TestHandleSourceDetail_MasksEntrypointSecret(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
seedSignedEntrypoint(
|
|
||||||
t, db, wh.ID,
|
|
||||||
database.SignatureSchemeGitHub, inboundSecret,
|
|
||||||
)
|
|
||||||
|
|
||||||
body := renderSourceDetailPage(t, h, sess, wh.ID)
|
|
||||||
|
|
||||||
assert.NotContains(t, body, inboundSecret)
|
|
||||||
assert.Contains(t, body, "GitHub")
|
|
||||||
assert.Contains(t, body, "X-Hub-Signature-256")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEntrypointViewsDropTheSecret pins the projection itself, so the
|
|
||||||
// barrier survives a template rewrite that stops rendering the field
|
|
||||||
// the page test above looks at.
|
|
||||||
func TestEntrypointViewsDropTheSecret(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
views := handlers.NewEntrypointViews([]database.Entrypoint{
|
|
||||||
{
|
|
||||||
Path: "p1",
|
|
||||||
Active: true,
|
|
||||||
SignatureScheme: database.SignatureSchemeGitHub,
|
|
||||||
SignatureSecret: inboundSecret,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
Path: "p2",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// Half a configuration. The receiver 500s every request
|
|
||||||
// to this row, so the UI must not call it unverified.
|
|
||||||
Path: "p2a",
|
|
||||||
SignatureScheme: database.SignatureSchemeGitLab,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// The other half.
|
|
||||||
Path: "p2b",
|
|
||||||
SignatureSecret: inboundSecret,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// A scheme this build does not know: described as
|
|
||||||
// unavailable, never echoed back.
|
|
||||||
Path: "p3",
|
|
||||||
SignatureScheme: database.SignatureScheme("stripe"),
|
|
||||||
SignatureSecret: inboundSecret,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
require.Len(t, views, 5)
|
|
||||||
|
|
||||||
assert.True(t, views[0].Configured)
|
|
||||||
assert.Equal(t, "GitHub", views[0].SchemeLabel)
|
|
||||||
assert.Equal(t, "X-Hub-Signature-256", views[0].SchemeHeader)
|
|
||||||
|
|
||||||
assert.False(t, views[1].Configured)
|
|
||||||
assert.Equal(t, "not verified", views[1].SchemeLabel)
|
|
||||||
assert.Empty(t, views[1].SchemeHeader)
|
|
||||||
|
|
||||||
for _, v := range []handlers.EntrypointView{views[2], views[3]} {
|
|
||||||
assert.False(t, v.Configured)
|
|
||||||
assert.Equal(t, "misconfigured", v.SchemeLabel)
|
|
||||||
assert.Empty(t, v.SchemeHeader)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.True(t, views[4].Configured)
|
|
||||||
assert.Equal(t, "(unavailable)", views[4].SchemeLabel)
|
|
||||||
|
|
||||||
// The struct has no field that could carry the secret, so this
|
|
||||||
// fails to compile rather than fails at runtime if one is added
|
|
||||||
// and populated. The assertion covers the labels it derives.
|
|
||||||
for _, v := range views {
|
|
||||||
assert.NotContains(t, v.SchemeLabel, inboundSecret)
|
|
||||||
assert.NotContains(t, v.SchemeHeader, inboundSecret)
|
|
||||||
assert.NotContains(t, string(v.Scheme), inboundSecret)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,91 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/signature"
|
|
||||||
)
|
|
||||||
|
|
||||||
// signatureUnavailable is what an entrypoint's scheme renders as when
|
|
||||||
// the stored value is not one this build supports. The stored string
|
|
||||||
// is never echoed as a fallback: it is operator-supplied and the row
|
|
||||||
// is already in a state the receiver refuses, so the UI says so
|
|
||||||
// rather than inventing a description for it.
|
|
||||||
const signatureUnavailable = "(unavailable)"
|
|
||||||
|
|
||||||
// signatureNotVerified is the label for an entrypoint that performs
|
|
||||||
// no inbound verification.
|
|
||||||
const signatureNotVerified = "not verified"
|
|
||||||
|
|
||||||
// signatureMisconfigured is the label for a row holding one half of
|
|
||||||
// the scheme/secret pair. The receiver answers every request to such
|
|
||||||
// an entrypoint 500, so calling it "not verified" would describe a
|
|
||||||
// receiver that is refusing everything as one that is accepting
|
|
||||||
// everything. The form cannot create the state; a hand-edited
|
|
||||||
// database or a downgrade past a scheme can.
|
|
||||||
const signatureMisconfigured = "misconfigured"
|
|
||||||
|
|
||||||
// EntrypointView is the display-safe projection of an entrypoint for
|
|
||||||
// the UI. It deliberately has no secret field, so no template —
|
|
||||||
// present or future — can render the shared secret, in the same way
|
|
||||||
// delivery.TargetView keeps a target's stored credential away from
|
|
||||||
// one.
|
|
||||||
type EntrypointView struct {
|
|
||||||
ID string
|
|
||||||
Path string
|
|
||||||
Description string
|
|
||||||
Active bool
|
|
||||||
|
|
||||||
// Configured reports whether inbound requests to this entrypoint
|
|
||||||
// are verified.
|
|
||||||
Configured bool
|
|
||||||
|
|
||||||
// Scheme is the stored scheme, carried so the form can preselect
|
|
||||||
// it. It names an algorithm, not a secret.
|
|
||||||
Scheme database.SignatureScheme
|
|
||||||
|
|
||||||
// SchemeLabel and SchemeHeader describe the configured scheme for
|
|
||||||
// display: the sender's name, and the header its signature
|
|
||||||
// arrives in.
|
|
||||||
SchemeLabel string
|
|
||||||
SchemeHeader string
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewEntrypointViews projects entrypoints for rendering, dropping the
|
|
||||||
// shared secret on the way.
|
|
||||||
func NewEntrypointViews(
|
|
||||||
entrypoints []database.Entrypoint,
|
|
||||||
) []EntrypointView {
|
|
||||||
views := make([]EntrypointView, 0, len(entrypoints))
|
|
||||||
|
|
||||||
for i := range entrypoints {
|
|
||||||
e := &entrypoints[i]
|
|
||||||
|
|
||||||
view := EntrypointView{
|
|
||||||
ID: e.ID,
|
|
||||||
Path: e.Path,
|
|
||||||
Description: e.Description,
|
|
||||||
Active: e.Active,
|
|
||||||
Configured: e.SignatureConfigured(),
|
|
||||||
Scheme: e.SignatureScheme,
|
|
||||||
SchemeLabel: signatureNotVerified,
|
|
||||||
SchemeHeader: "",
|
|
||||||
}
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case view.Configured:
|
|
||||||
view.SchemeLabel = signatureUnavailable
|
|
||||||
|
|
||||||
info, ok := signature.Info(e.SignatureScheme)
|
|
||||||
if ok {
|
|
||||||
view.SchemeLabel = info.Label
|
|
||||||
view.SchemeHeader = info.Header
|
|
||||||
}
|
|
||||||
case e.SignatureHalfConfigured():
|
|
||||||
view.SchemeLabel = signatureMisconfigured
|
|
||||||
}
|
|
||||||
|
|
||||||
views = append(views, view)
|
|
||||||
}
|
|
||||||
|
|
||||||
return views
|
|
||||||
}
|
|
||||||
@@ -1,199 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"database/sql"
|
|
||||||
"errors"
|
|
||||||
"net/http"
|
|
||||||
"strconv"
|
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// eventBodyQuery reads one event's stored body as bytes. The cast
|
|
||||||
// to blob is what makes the driver hand back the stored bytes
|
|
||||||
// rather than a string conversion, so Content-Length taken from
|
|
||||||
// the result matches what goes on the wire. The soft-delete
|
|
||||||
// predicate is spelled out because Raw bypasses GORM's default
|
|
||||||
// scope, and it is what stops a reaped event still being
|
|
||||||
// downloadable.
|
|
||||||
const eventBodyQuery = "SELECT cast(body as blob) " +
|
|
||||||
"FROM events WHERE id = ? AND webhook_id = ? AND deleted_at IS NULL"
|
|
||||||
|
|
||||||
// HandleEventBodyDownload serves one event's stored body in
|
|
||||||
// full, which the event log page cannot: it caps each rendered
|
|
||||||
// body at maxRenderedBodyBytes.
|
|
||||||
//
|
|
||||||
// The bytes are attacker-supplied — anyone who can reach the
|
|
||||||
// public receiver chooses them — and this route hands them back
|
|
||||||
// inside the operator's own authenticated origin, so the
|
|
||||||
// response is deliberately not renderable. Content-Disposition
|
|
||||||
// makes the browser download rather than display it, and the
|
|
||||||
// octet-stream type plus nosniff stop it being interpreted as
|
|
||||||
// HTML or script. Without those a stored payload would execute
|
|
||||||
// as the logged-in operator. The application's CSP does not
|
|
||||||
// help here: script-src allows 'unsafe-inline' from 'self', so
|
|
||||||
// a document served from this origin could run its own inline
|
|
||||||
// script.
|
|
||||||
func (h *Handlers) HandleEventBodyDownload() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
webhook, ok := h.ownedWebhook(w, r)
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parsing the id before use serves two purposes: a
|
|
||||||
// malformed id can never reach the SQL or the response
|
|
||||||
// header, and the canonical form below is drawn from
|
|
||||||
// uuid's own fixed alphabet rather than from the
|
|
||||||
// request, so the Content-Disposition value cannot be
|
|
||||||
// steered by a client.
|
|
||||||
eventID, err := uuid.Parse(chi.URLParam(r, "eventID"))
|
|
||||||
if err != nil {
|
|
||||||
http.NotFound(w, r)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.serveEventBody(w, r, webhook, eventID.String())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// serveEventBody writes the named event's stored body to w.
|
|
||||||
//
|
|
||||||
// The event must belong to webhook, which is what keeps this
|
|
||||||
// route from reading any event in the system by id alone. Two
|
|
||||||
// things enforce that and they are not equally strong. The
|
|
||||||
// operative one is that events live in a per-webhook SQLite
|
|
||||||
// file, so a sibling webhook's event is not in the database
|
|
||||||
// being queried at all. The webhook_id predicate on the query
|
|
||||||
// below is the second guard, and it is currently redundant
|
|
||||||
// against that isolation; it is there so the scoping survives
|
|
||||||
// any future change that puts more than one webhook's events in
|
|
||||||
// one file.
|
|
||||||
//
|
|
||||||
// The body is read in one query and held whole in memory while
|
|
||||||
// it is written. That costs roughly two body-sized allocations
|
|
||||||
// per concurrent download, not one: the driver's column buffer
|
|
||||||
// and the copy database/sql makes in convertAssign when a
|
|
||||||
// []byte column is scanned into a *[]byte are live at the same
|
|
||||||
// time. Measured allocation is ~2x the body plus ~45 KB, so at
|
|
||||||
// the 1 MB ingest cap a download costs ~2 MB of Go heap. On
|
|
||||||
// top of that, SQLite's own materialisation of the column
|
|
||||||
// value sits in the driver's allocator outside the Go heap, so
|
|
||||||
// process peak is higher again: 2x is a floor, not a ceiling.
|
|
||||||
// There is no cheaper bound available — database/sql exposes
|
|
||||||
// no incremental handle on a SQLite BLOB, and reading byte
|
|
||||||
// ranges with substr does not avoid the cost either, because
|
|
||||||
// SQLite materialises the whole column value to evaluate each
|
|
||||||
// substr call. Range reads only pay for that materialisation
|
|
||||||
// once per range.
|
|
||||||
//
|
|
||||||
// One consequence is worth keeping in view: the read finishes
|
|
||||||
// before the client is written to, so no read lock is held for
|
|
||||||
// the length of a slow download. These per-webhook databases
|
|
||||||
// run in SQLite's default journal mode rather than WAL, so a
|
|
||||||
// lock held that long would block the receiver from recording
|
|
||||||
// new events.
|
|
||||||
func (h *Handlers) serveEventBody(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
r *http.Request,
|
|
||||||
webhook database.Webhook,
|
|
||||||
eventID string,
|
|
||||||
) {
|
|
||||||
if !h.dbMgr.DBExists(webhook.ID) {
|
|
||||||
http.NotFound(w, r)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, "failed to get webhook database", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
body, found, err := eventBody(webhookDB, webhook.ID, eventID)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, "failed to read event body", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// A miss is a 404 whether the event belongs to another
|
|
||||||
// webhook or does not exist at all, so the response does
|
|
||||||
// not report which. Reading the body before any header is
|
|
||||||
// written is also what keeps an event reaped mid-request
|
|
||||||
// from producing a torn response: either the read finds the
|
|
||||||
// row and the whole body is served, or it does not and the
|
|
||||||
// response is a clean 404.
|
|
||||||
if !found {
|
|
||||||
http.NotFound(w, r)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
setEventBodyHeaders(w, eventID, int64(len(body)))
|
|
||||||
|
|
||||||
_, err = w.Write(body)
|
|
||||||
if err != nil {
|
|
||||||
// The status and Content-Length are already committed,
|
|
||||||
// so the client sees a short download. There is no way
|
|
||||||
// to report a 500 from here; the log is the record.
|
|
||||||
h.log.Error(
|
|
||||||
"failed to write event body",
|
|
||||||
"webhook_id", webhook.ID,
|
|
||||||
"event_id", eventID,
|
|
||||||
"error", err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// eventBody returns an event's stored body and whether the event
|
|
||||||
// exists within the webhook.
|
|
||||||
func eventBody(
|
|
||||||
webhookDB *gorm.DB,
|
|
||||||
webhookID, eventID string,
|
|
||||||
) ([]byte, bool, error) {
|
|
||||||
var body []byte
|
|
||||||
|
|
||||||
err := webhookDB.Raw(
|
|
||||||
eventBodyQuery, eventID, webhookID,
|
|
||||||
).Row().Scan(&body)
|
|
||||||
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
|
||||||
return nil, false, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return nil, false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return body, true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// setEventBodyHeaders applies the response headers that make
|
|
||||||
// this route safe to hand attacker-supplied bytes through. See
|
|
||||||
// HandleEventBodyDownload for why they are a security control
|
|
||||||
// and not a formatting choice.
|
|
||||||
//
|
|
||||||
// nosniff is also set by the global SecurityHeaders middleware.
|
|
||||||
// It is repeated here so the guarantee belongs to the route
|
|
||||||
// that needs it rather than to a middleware someone could
|
|
||||||
// reorder or scope away.
|
|
||||||
func setEventBodyHeaders(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
eventID string,
|
|
||||||
size int64,
|
|
||||||
) {
|
|
||||||
w.Header().Set("Content-Type", "application/octet-stream")
|
|
||||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
|
||||||
w.Header().Set(
|
|
||||||
"Content-Disposition",
|
|
||||||
`attachment; filename="webhooker-event-`+eventID+`.bin"`,
|
|
||||||
)
|
|
||||||
w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
|
|
||||||
}
|
|
||||||
@@ -1,506 +0,0 @@
|
|||||||
package handlers_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/url"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/gorm/clause"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
|
||||||
)
|
|
||||||
|
|
||||||
// paramEventID is the chi URL parameter the body download
|
|
||||||
// handler reads.
|
|
||||||
const paramEventID = "eventID"
|
|
||||||
|
|
||||||
// otherTestUserID owns webhooks the session user must not be
|
|
||||||
// able to read.
|
|
||||||
const otherTestUserID = "other-user-id"
|
|
||||||
|
|
||||||
// seedWebhookFor inserts a webhook owned by the given user.
|
|
||||||
func seedWebhookFor(
|
|
||||||
t *testing.T,
|
|
||||||
db *database.Database,
|
|
||||||
userID string,
|
|
||||||
) *database.Webhook {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
wh := &database.Webhook{
|
|
||||||
UserID: userID,
|
|
||||||
Name: "wh-" + userID,
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t,
|
|
||||||
db.DB().Omit(clause.Associations).Create(wh).Error,
|
|
||||||
)
|
|
||||||
|
|
||||||
return wh
|
|
||||||
}
|
|
||||||
|
|
||||||
// fetchEventBody runs the real download handler as the test user
|
|
||||||
// for the given source and event ids.
|
|
||||||
func fetchEventBody(
|
|
||||||
t *testing.T,
|
|
||||||
h *handlers.Handlers,
|
|
||||||
sess *session.Session,
|
|
||||||
sourceID, eventID string,
|
|
||||||
) *httptest.ResponseRecorder {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
// The path is escaped and the raw id goes in the route
|
|
||||||
// context, which is what chi hands a handler: the param is
|
|
||||||
// already percent-decoded by the time it is read.
|
|
||||||
req := httptest.NewRequestWithContext(
|
|
||||||
context.Background(),
|
|
||||||
http.MethodGet,
|
|
||||||
"/source/"+url.PathEscape(sourceID)+
|
|
||||||
"/logs/"+url.PathEscape(eventID)+"/body",
|
|
||||||
nil,
|
|
||||||
)
|
|
||||||
|
|
||||||
for _, c := range authenticatedCookies(
|
|
||||||
t, sess, deleteTestUserID, deleteTestUsername,
|
|
||||||
) {
|
|
||||||
req.AddCookie(c)
|
|
||||||
}
|
|
||||||
|
|
||||||
rctx := chi.NewRouteContext()
|
|
||||||
rctx.URLParams.Add(paramSourceID, sourceID)
|
|
||||||
rctx.URLParams.Add(paramEventID, eventID)
|
|
||||||
|
|
||||||
req = req.WithContext(
|
|
||||||
context.WithValue(
|
|
||||||
req.Context(), chi.RouteCtxKey, rctx,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
h.HandleEventBodyDownload().ServeHTTP(w, req)
|
|
||||||
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleEventBodyDownload_ServesOversizeBodyInFull is the
|
|
||||||
// capability the render cap took away: a body far above what the
|
|
||||||
// event log page will show comes back whole and byte-identical,
|
|
||||||
// with the headers that keep it from being rendered.
|
|
||||||
func TestHandleEventBodyDownload_ServesOversizeBodyInFull(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
// Far above the render cap, with multibyte runes and a
|
|
||||||
// distinctive tail, so a body that the log page can only
|
|
||||||
// show a slice of comes back whole and in order.
|
|
||||||
const sentinel = "TAIL-SENTINEL-1f4a9c"
|
|
||||||
|
|
||||||
stored := strings.Repeat("A", 200*1024) +
|
|
||||||
strings.Repeat(snowman, 1000) + sentinel
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
evt := seedEventWithBody(t, dbMgr, wh.ID, stored)
|
|
||||||
|
|
||||||
w := fetchEventBody(t, h, sess, wh.ID, evt.ID)
|
|
||||||
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
assert.Greater(t, len(stored), bodyCap)
|
|
||||||
assert.Equal(t, stored, w.Body.String())
|
|
||||||
assert.Equal(
|
|
||||||
t, strconv.Itoa(len(stored)),
|
|
||||||
w.Header().Get("Content-Length"),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleEventBodyDownload_BodiesRoundTripByteIdentical
|
|
||||||
// covers the sizes and byte values a stored body can actually
|
|
||||||
// take: empty, one byte, either side of the render cap, and
|
|
||||||
// bytes that are not text at all. Content-Length has to equal
|
|
||||||
// the bytes written in every case, since it is derived from the
|
|
||||||
// same read that produces them.
|
|
||||||
func TestHandleEventBodyDownload_BodiesRoundTripByteIdentical(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// A NUL, invalid UTF-8 and a multibyte rune, so nothing on
|
|
||||||
// the path can be treating the body as text.
|
|
||||||
binary := "\x00\x01\xff\xfe" + snowman + "\x00tail"
|
|
||||||
|
|
||||||
cases := map[string]string{
|
|
||||||
"empty": "",
|
|
||||||
"single byte": "x",
|
|
||||||
"one below cap": strings.Repeat("b", bodyCap-1),
|
|
||||||
"exactly cap": strings.Repeat("c", bodyCap),
|
|
||||||
"one above cap": strings.Repeat("d", bodyCap+1),
|
|
||||||
"binary": binary,
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, stored := range cases {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
evt := seedEventWithBody(t, dbMgr, wh.ID, stored)
|
|
||||||
|
|
||||||
w := fetchEventBody(t, h, sess, wh.ID, evt.ID)
|
|
||||||
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
assert.Equal(t, stored, w.Body.String())
|
|
||||||
assert.Equal(
|
|
||||||
t, strconv.Itoa(len(stored)),
|
|
||||||
w.Header().Get("Content-Length"),
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t, len(stored), w.Body.Len(),
|
|
||||||
"Content-Length must equal bytes written",
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleEventBodyDownload_HeadersAreNotRenderable pins the
|
|
||||||
// response headers that stop attacker-supplied bytes executing
|
|
||||||
// in the operator's own origin. They are a security control, not
|
|
||||||
// presentation.
|
|
||||||
func TestHandleEventBodyDownload_HeadersAreNotRenderable(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
evt := seedEventWithBody(t, dbMgr, wh.ID, `{"small":true}`)
|
|
||||||
|
|
||||||
w := fetchEventBody(t, h, sess, wh.ID, evt.ID)
|
|
||||||
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
assert.Equal(
|
|
||||||
t, "application/octet-stream",
|
|
||||||
w.Header().Get("Content-Type"),
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t, "nosniff",
|
|
||||||
w.Header().Get("X-Content-Type-Options"),
|
|
||||||
)
|
|
||||||
|
|
||||||
disposition := w.Header().Get("Content-Disposition")
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
`attachment; filename="webhooker-event-`+evt.ID+`.bin"`,
|
|
||||||
disposition,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleEventBodyDownload_ScriptBodyStaysInert proves a
|
|
||||||
// stored HTML payload is handed back as an attachment of opaque
|
|
||||||
// bytes rather than as anything a browser will execute. The
|
|
||||||
// bytes themselves are unaltered: this route reports what was
|
|
||||||
// delivered.
|
|
||||||
func TestHandleEventBodyDownload_ScriptBodyStaysInert(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
const payload = `<html><script>alert(document.cookie)` +
|
|
||||||
`</script></html>`
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
evt := seedEventWithBody(t, dbMgr, wh.ID, payload)
|
|
||||||
|
|
||||||
w := fetchEventBody(t, h, sess, wh.ID, evt.ID)
|
|
||||||
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
assert.Equal(t, payload, w.Body.String())
|
|
||||||
|
|
||||||
contentType := w.Header().Get("Content-Type")
|
|
||||||
assert.Equal(t, "application/octet-stream", contentType)
|
|
||||||
assert.NotContains(t, contentType, "html")
|
|
||||||
assert.NotContains(t, contentType, "xml")
|
|
||||||
assert.NotContains(t, contentType, "javascript")
|
|
||||||
assert.Contains(
|
|
||||||
t, w.Header().Get("Content-Disposition"), "attachment",
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t, "nosniff",
|
|
||||||
w.Header().Get("X-Content-Type-Options"),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleEventBodyDownload_OtherUsersEvent404s is the
|
|
||||||
// authorization test the definition of done asks for: an event
|
|
||||||
// stored under a webhook the session user does not own is not
|
|
||||||
// readable, and the miss does not distinguish itself from a
|
|
||||||
// nonexistent one.
|
|
||||||
func TestHandleEventBodyDownload_OtherUsersEvent404s(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
const theirPayload = "OTHER-USERS-PAYLOAD-8b1d"
|
|
||||||
|
|
||||||
theirs := seedWebhookFor(t, db, otherTestUserID)
|
|
||||||
evt := seedEventWithBody(t, dbMgr, theirs.ID, theirPayload)
|
|
||||||
|
|
||||||
w := fetchEventBody(t, h, sess, theirs.ID, evt.ID)
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusNotFound, w.Code)
|
|
||||||
assert.NotContains(t, w.Body.String(), theirPayload)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleEventBodyDownload_EventOfAnotherWebhook404s pins
|
|
||||||
// that holding a valid event id is not enough: the event has to
|
|
||||||
// belong to the webhook in the path. Both webhooks here are the
|
|
||||||
// session user's and both have event databases, so the
|
|
||||||
// ownership check cannot be what produces the 404.
|
|
||||||
//
|
|
||||||
// What does produce it is the per-webhook database file rather
|
|
||||||
// than the webhook_id predicate on the query — removing that
|
|
||||||
// predicate leaves this test green, because the sibling's event
|
|
||||||
// is in a different file. The test is kept as the behavioural
|
|
||||||
// guard the route owes; see serveEventBody for which mechanism
|
|
||||||
// is load-bearing.
|
|
||||||
func TestHandleEventBodyDownload_EventOfAnotherWebhook404s(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
const other = "BELONGS-TO-THE-OTHER-WEBHOOK-3c7e"
|
|
||||||
|
|
||||||
mine := seedWebhook(t, db)
|
|
||||||
seedEventWithBody(t, dbMgr, mine.ID, `{"mine":true}`)
|
|
||||||
|
|
||||||
sibling := seedWebhook(t, db)
|
|
||||||
evt := seedEventWithBody(t, dbMgr, sibling.ID, other)
|
|
||||||
|
|
||||||
w := fetchEventBody(t, h, sess, mine.ID, evt.ID)
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusNotFound, w.Code)
|
|
||||||
assert.NotContains(t, w.Body.String(), other)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleEventBodyDownload_UnknownEvent404s covers the plain
|
|
||||||
// miss, including an id that is not a uuid at all and so never
|
|
||||||
// reaches the query or the response header.
|
|
||||||
func TestHandleEventBodyDownload_UnknownEvent404s(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
seedEventWithBody(t, dbMgr, wh.ID, `{"mine":true}`)
|
|
||||||
|
|
||||||
for _, id := range []string{
|
|
||||||
uuid.New().String(),
|
|
||||||
`../../etc/passwd`,
|
|
||||||
"not-a-uuid",
|
|
||||||
`x"; rm -rf /`,
|
|
||||||
} {
|
|
||||||
w := fetchEventBody(t, h, sess, wh.ID, id)
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t, http.StatusNotFound, w.Code,
|
|
||||||
"event id %q", id,
|
|
||||||
)
|
|
||||||
assert.Empty(
|
|
||||||
t, w.Header().Get("Content-Disposition"),
|
|
||||||
"event id %q must not reach a header", id,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleEventBodyDownload_ReapedEvent404s pins what happens
|
|
||||||
// when the retention reaper takes an event out from under this
|
|
||||||
// route. The body is read in one query before any header is
|
|
||||||
// written, so a reaped event cannot produce a partial download:
|
|
||||||
// it is a clean 404 with no Content-Length and no
|
|
||||||
// Content-Disposition. Both removals the codebase performs are
|
|
||||||
// covered — the reaper hard-deletes, and a soft-deleted row is
|
|
||||||
// excluded by the query's own deleted_at predicate rather than
|
|
||||||
// by GORM's default scope, which Raw bypasses.
|
|
||||||
func TestHandleEventBodyDownload_ReapedEvent404s(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for name, hard := range map[string]bool{
|
|
||||||
"soft deleted": false,
|
|
||||||
"hard deleted": true,
|
|
||||||
} {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
const payload = "REAPED-PAYLOAD-4d2a"
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
evt := seedEventWithBody(t, dbMgr, wh.ID, payload)
|
|
||||||
|
|
||||||
webhookDB, err := dbMgr.GetDB(wh.ID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
del := webhookDB
|
|
||||||
if hard {
|
|
||||||
del = del.Unscoped()
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t,
|
|
||||||
del.Delete(&database.Event{}, "id = ?", evt.ID).
|
|
||||||
Error,
|
|
||||||
)
|
|
||||||
|
|
||||||
w := fetchEventBody(t, h, sess, wh.ID, evt.ID)
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusNotFound, w.Code)
|
|
||||||
assert.NotContains(t, w.Body.String(), payload)
|
|
||||||
assert.Empty(t, w.Header().Get("Content-Length"))
|
|
||||||
assert.Empty(
|
|
||||||
t, w.Header().Get("Content-Disposition"),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceLogs_TruncationMarkerLinksToDownload proves
|
|
||||||
// the page tells the reader where the rest of the body is, and
|
|
||||||
// only when there is a rest to fetch.
|
|
||||||
func TestHandleSourceLogs_TruncationMarkerLinksToDownload(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
big := seedWebhook(t, db)
|
|
||||||
bigEvt := seedEventWithBody(
|
|
||||||
t, dbMgr, big.ID, strings.Repeat("A", 4*bodyCap),
|
|
||||||
)
|
|
||||||
|
|
||||||
page := renderSourceLogsPage(t, h, sess, big.ID)
|
|
||||||
assert.Contains(
|
|
||||||
t, page,
|
|
||||||
"/source/"+big.ID+"/logs/"+bigEvt.ID+"/body",
|
|
||||||
)
|
|
||||||
|
|
||||||
small := seedWebhook(t, db)
|
|
||||||
smallEvt := seedEventWithBody(
|
|
||||||
t, dbMgr, small.ID, `{"kept":"whole"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
page = renderSourceLogsPage(t, h, sess, small.ID)
|
|
||||||
assert.NotContains(
|
|
||||||
t, page,
|
|
||||||
"/source/"+small.ID+"/logs/"+smallEvt.ID+"/body",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -1,120 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"time"
|
|
||||||
"unicode/utf8"
|
|
||||||
)
|
|
||||||
|
|
||||||
// maxRenderedBodyBytes caps how many bytes of a stored event
|
|
||||||
// body reach the event log page. Bodies come from the
|
|
||||||
// unauthenticated receiver under the 1 MB ingest cap and
|
|
||||||
// renderTemplate buffers a whole page before writing it, so
|
|
||||||
// an uncapped page of paginationPerPage events is tens of
|
|
||||||
// megabytes of resident memory per concurrent viewer.
|
|
||||||
const maxRenderedBodyBytes = 8192
|
|
||||||
|
|
||||||
// eventLogColumns is the event log's projection. The casts to
|
|
||||||
// blob are load-bearing: they make substr and length count
|
|
||||||
// bytes rather than characters, so the cap bounds the page in
|
|
||||||
// bytes whatever the payload's encoding. Cutting in SQLite
|
|
||||||
// rather than in Go is the point of the projection — an
|
|
||||||
// oversized body never becomes a Go string at all.
|
|
||||||
const eventLogColumns = "id, created_at, method, content_type, " +
|
|
||||||
"substr(cast(body as blob), 1, ?) AS body, " +
|
|
||||||
"length(cast(body as blob)) AS body_bytes"
|
|
||||||
|
|
||||||
// EventLogView is the display-safe projection of an event for
|
|
||||||
// the event log page, alongside DeliveryView and TargetView.
|
|
||||||
// It carries a capped body plus the true stored size, so the
|
|
||||||
// page can mark a body as truncated without ever holding the
|
|
||||||
// whole thing.
|
|
||||||
type EventLogView struct {
|
|
||||||
ID string
|
|
||||||
CreatedAt time.Time
|
|
||||||
Method string
|
|
||||||
ContentType string
|
|
||||||
|
|
||||||
// Body holds at most maxRenderedBodyBytes bytes of the
|
|
||||||
// stored body.
|
|
||||||
Body string
|
|
||||||
|
|
||||||
// BodyBytes is the true size of the stored body.
|
|
||||||
BodyBytes int64
|
|
||||||
|
|
||||||
// BodyTruncated reports that the stored body was larger
|
|
||||||
// than the cap, so the page owes the reader a marker.
|
|
||||||
BodyTruncated bool
|
|
||||||
|
|
||||||
Deliveries []DeliveryView
|
|
||||||
}
|
|
||||||
|
|
||||||
// BodyShownBytes is how many body bytes the page is actually
|
|
||||||
// rendering, which the truncation marker reports beside the
|
|
||||||
// true size.
|
|
||||||
func (v EventLogView) BodyShownBytes() int {
|
|
||||||
return len(v.Body)
|
|
||||||
}
|
|
||||||
|
|
||||||
// eventLogRow is one row of the event log projection. Its
|
|
||||||
// body column arrives already cut to the cap by SQLite, with
|
|
||||||
// the true size beside it.
|
|
||||||
type eventLogRow struct {
|
|
||||||
ID string
|
|
||||||
CreatedAt time.Time
|
|
||||||
Method string
|
|
||||||
ContentType string
|
|
||||||
Body []byte
|
|
||||||
BodyBytes int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// view projects a loaded row for rendering.
|
|
||||||
func (r *eventLogRow) view() EventLogView {
|
|
||||||
body := r.Body
|
|
||||||
truncated := r.BodyBytes > int64(len(body))
|
|
||||||
|
|
||||||
// Only a cut body can have been left mid-sequence by
|
|
||||||
// this query. A whole body is passed through exactly as
|
|
||||||
// stored, however malformed.
|
|
||||||
if truncated {
|
|
||||||
body = trimPartialRune(body)
|
|
||||||
}
|
|
||||||
|
|
||||||
return EventLogView{
|
|
||||||
ID: r.ID,
|
|
||||||
CreatedAt: r.CreatedAt,
|
|
||||||
Method: r.Method,
|
|
||||||
ContentType: r.ContentType,
|
|
||||||
Body: string(body),
|
|
||||||
BodyBytes: r.BodyBytes,
|
|
||||||
BodyTruncated: truncated,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// trimPartialRune drops a trailing UTF-8 sequence that the
|
|
||||||
// byte-wise cut left incomplete, so a multi-byte rune severed
|
|
||||||
// at the cap does not surface as a mojibake tail.
|
|
||||||
//
|
|
||||||
// Bytes that are merely invalid UTF-8 are left exactly as
|
|
||||||
// stored: this service receives binary payloads, and rewriting
|
|
||||||
// them would misreport what was delivered. The distinction is
|
|
||||||
// utf8.FullRune's — it reports a complete sequence for an
|
|
||||||
// invalid encoding too, since that decodes to a width-1 error
|
|
||||||
// rune, so only a valid prefix still waiting for its
|
|
||||||
// continuation bytes is removed. A tail with no rune start in
|
|
||||||
// its last utf8.UTFMax bytes cannot be an incomplete sequence
|
|
||||||
// either, and is likewise left alone.
|
|
||||||
func trimPartialRune(b []byte) []byte {
|
|
||||||
for i := len(b) - 1; i >= 0 && len(b)-i <= utf8.UTFMax; i-- {
|
|
||||||
if !utf8.RuneStart(b[i]) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if utf8.FullRune(b[i:]) {
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
|
|
||||||
return b[:i]
|
|
||||||
}
|
|
||||||
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
@@ -1,261 +0,0 @@
|
|||||||
package handlers_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"unicode/utf8"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/gorm/clause"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
|
||||||
)
|
|
||||||
|
|
||||||
// bodyCap is the number of body bytes the event log page is
|
|
||||||
// allowed to render for one event.
|
|
||||||
const bodyCap = handlers.MaxRenderedBodyBytesForTest
|
|
||||||
|
|
||||||
// snowman is a three-byte rune, so a body of them straddles the
|
|
||||||
// byte-wise cut: bodyCap is not a multiple of three.
|
|
||||||
const snowman = "☃"
|
|
||||||
|
|
||||||
// seedEventWithBody records one event with the given body in the
|
|
||||||
// webhook's own database and returns it, so a caller that needs
|
|
||||||
// the generated event id can have it.
|
|
||||||
func seedEventWithBody(
|
|
||||||
t *testing.T,
|
|
||||||
dbMgr *database.WebhookDBManager,
|
|
||||||
webhookID string,
|
|
||||||
body string,
|
|
||||||
) *database.Event {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
webhookDB, err := dbMgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
event := &database.Event{
|
|
||||||
WebhookID: webhookID,
|
|
||||||
Method: http.MethodPost,
|
|
||||||
Body: body,
|
|
||||||
ContentType: "application/octet-stream",
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, webhookDB.Omit(
|
|
||||||
clause.Associations,
|
|
||||||
).Create(event).Error)
|
|
||||||
|
|
||||||
return event
|
|
||||||
}
|
|
||||||
|
|
||||||
// seedAndProject stores one body and returns the projection the
|
|
||||||
// event log page would be handed for it.
|
|
||||||
func seedAndProject(
|
|
||||||
t *testing.T,
|
|
||||||
body string,
|
|
||||||
) handlers.EventLogView {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
seedEventWithBody(t, dbMgr, wh.ID, body)
|
|
||||||
|
|
||||||
views := h.LoadEventLogViewsForTest(
|
|
||||||
httptest.NewRecorder(), *wh, 1,
|
|
||||||
)
|
|
||||||
require.Len(t, views, 1)
|
|
||||||
|
|
||||||
return views[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceLogs_BoundsOversizeBody proves the rendered
|
|
||||||
// page is bounded by the cap rather than by the stored payload:
|
|
||||||
// the body here is 64 times the cap, and the ingest path would
|
|
||||||
// accept twice as much again.
|
|
||||||
func TestHandleSourceLogs_BoundsOversizeBody(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
const (
|
|
||||||
sentinel = "TAIL-SENTINEL-1f4a9c"
|
|
||||||
storedBytes = 512 * 1024
|
|
||||||
)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
seedEventWithBody(
|
|
||||||
t, dbMgr, wh.ID,
|
|
||||||
strings.Repeat("A", storedBytes-len(sentinel))+sentinel,
|
|
||||||
)
|
|
||||||
|
|
||||||
page := renderSourceLogsPage(t, h, sess, wh.ID)
|
|
||||||
|
|
||||||
// Nothing past the cap reaches the page, and the whole page
|
|
||||||
// stays far below the stored body it is reporting on.
|
|
||||||
assert.NotContains(t, page, sentinel)
|
|
||||||
assert.Less(t, len(page), 4*bodyCap)
|
|
||||||
|
|
||||||
// The marker states the true stored size, not the cut one.
|
|
||||||
assert.Contains(
|
|
||||||
t, page,
|
|
||||||
"showing "+strconv.Itoa(bodyCap)+
|
|
||||||
" of "+strconv.Itoa(storedBytes)+" bytes",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceLogs_SmallBodyRendersWhole guards the other
|
|
||||||
// side of the cap: a body under it is shown in full and carries
|
|
||||||
// no truncation marker.
|
|
||||||
func TestHandleSourceLogs_SmallBodyRendersWhole(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
seedEventWithBody(t, dbMgr, wh.ID, `{"kept":"whole"}`)
|
|
||||||
|
|
||||||
page := renderSourceLogsPage(t, h, sess, wh.ID)
|
|
||||||
|
|
||||||
assert.Contains(t, page, ""kept"")
|
|
||||||
assert.NotContains(t, page, "Body truncated for display")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventLogView_CutMidRune proves a multi-byte rune severed
|
|
||||||
// by the byte-wise cut is dropped rather than surfaced as a
|
|
||||||
// mojibake tail.
|
|
||||||
func TestEventLogView_CutMidRune(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
body := strings.Repeat(snowman, 4096)
|
|
||||||
view := seedAndProject(t, body)
|
|
||||||
|
|
||||||
// bodyCap bytes hold bodyCap/3 whole snowmen and two bytes
|
|
||||||
// of the next one; those two are dropped.
|
|
||||||
whole := bodyCap / len(snowman)
|
|
||||||
|
|
||||||
assert.True(t, view.BodyTruncated)
|
|
||||||
assert.Equal(t, int64(len(body)), view.BodyBytes)
|
|
||||||
assert.Equal(t, strings.Repeat(snowman, whole), view.Body)
|
|
||||||
assert.True(t, utf8.ValidString(view.Body))
|
|
||||||
assert.LessOrEqual(t, len(view.Body), bodyCap)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventLogView_BinaryBodyLeftAsStored proves a binary
|
|
||||||
// payload is passed through byte for byte. Its tail is invalid
|
|
||||||
// UTF-8 however the cut falls, so repairing it would misreport
|
|
||||||
// what the sender delivered.
|
|
||||||
func TestEventLogView_BinaryBodyLeftAsStored(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
raw := make([]byte, bodyCap+808)
|
|
||||||
for i := range raw {
|
|
||||||
// 0x80..0xBF: continuation bytes, never a rune start.
|
|
||||||
raw[i] = 0x80 | byte(i%0x40)
|
|
||||||
}
|
|
||||||
|
|
||||||
view := seedAndProject(t, string(raw))
|
|
||||||
|
|
||||||
assert.True(t, view.BodyTruncated)
|
|
||||||
assert.Equal(t, int64(len(raw)), view.BodyBytes)
|
|
||||||
assert.Equal(t, string(raw[:bodyCap]), view.Body)
|
|
||||||
assert.False(t, utf8.ValidString(view.Body))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTrimPartialRune covers the distinction the cut repair
|
|
||||||
// turns on: an incomplete but valid sequence is dropped, while
|
|
||||||
// bytes that are merely invalid UTF-8 are left alone.
|
|
||||||
func TestTrimPartialRune(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
in []byte
|
|
||||||
want []byte
|
|
||||||
}{{
|
|
||||||
name: "complete ascii",
|
|
||||||
in: []byte("abc"),
|
|
||||||
want: []byte("abc"),
|
|
||||||
}, {
|
|
||||||
name: "complete multibyte",
|
|
||||||
in: []byte("ab" + snowman),
|
|
||||||
want: []byte("ab" + snowman),
|
|
||||||
}, {
|
|
||||||
name: "two byte rune cut",
|
|
||||||
in: []byte{'a', 0xC3},
|
|
||||||
want: []byte{'a'},
|
|
||||||
}, {
|
|
||||||
name: "three byte rune cut after one",
|
|
||||||
in: []byte{'a', 0xE2},
|
|
||||||
want: []byte{'a'},
|
|
||||||
}, {
|
|
||||||
name: "three byte rune cut after two",
|
|
||||||
in: []byte{'a', 0xE2, 0x98},
|
|
||||||
want: []byte{'a'},
|
|
||||||
}, {
|
|
||||||
name: "four byte rune cut",
|
|
||||||
in: []byte{'a', 0xF0, 0x9F, 0x92}, // U+1F4A9 cut
|
|
||||||
want: []byte{'a'},
|
|
||||||
}, {
|
|
||||||
name: "invalid start byte kept",
|
|
||||||
in: []byte{'a', 0xFF},
|
|
||||||
want: []byte{'a', 0xFF},
|
|
||||||
}, {
|
|
||||||
name: "orphan continuation bytes kept",
|
|
||||||
in: []byte{0x80, 0x81, 0x82, 0x83, 0x84},
|
|
||||||
want: []byte{0x80, 0x81, 0x82, 0x83, 0x84},
|
|
||||||
}, {
|
|
||||||
name: "truncated sequence followed by junk kept",
|
|
||||||
in: []byte{0xE2, 0x98, 0xFF},
|
|
||||||
want: []byte{0xE2, 0x98, 0xFF},
|
|
||||||
}, {
|
|
||||||
name: "empty",
|
|
||||||
in: []byte{},
|
|
||||||
want: []byte{},
|
|
||||||
}}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t, tc.want,
|
|
||||||
handlers.TrimPartialRuneForTest(tc.in),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,72 +1,6 @@
|
|||||||
package handlers
|
package handlers
|
||||||
|
|
||||||
import (
|
import "net/http"
|
||||||
"html/template"
|
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// SetLogForTest replaces the handler's logger, so the handlers_test
|
|
||||||
// package can assert on what a log line actually contains rather than
|
|
||||||
// on what it is meant to contain.
|
|
||||||
func (s *Handlers) SetLogForTest(log *slog.Logger) {
|
|
||||||
s.log = log
|
|
||||||
}
|
|
||||||
|
|
||||||
// MaxRenderedBodyBytesForTest exposes the event log's body cap
|
|
||||||
// to the handlers_test package.
|
|
||||||
const MaxRenderedBodyBytesForTest = maxRenderedBodyBytes
|
|
||||||
|
|
||||||
// MaxRenderedResponseBytesForTest exposes the event log's
|
|
||||||
// delivery response cap to the handlers_test package.
|
|
||||||
const MaxRenderedResponseBytesForTest = maxRenderedResponseBytes
|
|
||||||
|
|
||||||
// MaxRenderedAttemptsForTest exposes the event log's
|
|
||||||
// per-delivery attempt ceiling to the handlers_test package.
|
|
||||||
const MaxRenderedAttemptsForTest = maxRenderedAttempts
|
|
||||||
|
|
||||||
// DummyVerificationsForTest reports how many equivalent-cost
|
|
||||||
// verifications were charged for usernames that do not exist. It
|
|
||||||
// lets a test prove the anti-enumeration path ran without timing
|
|
||||||
// anything.
|
|
||||||
func (s *Handlers) DummyVerificationsForTest() uint64 {
|
|
||||||
return s.dummyVerifications.Load()
|
|
||||||
}
|
|
||||||
|
|
||||||
// TrimPartialRuneForTest exposes trimPartialRune for use in the
|
|
||||||
// handlers_test package.
|
|
||||||
func TrimPartialRuneForTest(b []byte) []byte {
|
|
||||||
return trimPartialRune(b)
|
|
||||||
}
|
|
||||||
|
|
||||||
// LoadEventLogViewsForTest exposes loadEventsWithDeliveries for
|
|
||||||
// use in the handlers_test package. Assertions on the projected
|
|
||||||
// body need the bytes as loaded: html/template rewrites invalid
|
|
||||||
// UTF-8 on the way out, so the rendered page cannot show whether
|
|
||||||
// a binary body survived the projection intact.
|
|
||||||
func (s *Handlers) LoadEventLogViewsForTest(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
webhook database.Webhook,
|
|
||||||
page int,
|
|
||||||
) []EventLogView {
|
|
||||||
views, _, _ := s.loadEventsWithDeliveries(
|
|
||||||
w, webhook, nil, page,
|
|
||||||
)
|
|
||||||
|
|
||||||
return views
|
|
||||||
}
|
|
||||||
|
|
||||||
// AddTemplateForTest registers a template under a page name so that
|
|
||||||
// the handlers_test package can drive the render path with a
|
|
||||||
// template of its own.
|
|
||||||
func (s *Handlers) AddTemplateForTest(
|
|
||||||
pageTemplate string,
|
|
||||||
tmpl *template.Template,
|
|
||||||
) {
|
|
||||||
s.templates[pageTemplate] = tmpl
|
|
||||||
}
|
|
||||||
|
|
||||||
// RenderTemplateForTest exposes renderTemplate for use in the
|
// RenderTemplateForTest exposes renderTemplate for use in the
|
||||||
// handlers_test package.
|
// handlers_test package.
|
||||||
@@ -79,29 +13,18 @@ func (s *Handlers) RenderTemplateForTest(
|
|||||||
s.renderTemplate(w, r, pageTemplate, data)
|
s.renderTemplate(w, r, pageTemplate, data)
|
||||||
}
|
}
|
||||||
|
|
||||||
// BuildSlackTargetConfigForTest exposes
|
// BuildSlackTargetConfigForTest exposes buildURLTargetConfig
|
||||||
// buildSlackTargetConfig for use in the handlers_test package.
|
// with the Slack target parameters for use in the
|
||||||
|
// handlers_test package.
|
||||||
func (s *Handlers) BuildSlackTargetConfigForTest(
|
func (s *Handlers) BuildSlackTargetConfigForTest(
|
||||||
w http.ResponseWriter,
|
w http.ResponseWriter,
|
||||||
r *http.Request,
|
r *http.Request,
|
||||||
targetURL string,
|
targetURL string,
|
||||||
) (string, error) {
|
) (string, error) {
|
||||||
return s.buildSlackTargetConfig(w, r, targetURL)
|
return s.buildURLTargetConfig(
|
||||||
}
|
w, r, targetURL, "webhookUrl",
|
||||||
|
"Webhook URL is required for Slack targets",
|
||||||
// BuildHTTPTargetConfigForTest exposes buildHTTPTargetConfig
|
)
|
||||||
// for use in the handlers_test package, taking the form fields
|
|
||||||
// an HTTP target's configuration is built from.
|
|
||||||
func (s *Handlers) BuildHTTPTargetConfigForTest(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
r *http.Request,
|
|
||||||
targetURL, headers, timeout string,
|
|
||||||
) (string, error) {
|
|
||||||
return s.buildHTTPTargetConfig(w, r, targetFormInput{
|
|
||||||
URL: targetURL,
|
|
||||||
Headers: headers,
|
|
||||||
Timeout: timeout,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// BuildDatabaseTargetConfigForTest exposes
|
// BuildDatabaseTargetConfigForTest exposes
|
||||||
|
|||||||
@@ -1,462 +0,0 @@
|
|||||||
package handlers_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"io"
|
|
||||||
"log"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"os"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
gormlogger "gorm.io/gorm/logger"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
|
||||||
"sneak.berlin/go/webhooker/internal/middleware"
|
|
||||||
)
|
|
||||||
|
|
||||||
// gormBoundTailMarker sits at the far end of every client-chosen value
|
|
||||||
// this file sends. Its presence in the log means the whole value
|
|
||||||
// reached the log, so a value that merely happened to be short cannot
|
|
||||||
// pass for a truncated one.
|
|
||||||
const gormBoundTailMarker = "ENDOFCLIENTVALUE"
|
|
||||||
|
|
||||||
// gormBoundFills are the characters a client can drive through the
|
|
||||||
// receiver path segment and the login username, chosen for what a log
|
|
||||||
// handler charges for them.
|
|
||||||
//
|
|
||||||
// The bare C0 control is the one that matters: both handlers spell
|
|
||||||
// U+0001 as a six-byte escape for the one byte it costs to send, the
|
|
||||||
// widest multiplier available below U+10000 and the case a raw-byte
|
|
||||||
// budget breaks on first. GORM's default logger applies no budget at
|
|
||||||
// all, so under the mutation every one of these arrives whole.
|
|
||||||
func gormBoundFills() []struct {
|
|
||||||
name string
|
|
||||||
fill string
|
|
||||||
} {
|
|
||||||
return []struct {
|
|
||||||
name string
|
|
||||||
fill string
|
|
||||||
}{
|
|
||||||
{"plain", "x"},
|
|
||||||
{"quote", `"`},
|
|
||||||
{"backslash", `\`},
|
|
||||||
{"tab", "\t"},
|
|
||||||
{"newline", "\n"},
|
|
||||||
{"c0_control", "\x01"},
|
|
||||||
{"astral_nonprintable", "\U0001000C"},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// syncBuf collects captured output from the goroutine draining the
|
|
||||||
// pipe.
|
|
||||||
type syncBuf struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
b bytes.Buffer
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *syncBuf) Write(p []byte) (int, error) {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
|
|
||||||
return s.b.Write(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *syncBuf) String() string {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
|
|
||||||
return s.b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *syncBuf) reset() {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
|
|
||||||
s.b.Reset()
|
|
||||||
}
|
|
||||||
|
|
||||||
// stdoutCapture redirects os.Stdout for the duration of a test.
|
|
||||||
//
|
|
||||||
// internal/logger builds its handler over os.Stdout at construction
|
|
||||||
// time, so redirecting the variable before the application is built
|
|
||||||
// captures everything the service logger — and therefore the GORM
|
|
||||||
// adapter, which writes through it — emits.
|
|
||||||
type stdoutCapture struct {
|
|
||||||
buf *syncBuf
|
|
||||||
r *os.File
|
|
||||||
w *os.File
|
|
||||||
orig *os.File
|
|
||||||
done chan struct{}
|
|
||||||
seq int
|
|
||||||
}
|
|
||||||
|
|
||||||
func captureStdout(t *testing.T) *stdoutCapture {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
r, w, err := os.Pipe()
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
c := &stdoutCapture{
|
|
||||||
buf: &syncBuf{},
|
|
||||||
r: r,
|
|
||||||
w: w,
|
|
||||||
orig: os.Stdout,
|
|
||||||
done: make(chan struct{}),
|
|
||||||
}
|
|
||||||
|
|
||||||
os.Stdout = w
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
defer close(c.done)
|
|
||||||
|
|
||||||
_, _ = io.Copy(c.buf, r)
|
|
||||||
}()
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
os.Stdout = c.orig
|
|
||||||
_ = w.Close()
|
|
||||||
|
|
||||||
<-c.done
|
|
||||||
|
|
||||||
_ = r.Close()
|
|
||||||
})
|
|
||||||
|
|
||||||
return c
|
|
||||||
}
|
|
||||||
|
|
||||||
// drain returns everything written since the previous drain and
|
|
||||||
// clears the buffer.
|
|
||||||
//
|
|
||||||
// A sentinel is pushed through the same pipe and waited for, so the
|
|
||||||
// draining goroutine is known to have caught up before the buffer is
|
|
||||||
// read. Without it the comparison below would race the reader rather
|
|
||||||
// than measure the writers.
|
|
||||||
func (c *stdoutCapture) drain(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
c.seq++
|
|
||||||
|
|
||||||
sentinel := "\n<<drain-" + strconv.Itoa(c.seq) + ">>\n"
|
|
||||||
|
|
||||||
_, err := c.w.WriteString(sentinel)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
deadline := time.Now().Add(10 * time.Second)
|
|
||||||
for !strings.Contains(c.buf.String(), sentinel) {
|
|
||||||
require.False(
|
|
||||||
t, time.Now().After(deadline),
|
|
||||||
"timed out waiting for captured output",
|
|
||||||
)
|
|
||||||
time.Sleep(time.Millisecond)
|
|
||||||
}
|
|
||||||
|
|
||||||
out := strings.Replace(c.buf.String(), sentinel, "", 1)
|
|
||||||
|
|
||||||
c.buf.reset()
|
|
||||||
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// teeStdout writes to a buffer and to whatever os.Stdout is at the
|
|
||||||
// moment of the write.
|
|
||||||
//
|
|
||||||
// The second half is the point. GORM's package-level default logger
|
|
||||||
// resolves os.Stdout once, at package init, so a logger built over the
|
|
||||||
// variable would keep writing to the real terminal no matter what a
|
|
||||||
// test redirects. Resolving it per write puts the bytes a defaulted
|
|
||||||
// gorm.Config would cost in production into the same capture as
|
|
||||||
// everything else internal/logger emits, which is what lets the volume
|
|
||||||
// assertions below measure the whole writer set rather than one member
|
|
||||||
// of it.
|
|
||||||
type teeStdout struct {
|
|
||||||
buf *syncBuf
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w teeStdout) Write(p []byte) (int, error) {
|
|
||||||
_, _ = os.Stdout.Write(p)
|
|
||||||
|
|
||||||
return w.buf.Write(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
// captureGORMDefault replaces GORM's package-level default logger with
|
|
||||||
// one configured exactly as GORM configures its own, writing to a
|
|
||||||
// buffer and to os.Stdout.
|
|
||||||
//
|
|
||||||
// This is the mutation detector. gormlogger.Default is what a bare
|
|
||||||
// &gorm.Config{} installs, and its config here is GORM's verbatim —
|
|
||||||
// Warn, IgnoreRecordNotFoundError false — so a reverted call site
|
|
||||||
// behaves as it would in production rather than as a test dialed it.
|
|
||||||
// With every gorm.Open in this service naming its own logger, nothing
|
|
||||||
// consults this value and the buffer stays empty; revert any one of
|
|
||||||
// the three and the interpolated SQL lands here.
|
|
||||||
func captureGORMDefault(t *testing.T) *syncBuf {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
buf := &syncBuf{}
|
|
||||||
orig := gormlogger.Default
|
|
||||||
|
|
||||||
gormlogger.Default = gormlogger.New(
|
|
||||||
log.New(teeStdout{buf: buf}, "", log.LstdFlags),
|
|
||||||
gormlogger.Config{
|
|
||||||
SlowThreshold: 200 * time.Millisecond,
|
|
||||||
LogLevel: gormlogger.Warn,
|
|
||||||
IgnoreRecordNotFoundError: false,
|
|
||||||
Colorful: false,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
t.Cleanup(func() { gormlogger.Default = orig })
|
|
||||||
|
|
||||||
return buf
|
|
||||||
}
|
|
||||||
|
|
||||||
// floodUnauthenticated drives reps requests at each of the two
|
|
||||||
// unauthenticated lookups that miss by design, for every fill, with a
|
|
||||||
// client-chosen value of size raw bytes.
|
|
||||||
func floodUnauthenticated(
|
|
||||||
t *testing.T, h *handlers.Handlers, size, reps int,
|
|
||||||
) int {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
requests := 0
|
|
||||||
|
|
||||||
for _, f := range gormBoundFills() {
|
|
||||||
var b strings.Builder
|
|
||||||
|
|
||||||
for b.Len() < size {
|
|
||||||
b.WriteString(f.fill)
|
|
||||||
}
|
|
||||||
|
|
||||||
b.WriteString(gormBoundTailMarker)
|
|
||||||
|
|
||||||
value := b.String()
|
|
||||||
|
|
||||||
for range reps {
|
|
||||||
postWebhook(t, h, value)
|
|
||||||
postUnknownLogin(t, h, value)
|
|
||||||
|
|
||||||
requests += 2
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return requests
|
|
||||||
}
|
|
||||||
|
|
||||||
// floodPerWebhook drives the same client-chosen values at the second
|
|
||||||
// gorm.Open site, the per-webhook database internal/database's
|
|
||||||
// WebhookDBManager opens.
|
|
||||||
//
|
|
||||||
// That site is behind authentication in production, so this is not
|
|
||||||
// part of the unauthenticated flood above and is counted separately.
|
|
||||||
// It is here because the ceiling the README states covers every
|
|
||||||
// writer, and the manager is one of them: with nothing driving it, a
|
|
||||||
// bare &gorm.Config{} could be restored at
|
|
||||||
// internal/database/webhook_db_manager.go and the whole suite would
|
|
||||||
// stay green.
|
|
||||||
func floodPerWebhook(
|
|
||||||
t *testing.T, mgr *database.WebhookDBManager, size, reps int,
|
|
||||||
) int {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
requests := 0
|
|
||||||
|
|
||||||
for _, f := range gormBoundFills() {
|
|
||||||
var b strings.Builder
|
|
||||||
|
|
||||||
for b.Len() < size {
|
|
||||||
b.WriteString(f.fill)
|
|
||||||
}
|
|
||||||
|
|
||||||
b.WriteString(gormBoundTailMarker)
|
|
||||||
|
|
||||||
value := b.String()
|
|
||||||
|
|
||||||
db, err := mgr.GetDB("pin-" + f.name)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
for range reps {
|
|
||||||
var got database.Event
|
|
||||||
|
|
||||||
err = db.Where("id = ?", value).First(&got).Error
|
|
||||||
require.ErrorIs(t, err, gorm.ErrRecordNotFound)
|
|
||||||
|
|
||||||
requests++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return requests
|
|
||||||
}
|
|
||||||
|
|
||||||
// postWebhook drives the receiver with an invented entrypoint path.
|
|
||||||
// The route pattern matches any single segment, so every byte of the
|
|
||||||
// value is the client's, and the lookup behind it misses by design.
|
|
||||||
func postWebhook(
|
|
||||||
t *testing.T, h *handlers.Handlers, entrypoint string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(
|
|
||||||
context.Background(), http.MethodPost, "/webhook/x",
|
|
||||||
strings.NewReader("{}"),
|
|
||||||
)
|
|
||||||
|
|
||||||
rctx := chi.NewRouteContext()
|
|
||||||
rctx.URLParams.Add("uuid", entrypoint)
|
|
||||||
|
|
||||||
req = req.WithContext(context.WithValue(
|
|
||||||
req.Context(), chi.RouteCtxKey, rctx,
|
|
||||||
))
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
h.HandleWebhook().ServeHTTP(w, req)
|
|
||||||
|
|
||||||
require.Equal(t, http.StatusNotFound, w.Code)
|
|
||||||
}
|
|
||||||
|
|
||||||
// postUnknownLogin submits the login form with an unknown username,
|
|
||||||
// through the postLogin helper in logbound_test.go. The field is
|
|
||||||
// bounded only by the 1 MB body cap, and the lookup behind it misses
|
|
||||||
// by design.
|
|
||||||
func postUnknownLogin(
|
|
||||||
t *testing.T, h *handlers.Handlers, username string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
// 401 while the client still has failure budget against this
|
|
||||||
// username, 429 once the login guard has taken it away. Both
|
|
||||||
// outcomes sit behind the user lookup, which is the query this
|
|
||||||
// test is here to drive.
|
|
||||||
require.Contains(
|
|
||||||
t,
|
|
||||||
[]int{http.StatusUnauthorized, http.StatusTooManyRequests},
|
|
||||||
postLogin(t, h, username),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertFloodBounded holds every captured line to the stated ceiling
|
|
||||||
// and proves nothing carried a whole client value.
|
|
||||||
func assertFloodBounded(t *testing.T, label, out string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
assert.NotContains(
|
|
||||||
t, out, gormBoundTailMarker,
|
|
||||||
"%s: the far end of a client-chosen value reached the "+
|
|
||||||
"log, so nothing truncated it", label,
|
|
||||||
)
|
|
||||||
|
|
||||||
for line := range strings.SplitSeq(
|
|
||||||
strings.TrimRight(out, "\n"), "\n",
|
|
||||||
) {
|
|
||||||
if line == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.LessOrEqual(
|
|
||||||
t, len(line), middleware.MaxAccessLogLineBytes,
|
|
||||||
"%s: log line exceeded its bound: %s",
|
|
||||||
label, line[:min(len(line), 300)],
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestFlood_NoWriterGrowsWithTheInput is the definition of done for
|
|
||||||
// the GORM logger defect, stated over every writer at once, for two of
|
|
||||||
// this service's three gorm.Open sites: the main database behind the
|
|
||||||
// two unauthenticated lookups, and the per-webhook database the
|
|
||||||
// WebhookDBManager opens. The third, the archive writer, is pinned in
|
|
||||||
// internal/delivery, where its type lives.
|
|
||||||
//
|
|
||||||
// What each assertion is worth, since two of the three would pass
|
|
||||||
// against a service that had never been fixed if the capture were set
|
|
||||||
// up differently:
|
|
||||||
//
|
|
||||||
// - The gormDefault check is the sharp one. It fires the moment any
|
|
||||||
// gorm.Open in this service goes back to a bare &gorm.Config{}.
|
|
||||||
// - The volume and per-line checks bite only because the replaced
|
|
||||||
// default logger tees into os.Stdout, so a reverted call site
|
|
||||||
// shows up in the same capture as everything internal/logger
|
|
||||||
// writes — the way it would in production. Without that tee both
|
|
||||||
// were vacuous: at INFO the two handler misses log at DEBUG and
|
|
||||||
// the adapter drops the record-not-found, so the capture holds
|
|
||||||
// nothing but fixed-string warnings.
|
|
||||||
//
|
|
||||||
// The level is left where newTestApp leaves it, at INFO: the level an
|
|
||||||
// operator runs at by default, and the one the defect was visible at.
|
|
||||||
// The handlers' own miss lines sit at DEBUG and spend the same
|
|
||||||
// logfield budget as everything else, so they are not what makes
|
|
||||||
// either assertion above bite at any level.
|
|
||||||
//
|
|
||||||
// It is deliberately not parallel: it redirects os.Stdout and replaces
|
|
||||||
// gormlogger.Default, both of which are process-global. Go runs every
|
|
||||||
// non-parallel top-level test to completion before it resumes the
|
|
||||||
// parallel ones, so nothing else in this package is running while the
|
|
||||||
// capture is installed.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // Deliberately sequential; see above.
|
|
||||||
func TestFlood_NoWriterGrowsWithTheInput(t *testing.T) {
|
|
||||||
const (
|
|
||||||
smallBytes = 128
|
|
||||||
bigBytes = 8 << 10
|
|
||||||
reps = 5
|
|
||||||
)
|
|
||||||
|
|
||||||
gormDefault := captureGORMDefault(t)
|
|
||||||
capture := captureStdout(t)
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
mgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &mgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
// Startup chatter is not what this test measures.
|
|
||||||
capture.drain(t)
|
|
||||||
|
|
||||||
floodUnauthenticated(t, h, smallBytes, reps)
|
|
||||||
floodPerWebhook(t, mgr, smallBytes, reps)
|
|
||||||
|
|
||||||
small := capture.drain(t)
|
|
||||||
|
|
||||||
requests := floodUnauthenticated(t, h, bigBytes, reps)
|
|
||||||
requests += floodPerWebhook(t, mgr, bigBytes, reps)
|
|
||||||
big := capture.drain(t)
|
|
||||||
|
|
||||||
assertFloodBounded(t, "small flood", small)
|
|
||||||
assertFloodBounded(t, "big flood", big)
|
|
||||||
|
|
||||||
// GORM's default logger is what the defect was. Nothing in this
|
|
||||||
// service may reach it.
|
|
||||||
got := gormDefault.String()
|
|
||||||
assert.Empty(
|
|
||||||
t, got,
|
|
||||||
"GORM's default logger wrote %d bytes; the first of them: %s",
|
|
||||||
len(got), got[:min(len(got), 300)],
|
|
||||||
)
|
|
||||||
|
|
||||||
// The same flood, with 64 times the client-chosen input, must not
|
|
||||||
// buy 64 times the log. A few bytes of slack covers a latency
|
|
||||||
// field changing width; the input grew by roughly half a megabyte.
|
|
||||||
const slackPerRequest = 64
|
|
||||||
|
|
||||||
assert.LessOrEqual(
|
|
||||||
t, len(big), len(small)+slackPerRequest*requests,
|
|
||||||
"log volume tracked the size of the client's input: "+
|
|
||||||
"%d bytes at %d bytes of input per request, %d bytes "+
|
|
||||||
"at %d",
|
|
||||||
len(small), smallBytes, len(big), bigBytes,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -3,14 +3,12 @@
|
|||||||
package handlers
|
package handlers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"html/template"
|
"html/template"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"sync/atomic"
|
|
||||||
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
@@ -18,7 +16,6 @@ import (
|
|||||||
"sneak.berlin/go/webhooker/internal/globals"
|
"sneak.berlin/go/webhooker/internal/globals"
|
||||||
"sneak.berlin/go/webhooker/internal/healthcheck"
|
"sneak.berlin/go/webhooker/internal/healthcheck"
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
"sneak.berlin/go/webhooker/internal/metrics"
|
|
||||||
"sneak.berlin/go/webhooker/internal/middleware"
|
"sneak.berlin/go/webhooker/internal/middleware"
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
"sneak.berlin/go/webhooker/internal/session"
|
||||||
"sneak.berlin/go/webhooker/templates"
|
"sneak.berlin/go/webhooker/templates"
|
||||||
@@ -29,6 +26,8 @@ const (
|
|||||||
maxBodyShift = 20
|
maxBodyShift = 20
|
||||||
// recentEventLimit is the number of recent events to show.
|
// recentEventLimit is the number of recent events to show.
|
||||||
recentEventLimit = 20
|
recentEventLimit = 20
|
||||||
|
// defaultRetentionDays is the default event retention period.
|
||||||
|
defaultRetentionDays = 30
|
||||||
// paginationPerPage is the number of items per page.
|
// paginationPerPage is the number of items per page.
|
||||||
paginationPerPage = 25
|
paginationPerPage = 25
|
||||||
|
|
||||||
@@ -41,12 +40,6 @@ const (
|
|||||||
// errInvalidPassword is returned when a password does not match.
|
// errInvalidPassword is returned when a password does not match.
|
||||||
var errInvalidPassword = errors.New("invalid password")
|
var errInvalidPassword = errors.New("invalid password")
|
||||||
|
|
||||||
// errVerificationBusy is returned when no password-verification slot
|
|
||||||
// became free before the wait elapsed, so no password was verified.
|
|
||||||
var errVerificationBusy = errors.New(
|
|
||||||
"password verification capacity exhausted",
|
|
||||||
)
|
|
||||||
|
|
||||||
//nolint:revive // HandlersParams is a standard fx naming convention.
|
//nolint:revive // HandlersParams is a standard fx naming convention.
|
||||||
type HandlersParams struct {
|
type HandlersParams struct {
|
||||||
fx.In
|
fx.In
|
||||||
@@ -57,10 +50,8 @@ type HandlersParams struct {
|
|||||||
WebhookDBMgr *database.WebhookDBManager
|
WebhookDBMgr *database.WebhookDBManager
|
||||||
Healthcheck *healthcheck.Healthcheck
|
Healthcheck *healthcheck.Healthcheck
|
||||||
Session *session.Session
|
Session *session.Session
|
||||||
Middleware *middleware.Middleware
|
|
||||||
Notifier delivery.Notifier
|
Notifier delivery.Notifier
|
||||||
Evictor delivery.WebhookEvictor
|
Evictor delivery.WebhookEvictor
|
||||||
SSRFGuard *delivery.Guard
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handlers provides HTTP handler methods for all application
|
// Handlers provides HTTP handler methods for all application
|
||||||
@@ -72,21 +63,9 @@ type Handlers struct {
|
|||||||
db *database.Database
|
db *database.Database
|
||||||
dbMgr *database.WebhookDBManager
|
dbMgr *database.WebhookDBManager
|
||||||
session *session.Session
|
session *session.Session
|
||||||
mw *middleware.Middleware
|
|
||||||
notifier delivery.Notifier
|
notifier delivery.Notifier
|
||||||
evictor delivery.WebhookEvictor
|
evictor delivery.WebhookEvictor
|
||||||
mtr *metrics.Set
|
|
||||||
templates map[string]*template.Template
|
templates map[string]*template.Template
|
||||||
|
|
||||||
// ssrf validates submitted target URLs. It is the same guard
|
|
||||||
// the delivery engine dials through, so a URL accepted here
|
|
||||||
// is one delivery will actually attempt.
|
|
||||||
ssrf *delivery.Guard
|
|
||||||
|
|
||||||
// dummyVerifications counts the equivalent-cost verifications
|
|
||||||
// charged for usernames that do not exist. It exists so a test
|
|
||||||
// can prove that path runs without measuring wall-clock time.
|
|
||||||
dummyVerifications atomic.Uint64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// parsePageTemplate parses a page-specific template set from the
|
// parsePageTemplate parses a page-specific template set from the
|
||||||
@@ -119,11 +98,8 @@ func New(
|
|||||||
s.db = params.Database
|
s.db = params.Database
|
||||||
s.dbMgr = params.WebhookDBMgr
|
s.dbMgr = params.WebhookDBMgr
|
||||||
s.session = params.Session
|
s.session = params.Session
|
||||||
s.mw = params.Middleware
|
|
||||||
s.notifier = params.Notifier
|
s.notifier = params.Notifier
|
||||||
s.evictor = params.Evictor
|
s.evictor = params.Evictor
|
||||||
s.mtr = metrics.Default()
|
|
||||||
s.ssrf = params.SSRFGuard
|
|
||||||
|
|
||||||
// Parse all page templates once at startup
|
// Parse all page templates once at startup
|
||||||
s.templates = map[string]*template.Template{
|
s.templates = map[string]*template.Template{
|
||||||
@@ -134,7 +110,6 @@ func New(
|
|||||||
"source_detail.html": parsePageTemplate("source_detail.html"),
|
"source_detail.html": parsePageTemplate("source_detail.html"),
|
||||||
"source_edit.html": parsePageTemplate("source_edit.html"),
|
"source_edit.html": parsePageTemplate("source_edit.html"),
|
||||||
"source_logs.html": parsePageTemplate("source_logs.html"),
|
"source_logs.html": parsePageTemplate("source_logs.html"),
|
||||||
"target_edit.html": parsePageTemplate("target_edit.html"),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
lc.Append(fx.Hook{
|
lc.Append(fx.Hook{
|
||||||
@@ -251,22 +226,13 @@ func (s *Handlers) renderTemplate(
|
|||||||
s.executeTemplate(w, tmpl, wrapper)
|
s.executeTemplate(w, tmpl, wrapper)
|
||||||
}
|
}
|
||||||
|
|
||||||
// executeTemplate renders the template into a buffer and writes to
|
// executeTemplate runs the template and handles errors.
|
||||||
// the response only once rendering has fully succeeded. Executing
|
|
||||||
// straight into the ResponseWriter commits a partial body and a 200
|
|
||||||
// status before a mid-render error can be reported, leaving no way
|
|
||||||
// to serve a 500. Buffering makes a page's rendered size resident
|
|
||||||
// memory per concurrent viewer, so every page owes it a bound: the
|
|
||||||
// event log caps each stored body at maxRenderedBodyBytes for exactly
|
|
||||||
// this reason.
|
|
||||||
func (s *Handlers) executeTemplate(
|
func (s *Handlers) executeTemplate(
|
||||||
w http.ResponseWriter,
|
w http.ResponseWriter,
|
||||||
tmpl *template.Template,
|
tmpl *template.Template,
|
||||||
data any,
|
data any,
|
||||||
) {
|
) {
|
||||||
var buf bytes.Buffer
|
err := tmpl.Execute(w, data)
|
||||||
|
|
||||||
err := tmpl.Execute(&buf, data)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.log.Error(
|
s.log.Error(
|
||||||
"failed to execute template", "error", err,
|
"failed to execute template", "error", err,
|
||||||
@@ -275,16 +241,5 @@ func (s *Handlers) executeTemplate(
|
|||||||
w, "Internal server error",
|
w, "Internal server error",
|
||||||
http.StatusInternalServerError,
|
http.StatusInternalServerError,
|
||||||
)
|
)
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
||||||
|
|
||||||
_, err = buf.WriteTo(w)
|
|
||||||
if err != nil {
|
|
||||||
s.log.Error(
|
|
||||||
"failed to write rendered page", "error", err,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user