Compare commits
1
Commits
ab7eec4b3e
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
766f33a956 |
+24
-14
@@ -76,6 +76,8 @@ RUN --network=none node_modules/.bin/prettier --check '**/*.md'
|
|||||||
# depends on it.
|
# depends on it.
|
||||||
#
|
#
|
||||||
# golangci/golangci-lint:v2.14.0 (Debian-based), 2026-09-24
|
# golangci/golangci-lint:v2.14.0 (Debian-based), 2026-09-24
|
||||||
|
# Using Debian-based image because mattn/go-sqlite3 (CGO) does not
|
||||||
|
# compile on Alpine musl (off64_t is a glibc type).
|
||||||
FROM golangci/golangci-lint:v2.14.0@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
FROM golangci/golangci-lint:v2.14.0@sha256:ad862ba6b3798cbe0fd9fd7408d498fd74fbd2623a92406b2fd3898faf0bf98f AS lint
|
||||||
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
@@ -165,7 +167,9 @@ RUN --mount=type=tmpfs,target=/tmp/tests,size=512m \
|
|||||||
|
|
||||||
# Build stage
|
# Build stage
|
||||||
# golang:1.26.1-bookworm (Debian-based), 2026-03-17
|
# golang:1.26.1-bookworm (Debian-based), 2026-03-17
|
||||||
# The image ships git and make, which the version step below uses.
|
# Using Debian-based image because gorm.io/driver/sqlite pulls in
|
||||||
|
# mattn/go-sqlite3 (CGO), which does not compile on Alpine musl. The image
|
||||||
|
# ships git and make, which the version step below uses.
|
||||||
FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a3492282a6c820bf4755fd64a4 AS builder
|
FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a3492282a6c820bf4755fd64a4 AS builder
|
||||||
|
|
||||||
# Nothing is wanted from the lint and test phases or from the stylesheet and
|
# Nothing is wanted from the lint and test phases or from the stylesheet and
|
||||||
@@ -207,11 +211,26 @@ RUN version="$(make version VERSION="$VERSION")"; \
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
# Builds through the Makefile's build target, which runs script/assets
|
# Builds through the Makefile's build target, which runs script/assets
|
||||||
# (Alpine.js, extracted from its tarball in 3p/) first. The target builds
|
# (Alpine.js, extracted from its tarball in 3p/) first.
|
||||||
# with cgo off, so the binary is statically linked and runs on the Alpine
|
|
||||||
# runtime image below as it is.
|
|
||||||
RUN make build VERSION="$VERSION"
|
RUN make build VERSION="$VERSION"
|
||||||
|
|
||||||
|
# Rebuild with static linking for the Alpine runtime. The cgo binary the
|
||||||
|
# step above builds is dynamically linked against glibc, which Alpine
|
||||||
|
# (musl) does not have.
|
||||||
|
#
|
||||||
|
# A statically linked glibc still loads shared libraries at run time for
|
||||||
|
# getaddrinfo and dlopen, and Alpine has none to load, so the build tags
|
||||||
|
# keep both out of the binary: netgo and osusergo select Go's own DNS
|
||||||
|
# resolver and user lookup instead of glibc's, and
|
||||||
|
# sqlite_omit_load_extension leaves out SQLite extension loading, which
|
||||||
|
# webhooker never uses.
|
||||||
|
#
|
||||||
|
# The static flags and tags go in through GO_LDFLAGS and GO_TAGS rather
|
||||||
|
# than a -ldflags of their own: the build target composes them with the
|
||||||
|
# -X that stamps the version, so this relink cannot silently drop the
|
||||||
|
# stamp.
|
||||||
|
RUN CGO_ENABLED=1 make build VERSION="$VERSION" GO_TAGS=netgo,osusergo,sqlite_omit_load_extension GO_LDFLAGS='-extldflags "-static"'
|
||||||
|
|
||||||
# Runtime stage
|
# Runtime stage
|
||||||
# alpine:3.21, 2026-03-17
|
# alpine:3.21, 2026-03-17
|
||||||
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
|
FROM alpine:3.21@sha256:c3f8e73fdb79deaebaa2037150150191b9dcbfba68b4a46d70103204c53f4709
|
||||||
@@ -255,17 +274,8 @@ EXPOSE 8080
|
|||||||
# port.
|
# port.
|
||||||
ENV BIND_ADDRESS=0.0.0.0
|
ENV BIND_ADDRESS=0.0.0.0
|
||||||
|
|
||||||
# The probe requests the container's own address, not loopback, so a
|
|
||||||
# listener that answers only on loopback (an empty or loopback
|
|
||||||
# BIND_ADDRESS) fails it. On a network with IPv4, hostname -i prints the
|
|
||||||
# container's IPv4 address, which 0.0.0.0 and :: both answer. On an
|
|
||||||
# IPv6-only network it prints none, and the probe requests the IPv6
|
|
||||||
# address /etc/hosts lists under the container's hostname instead, which
|
|
||||||
# only :: answers.
|
|
||||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
||||||
CMD addr="$(hostname -i 2>/dev/null | cut -d ' ' -f 1)"; \
|
CMD wget --no-verbose --tries=1 --spider http://localhost:8080/.well-known/healthcheck || exit 1
|
||||||
[ -n "$addr" ] || addr="[$(getent hosts "$(hostname)" | cut -d ' ' -f 1)]"; \
|
|
||||||
wget --no-verbose --tries=1 --spider "http://$addr:8080/.well-known/healthcheck" || exit 1
|
|
||||||
|
|
||||||
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
|
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
|
||||||
CMD ["/app/webhooker"]
|
CMD ["/app/webhooker"]
|
||||||
|
|||||||
@@ -16,6 +16,17 @@ VERSION ?= $(shell script/version)
|
|||||||
# exists to correct.
|
# exists to correct.
|
||||||
override VERSION := $(or $(strip $(VERSION)),$(shell script/version))
|
override VERSION := $(or $(strip $(VERSION)),$(shell script/version))
|
||||||
|
|
||||||
|
# Extra linker flags and build tags for the build target. The static
|
||||||
|
# relink in the Dockerfile passes -extldflags and its build tags through
|
||||||
|
# these rather than a -ldflags of its own, so composing flags cannot drop
|
||||||
|
# the version stamp.
|
||||||
|
#
|
||||||
|
# The build target itself always passes -trimpath and -s -w, as the Go
|
||||||
|
# Dockerfile in REPO_POLICIES.md does: no build paths, symbol table or
|
||||||
|
# debug information in the binary.
|
||||||
|
GO_LDFLAGS ?=
|
||||||
|
GO_TAGS ?=
|
||||||
|
|
||||||
bootstrap:
|
bootstrap:
|
||||||
@script/bootstrap
|
@script/bootstrap
|
||||||
|
|
||||||
@@ -43,12 +54,8 @@ fmt-check:
|
|||||||
check:
|
check:
|
||||||
@script/check
|
@script/check
|
||||||
|
|
||||||
# Built with cgo off, -trimpath and -s -w, as the Go Dockerfile in
|
|
||||||
# REPO_POLICIES.md does: the binary carries no build paths, symbol table or
|
|
||||||
# debug information, and on Linux, the image's included, it is statically
|
|
||||||
# linked.
|
|
||||||
build: assets
|
build: assets
|
||||||
CGO_ENABLED=0 go build -trimpath -ldflags '-s -w -X main.version=$(VERSION)' -o bin/webhooker ./cmd/webhooker
|
go build -trimpath -tags '$(GO_TAGS)' -ldflags '$(strip -s -w -X main.version=$(VERSION) $(GO_LDFLAGS))' -o bin/webhooker ./cmd/webhooker
|
||||||
|
|
||||||
run: build
|
run: build
|
||||||
./bin/webhooker
|
./bin/webhooker
|
||||||
|
|||||||
@@ -310,17 +310,16 @@ the namespace, whatever the process bound. Exposure is decided at the publish
|
|||||||
flag instead — `-p 127.0.0.1:8080:8080` rather than `-p 8080:8080` — which is
|
flag instead — `-p 127.0.0.1:8080:8080` rather than `-p 8080:8080` — which is
|
||||||
the operator's to choose and is what [Running with Docker](#running-with-docker)
|
the operator's to choose and is what [Running with Docker](#running-with-docker)
|
||||||
shows. A loopback bind inside a container buys nothing and makes the process
|
shows. A loopback bind inside a container buys nothing and makes the process
|
||||||
unreachable through its own published port. A container on an IPv6-only Docker
|
unreachable through its own published port.
|
||||||
network has no IPv4 address for `0.0.0.0` to answer on, so there it needs
|
|
||||||
`BIND_ADDRESS=::`.
|
|
||||||
|
|
||||||
The value must be an IP address literal:
|
The value must be an IP address literal:
|
||||||
|
|
||||||
- `127.0.0.1` — loopback only (the binary's default). Use this with a reverse
|
- `127.0.0.1` — loopback only (the binary's default). Use this with a reverse
|
||||||
proxy on the same host.
|
proxy on the same host.
|
||||||
- `0.0.0.0` — every IPv4 address and no IPv6 address. The image's default; on a
|
- `0.0.0.0` — every IPv4 address. The image's default; on a bare host, only
|
||||||
bare host, only behind a firewall on the port.
|
behind a firewall on the port.
|
||||||
- `::` — every IPv6 address and every IPv4 address.
|
- `::` — every address, IPv6 and (on Linux, with the default
|
||||||
|
`net.ipv6.bindv6only=0`) IPv4 as well.
|
||||||
- A specific address such as `10.0.0.5` — that interface only.
|
- A specific address such as `10.0.0.5` — that interface only.
|
||||||
|
|
||||||
An **empty** value is treated as unset, as everywhere else here, and takes the
|
An **empty** value is treated as unset, as everywhere else here, and takes the
|
||||||
@@ -341,8 +340,7 @@ machine that has no such interface — is a valid literal, so it reaches the
|
|||||||
listener and fails there. The process logs the bind error and exits non-zero
|
listener and fails there. The process logs the bind error and exits non-zero
|
||||||
rather than staying up with nothing listening. The effective value is in the
|
rather than staying up with nothing listening. The effective value is in the
|
||||||
`bindAddress` field of the startup log line, which is the way to check what a
|
`bindAddress` field of the startup log line, which is the way to check what a
|
||||||
running deployment actually bound. On a bare host, `ss -ltn` lists the socket
|
running deployment actually bound.
|
||||||
itself as `0.0.0.0:8080` for a `0.0.0.0` bind and as `*:8080` for `::`.
|
|
||||||
|
|
||||||
#### Metrics credentials
|
#### Metrics credentials
|
||||||
|
|
||||||
@@ -670,9 +668,9 @@ Either set a literal or leave the variable out entirely.
|
|||||||
Overriding `BIND_ADDRESS` to a loopback address in a container — by that route
|
Overriding `BIND_ADDRESS` to a loopback address in a container — by that route
|
||||||
or deliberately — makes the container unreachable from outside its namespace
|
or deliberately — makes the container unreachable from outside its namespace
|
||||||
even with `-p`. The published port answers nothing, and the health check fails
|
even with `-p`. The published port answers nothing, and the health check fails
|
||||||
too: it requests the container's own address (its IPv4 address, or its IPv6
|
too: it requests `http://localhost:8080`, `localhost` resolves to `::1` first,
|
||||||
address on an IPv6-only network), where a loopback bind is not listening. The
|
and a `127.0.0.1` bind is not listening there. The container then goes
|
||||||
container then goes `unhealthy` about **65 seconds** after start — from
|
`unhealthy` about **65 seconds** after start — from
|
||||||
`HEALTHCHECK --start-period=5s --interval=30s --retries=3`, so failing probes at
|
`HEALTHCHECK --start-period=5s --interval=30s --retries=3`, so failing probes at
|
||||||
5s, 35s and 65s, and `unhealthy` on the third. (Docker's probe cadence during
|
5s, 35s and 65s, and `unhealthy` on the third. (Docker's probe cadence during
|
||||||
the start period has changed between versions; re-derive from those three values
|
the start period has changed between versions; re-derive from those three values
|
||||||
@@ -728,8 +726,8 @@ the credentials.
|
|||||||
address (directly, or through anything that can rewrite source addresses,
|
address (directly, or through anything that can rewrite source addresses,
|
||||||
such as NAT or a published container port). The `remoteIP` field of the
|
such as NAT or a published container port). The `remoteIP` field of the
|
||||||
`http request` log line for a request that came through the proxy shows
|
`http request` log line for a request that came through the proxy shows
|
||||||
that address; the health check's own lines show the container's own
|
that address; the health check's own lines show `::1`. See
|
||||||
address. See [Trusted proxies](#trusted-proxies).
|
[Trusted proxies](#trusted-proxies).
|
||||||
- Leave `BIND_ADDRESS` and `DATA_DIR` unset: the image sets `BIND_ADDRESS`
|
- Leave `BIND_ADDRESS` and `DATA_DIR` unset: the image sets `BIND_ADDRESS`
|
||||||
to `0.0.0.0`, and `DATA_DIR` defaults to `/var/lib/webhooker`.
|
to `0.0.0.0`, and `DATA_DIR` defaults to `/var/lib/webhooker`.
|
||||||
- Everything else is optional; see [Configuration](#configuration).
|
- Everything else is optional; see [Configuration](#configuration).
|
||||||
@@ -804,12 +802,7 @@ comes up, serves pages, and is broken in a way nothing reports.
|
|||||||
proxy's log names it regardless. nginx's default `combined` format already
|
proxy's log names it regardless. nginx's default `combined` format already
|
||||||
logs `$remote_addr`; do not replace it with one that drops the client
|
logs `$remote_addr`; do not replace it with one that drops the client
|
||||||
address, and retain those logs as long as you would want to answer a question
|
address, and retain those logs as long as you would want to answer a question
|
||||||
about traffic. That log also records the full request path, so it holds the
|
about traffic.
|
||||||
entrypoint UUID of every event a sender posts. webhooker's own log does not,
|
|
||||||
apart from the line a `log` target writes. Protect the proxy's log as you
|
|
||||||
would the entrypoint URLs themselves: anyone who can read it can post events
|
|
||||||
to every entrypoint in it (see
|
|
||||||
[The entrypoint URL is the authentication secret](#the-entrypoint-url-is-the-authentication-secret)).
|
|
||||||
|
|
||||||
### nginx
|
### nginx
|
||||||
|
|
||||||
@@ -835,9 +828,7 @@ server {
|
|||||||
client_max_body_size 1m;
|
client_max_body_size 1m;
|
||||||
|
|
||||||
# $remote_addr is the client. webhooker's own log names it, as
|
# $remote_addr is the client. webhooker's own log names it, as
|
||||||
# clientIP, only while TRUSTED_PROXIES covers this proxy. The
|
# clientIP, only while TRUSTED_PROXIES covers this proxy.
|
||||||
# request path in this log holds the entrypoint UUID, so protect
|
|
||||||
# the file like the entrypoint URL itself.
|
|
||||||
access_log /var/log/nginx/webhooker.access.log combined;
|
access_log /var/log/nginx/webhooker.access.log combined;
|
||||||
|
|
||||||
location / {
|
location / {
|
||||||
@@ -1169,16 +1160,7 @@ What that means for an operator:
|
|||||||
|
|
||||||
- **The URL is a capability, so treat it as a secret.** Keep it out of logs,
|
- **The URL is a capability, so treat it as a secret.** Keep it out of logs,
|
||||||
ticket bodies, chat messages and screenshots. Anyone who reads it anywhere can
|
ticket bodies, chat messages and screenshots. Anyone who reads it anywhere can
|
||||||
post events as that sender. Apart from a `log` target's line, webhooker's own
|
post events as that sender.
|
||||||
log lines never carry the UUID: they name a request to the receiver by the
|
|
||||||
route pattern `/h/{uuid}`, the `webhook request received` line names the
|
|
||||||
entrypoint by its `entrypoint_id`, and the access log keeps only the scheme
|
|
||||||
and host of a `Referer`. A `log` target writes the request's headers as
|
|
||||||
received, so a `Referer` naming the entrypoint URL appears in its line, as it
|
|
||||||
does when a sender has followed a redirect from `http://` to `https://`. The
|
|
||||||
reverse proxy's access log does carry the UUID, in the request path, so it
|
|
||||||
must be protected like the URL itself (see
|
|
||||||
[Deployment behind a reverse proxy](#deployment-behind-a-reverse-proxy)).
|
|
||||||
- **Rotating means minting a new entrypoint, not changing a key.** There is no
|
- **Rotating means minting a new entrypoint, not changing a key.** There is no
|
||||||
way to rotate the UUID in place. To retire one, delete the entrypoint (or
|
way to rotate the UUID in place. To retire one, delete the entrypoint (or
|
||||||
deactivate it, which answers `410`) and create a new one, then point the
|
deactivate it, which answers `410`) and create a new one, then point the
|
||||||
@@ -1253,37 +1235,34 @@ markup. The CSP build runs no expressions, so every Alpine directive in
|
|||||||
`static/js/app.js`: `x-data="collapsible"` and `@click="toggle"`, never
|
`static/js/app.js`: `x-data="collapsible"` and `@click="toggle"`, never
|
||||||
`x-data="{ open: false }"` or `@click="open = !open"`.
|
`x-data="{ open: false }"` or `@click="open = !open"`.
|
||||||
|
|
||||||
A browser test in `internal/server` loads the sign-in page, the webhook page,
|
A browser test in `internal/server` loads the webhook page, its edit pages and
|
||||||
its edit pages, the event log, the webhook list, the settings page, the profile
|
the event log under the real policy and checks that: the add entrypoint form
|
||||||
page and the `404` error page shown for a path that does not exist, under the
|
stays hidden until Add is clicked; for every target type, the targets section's
|
||||||
real policy and checks that: the add entrypoint form stays hidden until Add is
|
Add shows only a choice of type with Next and Cancel, Next shows only that
|
||||||
clicked; for every target type, the targets section's Add shows only a choice of
|
type's fields (no url field for `database` or `log`), Cancel at either step
|
||||||
type with Next and Cancel, Next shows only that type's fields (no url field for
|
closes the form, and saving adds the target; a refused target comes back with
|
||||||
`database` or `log`), Cancel at either step closes the form, and saving adds the
|
its form open, the values entered and the reason, and after Cancel the next Add
|
||||||
target; a refused target comes back with its form open, the values entered and
|
starts with an empty form and no reason; a refused save on the target edit page
|
||||||
the reason, and after Cancel the next Add starts with an empty form and no
|
and on the webhook edit page comes back with the reason and every value entered;
|
||||||
reason; a refused save on the target edit page and on the webhook edit page
|
the Copy button beside an entrypoint URL reads "Copied" once clicked; an
|
||||||
comes back with the reason and every value entered; the Copy button beside an
|
entrypoint's Edit button shows its edit form in place of its description and
|
||||||
entrypoint URL reads "Copied" once clicked; an entrypoint's Edit button shows
|
hides until the form closes, Cancel hides the form and drops what was typed, as
|
||||||
its edit form in place of its description and hides until the form closes,
|
does leaving the page and going back to it, and Save changes the description; of
|
||||||
Cancel hides the form and drops what was typed, as does leaving the page and
|
the recent events on the webhook page only the newest starts expanded, each
|
||||||
going back to it, and Save changes the description; of the recent events on the
|
expands and collapses, and Open leads to the event's own page; of the events in
|
||||||
webhook page only the newest starts expanded, each expands and collapses, and
|
the event log only the newest starts expanded, and an event there expands and
|
||||||
Open leads to the event's own page; of the events in the event log only the
|
collapses when its row's caret or its ID is clicked, and from the keyboard, but
|
||||||
newest starts expanded, and an event there expands and collapses when its row's
|
not when its ID is selected with the mouse, and a delivery's attempts inside it
|
||||||
caret or its ID is clicked, and from the keyboard, but not when its ID is
|
expand and collapse; and at phone width the menu button opens and closes the
|
||||||
selected with the mouse, and a delivery's attempts inside it expand and
|
mobile menu, and neither the webhook page nor the event log, with a delivery's
|
||||||
collapse; at phone width the menu button opens and closes the mobile menu, and
|
attempts open, scrolls sideways or cuts anything off at the page's or a card's
|
||||||
neither the webhook page nor the event log, with a delivery's attempts open,
|
edge. It also fails if the browser reports a console warning or error, an
|
||||||
scrolls sideways or cuts anything off at the page's or a card's edge; and a
|
uncaught exception, or anything the policy refused. `make check` and the image
|
||||||
target's Delete asks first, naming the target, sends nothing when the answer is
|
build lint it but do not run it, and `make test` leaves it out (its file is
|
||||||
no and deletes the target when it is yes. It also fails if the browser reports a
|
built only with the `browser` build tag). Run it with `make test-browser` after
|
||||||
console warning or error, an uncaught exception, or anything the policy refused.
|
changing `templates/` or `static/js/`: that builds `Dockerfile.browser`, which
|
||||||
`make check` and the image build lint it but do not run it, and `make test`
|
runs the test in a digest-pinned headless browser image, so the host needs no
|
||||||
leaves it out (its file is built only with the `browser` build tag). Run it with
|
browser.
|
||||||
`make test-browser` after changing `templates/` or `static/js/`: that builds
|
|
||||||
`Dockerfile.browser`, which runs the test in a digest-pinned headless browser
|
|
||||||
image, so the host needs no browser.
|
|
||||||
|
|
||||||
The package's tarball is committed as `3p/alpinejs-csp-3.14.9.tgz`, byte for
|
The package's tarball is committed as `3p/alpinejs-csp-3.14.9.tgz`, byte for
|
||||||
byte as the npm registry publishes it. It is a dependency, not this repo's build
|
byte as the npm registry publishes it. It is a dependency, not this repo's build
|
||||||
@@ -1316,9 +1295,7 @@ and commit the regenerated file with the change. Tailwind takes classes only
|
|||||||
from the files that `input.css` names in its `@source` lines; a class written in
|
from the files that `input.css` names in its `@source` lines; a class written in
|
||||||
any other file is not generated until that file is named there too. `make check`
|
any other file is not generated until that file is named there too. `make check`
|
||||||
and the image build fail when the committed file differs from what `make css`
|
and the image build fail when the committed file differs from what `make css`
|
||||||
generates. `static/css/style.css` is hand-written and is not generated. The
|
generates. `static/css/style.css` is hand-written and is not generated.
|
||||||
pages' Content-Security-Policy refuses inline styles, so a template styles an
|
|
||||||
element with a class, never a `style` attribute.
|
|
||||||
|
|
||||||
`make css` runs the Tailwind standalone CLI in Docker, at the version and sha256
|
`make css` runs the Tailwind standalone CLI in Docker, at the version and sha256
|
||||||
pinned in the Dockerfile's stylesheet stages; it is never installed on the host.
|
pinned in the Dockerfile's stylesheet stages; it is never installed on the host.
|
||||||
@@ -1340,22 +1317,20 @@ webhooker solves this by acting as a durable intermediary:
|
|||||||
nothing is lost if downstream targets are unavailable.
|
nothing is lost if downstream targets are unavailable.
|
||||||
|
|
||||||
2. **Guaranteed delivery** — Events are queued for delivery to each configured
|
2. **Guaranteed delivery** — Events are queued for delivery to each configured
|
||||||
target. Failed deliveries are retried with configurable backoff. Delivery
|
target. Failed deliveries are retried with configurable backoff. Every
|
||||||
attempts are logged with status codes, response bodies, and timing.
|
delivery attempt is logged with status codes, response bodies, and timing.
|
||||||
|
|
||||||
**That guarantee is at-least-once, not exactly-once.** When a send reaches
|
**That guarantee is at-least-once, not exactly-once.** When a send reaches
|
||||||
its target but the write recording that outcome fails, the delivery is
|
its target but the write recording that outcome fails, the delivery is
|
||||||
deliberately left in a recoverable state rather than marked done — losing a
|
deliberately left in a recoverable state rather than marked done — losing a
|
||||||
delivery is the worse failure — so the pending sweep picks it up about
|
delivery is the worse failure — so the pending sweep picks it up about
|
||||||
fifteen minutes later, or the next restart does, and the target receives a
|
fifteen minutes later, or the next restart does, and the target receives a
|
||||||
payload it already got. A send still in flight when the service stops is
|
payload it already got. webhooker adds no delivery identifier of its own to
|
||||||
left the same way: its attempt is not recorded, and the next start sends it
|
an outbound request, so **make your receiver idempotent** against whatever
|
||||||
again (see Shutdown). webhooker adds no delivery identifier of its own to an
|
the payload itself carries.
|
||||||
outbound request, so **make your receiver idempotent** against whatever the
|
|
||||||
payload itself carries.
|
|
||||||
|
|
||||||
3. **Observability** — Full request/response logging for every webhook received
|
3. **Observability** — Full request/response logging for every webhook received
|
||||||
and for delivery attempts. Prometheus metrics expose volume, latency, and
|
and every delivery attempted. Prometheus metrics expose volume, latency, and
|
||||||
error rates. The web UI provides real-time visibility into event flow.
|
error rates. The web UI provides real-time visibility into event flow.
|
||||||
|
|
||||||
4. **Fan-out** — A single incoming webhook can be delivered to multiple targets
|
4. **Fan-out** — A single incoming webhook can be delivered to multiple targets
|
||||||
@@ -1393,10 +1368,10 @@ It uses:
|
|||||||
management
|
management
|
||||||
- **[go-chi](https://github.com/go-chi/chi)** for HTTP routing
|
- **[go-chi](https://github.com/go-chi/chi)** for HTTP routing
|
||||||
- **[GORM](https://gorm.io)** for database access with
|
- **[GORM](https://gorm.io)** for database access with
|
||||||
**[modernc.org/sqlite](https://pkg.go.dev/modernc.org/sqlite)**, which is pure
|
**[modernc.org/sqlite](https://pkg.go.dev/modernc.org/sqlite)** as the runtime
|
||||||
Go, as the SQLite driver. `gorm.io/driver/sqlite` also imports
|
SQLite driver. Note: `gorm.io/driver/sqlite` transitively depends on
|
||||||
`mattn/go-sqlite3`, whose stub for builds without cgo is linked into the
|
`mattn/go-sqlite3`, which requires CGO at build time (see [Docker](#docker)
|
||||||
binary but never used
|
section)
|
||||||
- **[slog](https://pkg.go.dev/log/slog)** (stdlib) for structured logging with
|
- **[slog](https://pkg.go.dev/log/slog)** (stdlib) for structured logging with
|
||||||
TTY detection (text for dev, JSON for prod)
|
TTY detection (text for dev, JSON for prod)
|
||||||
- **[gorilla/sessions](https://github.com/gorilla/sessions)** for encrypted
|
- **[gorilla/sessions](https://github.com/gorilla/sessions)** for encrypted
|
||||||
@@ -1520,34 +1495,46 @@ validated as on the add target form, and the webhook and its targets are created
|
|||||||
together or not at all.
|
together or not at all.
|
||||||
|
|
||||||
| Field | Type | Description |
|
| Field | Type | Description |
|
||||||
| ---------------- | ------- | --------------------------------------------- |
|
| ---------------- | ------- | ----------------------------------------------------------- |
|
||||||
| `id` | UUID | Primary key |
|
| `id` | UUID | Primary key |
|
||||||
| `user_id` | UUID | Foreign key → User |
|
| `user_id` | UUID | Foreign key → User |
|
||||||
| `name` | string | Human-readable name |
|
| `name` | string | Human-readable name |
|
||||||
| `description` | string | Optional description |
|
| `description` | string | Optional description |
|
||||||
| `retention_days` | integer | Days to retain events; 0 means retain forever |
|
| `retention_days` | integer | Days to retain events (default: 30; 0 means retain forever) |
|
||||||
|
|
||||||
**Relations:** Belongs to User. Has many Entrypoints. Has many Targets.
|
**Relations:** Belongs to User. Has many Entrypoints. Has many Targets.
|
||||||
|
|
||||||
The `retention_days` field controls how long event data is kept in the webhook's
|
The `retention_days` field controls how long event data is kept in the webhook's
|
||||||
dedicated database before automatic cleanup.
|
dedicated database before automatic cleanup.
|
||||||
|
|
||||||
The value is stored as entered. Setting `retention_days` to `0` means "retain
|
Setting `retention_days` to `0` means "retain events forever". Because the
|
||||||
events forever": the retention reaper skips the webhook entirely, and the web UI
|
column carries a default of 30, a literal zero cannot survive an insert, so a
|
||||||
displays its retention as "forever" rather than as a day count.
|
zero is rewritten on save to a sentinel of `365 * 1000` days
|
||||||
|
(`database.RetentionForeverDays`). The retention reaper recognises that sentinel
|
||||||
|
and skips the webhook entirely, and the web UI displays such a webhook's
|
||||||
|
retention as "forever" rather than as a day count.
|
||||||
|
|
||||||
The new webhook and edit forms accept `0`, or a whole number of days from `1` up
|
Submitted `retention_days` values therefore fall into three bands, not two:
|
||||||
to `database.MaxFiniteRetentionDays` (106751 days, about 292 years). Anything
|
|
||||||
else, a negative value or one above that cap included, is rejected with a 400. A
|
- `1` up to `database.MaxFiniteRetentionDays` (106751 days, about 292 years) is
|
||||||
blank field gives the default of 30 days (`database.DefaultRetentionDays`) on
|
accepted as a finite retention.
|
||||||
the new webhook form, and leaves the stored value unchanged on the edit form.
|
- Above that ceiling but below the retain-forever sentinel of 365000
|
||||||
|
(`database.RetentionForeverDays`) is rejected with a 400. This is the band the
|
||||||
|
cap exists for.
|
||||||
|
- `0`, and `365000` or above, are accepted and mean retain forever, collapsing
|
||||||
|
to the sentinel — `0` in `Webhook.BeforeSave`, the large values in
|
||||||
|
`parseRetentionDays`. The large values are not out of range: the edit form
|
||||||
|
pre-fills the sentinel for a retain-forever webhook, so submitting that form
|
||||||
|
back unchanged has to keep meaning "forever".
|
||||||
|
|
||||||
|
A negative value is in none of the three: `parseRetentionDays` rejects it with a
|
||||||
|
400 before `BeforeSave` ever sees it.
|
||||||
|
|
||||||
The cap is not arbitrary: the reaper computes its cutoff as a `time.Duration`,
|
The cap is not arbitrary: the reaper computes its cutoff as a `time.Duration`,
|
||||||
an int64 nanosecond count, and a longer period overflows it. An overflowed
|
an int64 nanosecond count, and a longer period overflows it. An overflowed
|
||||||
cutoff lands in the future, where it matches every row, so the sweep would
|
cutoff lands in the future, where it matches every row, so the sweep would
|
||||||
delete every event the webhook has instead of none. The reaper also clamps the
|
delete every event the webhook has instead of none. The reaper also clamps the
|
||||||
value it reads to the cap, and reads a negative value as forever, so no stored
|
value it reads to the cap, so no stored value can trigger that either.
|
||||||
value can trigger that either.
|
|
||||||
|
|
||||||
#### Entrypoint
|
#### Entrypoint
|
||||||
|
|
||||||
@@ -1580,26 +1567,28 @@ A delivery destination for events. Each target defines where and how events
|
|||||||
should be forwarded.
|
should be forwarded.
|
||||||
|
|
||||||
| Field | Type | Description |
|
| Field | Type | Description |
|
||||||
| -------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
| ------------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
| `id` | UUID | Primary key |
|
| `id` | UUID | Primary key |
|
||||||
| `webhook_id` | UUID | Foreign key → Webhook |
|
| `webhook_id` | UUID | Foreign key → Webhook |
|
||||||
| `name` | string | Human-readable name |
|
| `name` | string | Human-readable name |
|
||||||
| `type` | TargetType | One of: `http`, `slack`, `database`, `log` |
|
| `type` | TargetType | One of: `http`, `slack`, `database`, `log` |
|
||||||
| `active` | boolean | Whether deliveries are enabled (default: true) |
|
| `active` | boolean | Whether deliveries are enabled (default: true) |
|
||||||
| `config` | JSON text | Type-specific configuration |
|
| `config` | JSON text | Type-specific configuration |
|
||||||
| `max_attempts` | integer | How many times an `http` or `slack` target attempts each delivery in all: at least 1, and 1 by default. Ignored by `database` and `log` targets. The web UI labels it Delivery attempts |
|
| `max_retries` | integer | Total delivery attempts for `http` and `slack` targets, not retries on top of the first: 0 is a single fire-and-forget attempt with no retries and no circuit breaker, and a value of N makes N attempts in all, with exponential backoff and a per-target circuit breaker. Ignored by `database` and `log` targets. The web UI labels it Delivery attempts |
|
||||||
|
|
||||||
**Relations:** Belongs to Webhook. Has many Deliveries.
|
**Relations:** Belongs to Webhook. Has many Deliveries.
|
||||||
|
|
||||||
**Target types:**
|
**Target types:**
|
||||||
|
|
||||||
- **`http`** — Forward the event as an HTTP POST to a configured URL. With
|
- **`http`** — Forward the event as an HTTP POST to a configured URL.
|
||||||
`max_attempts` at 1 the target makes a single attempt; above 1 it retries a
|
`max_retries` is the total number of delivery attempts, not retries on top of
|
||||||
failed delivery with exponential backoff, up to that many attempts in all,
|
the first: when `max_retries` is 0 (the default), the target operates in
|
||||||
behind a per-target circuit breaker.
|
fire-and-forget mode, a single attempt with no retries and no circuit breaker;
|
||||||
|
a value of N makes up to N attempts in all, retrying failed deliveries with
|
||||||
|
exponential backoff and protecting them with a per-target circuit breaker.
|
||||||
- **`slack`** — Post the event as a formatted message to a Slack-compatible
|
- **`slack`** — Post the event as a formatted message to a Slack-compatible
|
||||||
incoming webhook URL (`webhookUrl` in `config`). It is built on the same HTTP
|
incoming webhook URL (`webhookUrl` in `config`). It is built on the same HTTP
|
||||||
core as `http` and honours `max_attempts` identically, circuit breaker
|
core as `http` and honours `max_retries` identically, circuit breaker
|
||||||
included. See the Slack target section under "Per-Webhook Event Databases" for
|
included. See the Slack target section under "Per-Webhook Event Databases" for
|
||||||
the message format.
|
the message format.
|
||||||
- **`database`** — Archive the full event as a row into the target's own archive
|
- **`database`** — Archive the full event as a row into the target's own archive
|
||||||
@@ -1779,7 +1768,7 @@ is what bounds it.
|
|||||||
#### DeliveryResult
|
#### DeliveryResult
|
||||||
|
|
||||||
The result of a single delivery attempt. Every attempt (including retries) is
|
The result of a single delivery attempt. Every attempt (including retries) is
|
||||||
individually logged, except one cut off by a stop (see Shutdown).
|
individually logged for full observability.
|
||||||
|
|
||||||
| Field | Type | Description |
|
| Field | Type | Description |
|
||||||
| --------------- | ------- | ----------------------------------------- |
|
| --------------- | ------- | ----------------------------------------- |
|
||||||
@@ -2100,15 +2089,15 @@ writes go on meanwhile, since under WAL a reader never blocks a writer. Until
|
|||||||
the open file is closed its `-wal` cannot be checkpointed past what the download
|
the open file is closed its `-wal` cannot be checkpointed past what the download
|
||||||
reads, so a long download lets that `-wal` grow.
|
reads, so a long download lets that `-wal` grow.
|
||||||
|
|
||||||
Deleting a webhook releases its archives: its cached archive writers are dropped
|
Deleting a webhook releases its archives: the delivery engine's cached archive
|
||||||
and their file handles closed, so nothing lingers after the webhook is gone. The
|
writers are dropped and their file handles closed, so nothing lingers after the
|
||||||
archive **files themselves are deliberately left on disk**. Unlike the event
|
webhook is gone. The archive **files themselves are deliberately left on disk**.
|
||||||
database — per-webhook working storage that is hard-deleted with the webhook —
|
Unlike the event database — per-webhook working storage that is hard-deleted
|
||||||
an archive is long-term storage an operator may still want to keep or move away
|
with the webhook — an archive is long-term storage an operator may still want to
|
||||||
for offline retention, and destroying it as a side effect of deleting a webhook
|
keep or move away for offline retention, and destroying it as a side effect of
|
||||||
would be unrecoverable. Removing an `archive-….db` is the operator's call.
|
deleting a webhook would be unrecoverable. Removing an `archive-….db` is the
|
||||||
Deleting a `database` target releases its writer the same way, and for the same
|
operator's call. Deleting a `database` target releases its writer the same way,
|
||||||
reason leaves its file alone.
|
and for the same reason leaves its file alone.
|
||||||
|
|
||||||
The **Slack target type** sends webhook events as formatted messages to any
|
The **Slack target type** sends webhook events as formatted messages to any
|
||||||
Slack-compatible incoming webhook URL (works with Slack, Mattermost, and other
|
Slack-compatible incoming webhook URL (works with Slack, Mattermost, and other
|
||||||
@@ -2121,7 +2110,9 @@ endpoint. That is the JSON key; the error text for a missing one reads
|
|||||||
`webhook_url is required`, which is the message, not the key.
|
`webhook_url is required`, which is the message, not the key.
|
||||||
|
|
||||||
The database uses the
|
The database uses the
|
||||||
[modernc.org/sqlite](https://pkg.go.dev/modernc.org/sqlite) driver.
|
[modernc.org/sqlite](https://pkg.go.dev/modernc.org/sqlite) driver at runtime,
|
||||||
|
though CGO is required at build time due to the transitive `mattn/go-sqlite3`
|
||||||
|
dependency from `gorm.io/driver/sqlite`.
|
||||||
|
|
||||||
#### Schema
|
#### Schema
|
||||||
|
|
||||||
@@ -2172,11 +2163,11 @@ External Service
|
|||||||
▼ ▼ ▼
|
▼ ▼ ▼
|
||||||
┌────────────┐ ┌────────────┐ ┌────────────┐
|
┌────────────┐ ┌────────────┐ ┌────────────┐
|
||||||
│ HTTP Target│ │ HTTP Target│ │ Log Target │
|
│ HTTP Target│ │ HTTP Target│ │ Log Target │
|
||||||
│max_attempts│ │max_attempts│ │ (stdout) │
|
│(max_retries│ │(max_retries│ │ (stdout) │
|
||||||
│ == 1, │ │ > 1, │ └────────────┘
|
│ == 0) │ │ > 0, │ └────────────┘
|
||||||
│ one attempt│ │ backoff + │
|
│ fire+forget│ │ backoff + │
|
||||||
└────────────┘ │ circuit │
|
└────────────┘ │ circuit │
|
||||||
│ breaker │
|
│ breaker) │
|
||||||
└────────────┘
|
└────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -2215,16 +2206,12 @@ This means:
|
|||||||
buffer rather than spawning more goroutines.
|
buffer rather than spawning more goroutines.
|
||||||
- **Independent results** — each worker records its own delivery result in the
|
- **Independent results** — each worker records its own delivery result in the
|
||||||
per-webhook database without coordination.
|
per-webhook database without coordination.
|
||||||
- **Graceful shutdown** — the stop hook cancels the workers' context. A worker
|
- **Graceful shutdown** — cancel the context, workers finish their current task
|
||||||
may still take tasks already queued before it exits. An `http` or `slack`
|
and exit. The stop hook waits for the pool via `lifecycle.WaitForShutdown`,
|
||||||
request in flight is cut off, and one started after the cancel fails at once.
|
which bounds that wait by fx's stop timeout rather than blocking forever on a
|
||||||
Neither is recorded or counted in the metrics, and the next start sends the
|
wedged worker. On timeout it logs at `ERROR` and returns an error, and the
|
||||||
delivery again (see Shutdown). A `database` or `log` task runs as normal. The
|
goroutines that did not finish are still running — an unclean shutdown is
|
||||||
stop hook waits for the pool via `lifecycle.WaitForShutdown`, which bounds
|
reported rather than hidden.
|
||||||
that wait by fx's stop timeout rather than blocking forever on a wedged
|
|
||||||
worker. On timeout it logs at `ERROR` and returns an error, and the goroutines
|
|
||||||
that did not finish are still running — an unclean shutdown is reported rather
|
|
||||||
than hidden.
|
|
||||||
|
|
||||||
**Recovery paths:**
|
**Recovery paths:**
|
||||||
|
|
||||||
@@ -2253,7 +2240,7 @@ re-inject it to every currently active target.
|
|||||||
|
|
||||||
### Circuit Breaker (HTTP and Slack Targets with Retries)
|
### Circuit Breaker (HTTP and Slack Targets with Retries)
|
||||||
|
|
||||||
`http` and `slack` targets with `max_attempts` > 1 are protected by a
|
`http` and `slack` targets with `max_retries` > 0 are protected by a
|
||||||
**per-target circuit breaker** that prevents hammering a down target with
|
**per-target circuit breaker** that prevents hammering a down target with
|
||||||
repeated failed delivery attempts. The circuit breaker is in-memory only and
|
repeated failed delivery attempts. The circuit breaker is in-memory only and
|
||||||
resets on restart (which is fine — startup recovery rescans the database
|
resets on restart (which is fine — startup recovery rescans the database
|
||||||
@@ -2294,10 +2281,10 @@ anyway).
|
|||||||
- **Cooldown:** 30 seconds in open state before probing
|
- **Cooldown:** 30 seconds in open state before probing
|
||||||
|
|
||||||
**Scope:** Circuit breakers apply to **`http` and `slack` targets with
|
**Scope:** Circuit breakers apply to **`http` and `slack` targets with
|
||||||
`max_attempts` > 1**. The Slack target is built on the same HTTP core and hands
|
`max_retries` > 0**. The Slack target is built on the same HTTP core and hands
|
||||||
its own `max_attempts` to the same retry path, so it gets a breaker with the
|
its own `max_retries` to the same retry path, so it gets a breaker with the same
|
||||||
same 5-failure / 30-second defaults. Single-attempt targets of either type
|
5-failure / 30-second defaults. Fire-and-forget targets of either type
|
||||||
(`max_attempts` == 1), database targets (local operations), and log targets
|
(`max_retries` == 0), database targets (local operations), and log targets
|
||||||
(stdout) do not use circuit breakers.
|
(stdout) do not use circuit breakers.
|
||||||
|
|
||||||
When a circuit is open and a new delivery arrives, the engine marks the delivery
|
When a circuit is open and a new delivery arrives, the engine marks the delivery
|
||||||
@@ -2330,7 +2317,7 @@ getting anywhere.
|
|||||||
| Metric | Type | Meaning |
|
| Metric | Type | Meaning |
|
||||||
| -------------------------------------- | --------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
| -------------------------------------- | --------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
| `webhooker_events_received_total` | counter | Events received and durably stored. Compare against the delivery counters on one dashboard |
|
| `webhooker_events_received_total` | counter | Events received and durably stored. Compare against the delivery counters on one dashboard |
|
||||||
| `webhooker_delivery_attempts_total` | counter | Delivery attempts actually dispatched to a target. A delivery a circuit breaker refused is not one: it is counted as a retry instead, but only when the refusal moves it into `retrying`. An attempt cut off by a stop is not counted (see Shutdown) |
|
| `webhooker_delivery_attempts_total` | counter | Delivery attempts actually dispatched to a target. A delivery a circuit breaker refused is not one: it is counted as a retry instead, but only when the refusal moves it into `retrying` |
|
||||||
| `webhooker_deliveries_succeeded_total` | counter | Deliveries that reached `delivered` |
|
| `webhooker_deliveries_succeeded_total` | counter | Deliveries that reached `delivered` |
|
||||||
| `webhooker_deliveries_failed_total` | counter | Deliveries that failed terminally and will not be retried |
|
| `webhooker_deliveries_failed_total` | counter | Deliveries that failed terminally and will not be retried |
|
||||||
| `webhooker_delivery_retries_total` | counter | Deliveries put back into `retrying` |
|
| `webhooker_delivery_retries_total` | counter | Deliveries put back into `retrying` |
|
||||||
@@ -2429,32 +2416,27 @@ log volume it caps rather than eliminates. A path that names no entrypoint is
|
|||||||
recorded by the handler at `DEBUG`, and the aggregate limiter logs its own
|
recorded by the handler at `DEBUG`, and the aggregate limiter logs its own
|
||||||
rejections at `DEBUG` and without the path, so neither appears at all under the
|
rejections at `DEBUG` and without the path, so neither appears at all under the
|
||||||
default level. The per-entrypoint limiter is the loud one: it logs every
|
default level. The per-entrypoint limiter is the loud one: it logs every
|
||||||
rejection at `WARN`, naming the route pattern `/h/{uuid}` rather than the
|
rejection at `WARN` with the request path, which on this route is
|
||||||
request path. A client hammering a single invented path is served
|
attacker-controlled text. A client hammering a single invented path is served
|
||||||
`RECEIVER_RATE_LIMIT` requests and has the rest of its aggregate budget rejected
|
`RECEIVER_RATE_LIMIT` requests and has the rest of its aggregate budget rejected
|
||||||
there, so the aggregate limit is what bounds the _number_ of those `WARN` lines
|
there, so the aggregate limit is what bounds the _number_ of those `WARN` lines
|
||||||
— to under ten times `RECEIVER_RATE_LIMIT` per minute per client IP, 1080 at the
|
— to under ten times `RECEIVER_RATE_LIMIT` per minute per client IP, 1080 at the
|
||||||
defaults, where before it there was no bound at all. The access log is bounded
|
defaults, where before it there was no bound at all. Their _width_ is bounded by
|
||||||
by neither limit: every request is recorded once at `INFO`, served or rejected
|
the field budgets below, the same ones the access log spends. The access log is
|
||||||
alike.
|
bounded by neither limit: every request is recorded once at `INFO`, served or
|
||||||
|
rejected alike.
|
||||||
|
|
||||||
What the access log does bound is the _content_ of those lines. On the receiver
|
What the access log does bound is the _content_ of those lines. A 3xx or 4xx
|
||||||
route it logs the route pattern `/h/{uuid}` whatever the status, because the
|
response logs the chi route pattern — `/h/{uuid}`, `/user/{username}//`, or the
|
||||||
concrete path carries the entrypoint UUID, which is the entrypoint's credential
|
literal `(unmatched)` when the request hit no route at all — in place of the
|
||||||
(see
|
concrete URL. Those are the outcomes an unauthenticated client can drive for
|
||||||
[The entrypoint URL is the authentication secret](#the-entrypoint-url-is-the-authentication-secret)).
|
free: 404 and 429 on any invented receiver path, a login redirect on any
|
||||||
A request chi never routed, because it matched no route or because a middleware
|
invented profile path. Logging the URL there would let a flood write text of its
|
||||||
answered it first, logs the literal `(unmatched)` whatever the status. In the
|
own choosing, at a length of its own choosing, into the log. 2xx and 5xx
|
||||||
`dev` environment the CORS middleware answers a preflight that way, so a
|
responses keep the concrete path — a success resolved against a static route or
|
||||||
preflight to the receiver does not log its UUID either. On every other route, a
|
against the operator's own data (on the receiver, against a stored entrypoint
|
||||||
3xx or 4xx response logs the chi route pattern, such as `/user/{username}//`, in
|
UUID), and a 5xx is a bug in this service, where the exact path is the evidence
|
||||||
place of the concrete URL. Those are the outcomes an unauthenticated client can
|
and no client can provoke one at will.
|
||||||
drive for free: a 404 on any invented path, a login redirect on any invented
|
|
||||||
profile path. Logging the URL there would let a flood write text of its own
|
|
||||||
choosing, at a length of its own choosing, into the log. 2xx and 5xx responses
|
|
||||||
keep the concrete path — a success resolved against a static route or against
|
|
||||||
the operator's own data, and a 5xx is a bug in this service, where the exact
|
|
||||||
path is the evidence and no client can provoke one at will.
|
|
||||||
|
|
||||||
The query string is never logged; it is replaced by the fixed marker
|
The query string is never logged; it is replaced by the fixed marker
|
||||||
`?(redacted)`. It is client-chosen on every route, and
|
`?(redacted)`. It is client-chosen on every route, and
|
||||||
@@ -2466,11 +2448,6 @@ to, `notice`, which names the line a page shows after an action, and the event
|
|||||||
log's `show`, which picks the events it lists. A query string sent to an
|
log's `show`, which picks the events it lists. A query string sent to an
|
||||||
entrypoint is not lost either: the event stores it, and the event log shows it.
|
entrypoint is not lost either: the event stores it, and the event log shows it.
|
||||||
|
|
||||||
The `referer` field keeps only the scheme and host of the request's `Referer`,
|
|
||||||
and logs `(redacted)` for one without them, because its path can be an
|
|
||||||
entrypoint URL: a sender that follows the `http://` to `https://` redirect in
|
|
||||||
the nginx example sends the URL it was redirected from.
|
|
||||||
|
|
||||||
Client-supplied request content does not leave the host by the other route
|
Client-supplied request content does not leave the host by the other route
|
||||||
either. The Sentry SDK attaches the request to every event it captures,
|
either. The Sentry SDK attaches the request to every event it captures,
|
||||||
independently of the access log, and `SendDefaultPII=false` does not cover all
|
independently of the access log, and `SendDefaultPII=false` does not cover all
|
||||||
@@ -2480,18 +2457,14 @@ are both taken unconditionally, the body precisely because these handlers call
|
|||||||
body with `(redacted)`, drops cookies and the remote-address environment, and
|
body with `(redacted)`, drops cookies and the remote-address environment, and
|
||||||
reduces the headers to a fixed allowlist — `Accept`, `Content-Length`,
|
reduces the headers to a fixed allowlist — `Accept`, `Content-Length`,
|
||||||
`Content-Type`, `Host`, `Origin`, `Referer`, `User-Agent` and `X-Request-Id`.
|
`Content-Type`, `Host`, `Origin`, `Referer`, `User-Agent` and `X-Request-Id`.
|
||||||
`Referer` is sent as its scheme and host only, because its path can be an
|
|
||||||
entrypoint URL; `internal/referer.SchemeAndHost` does that reduction for both
|
|
||||||
the access log's `referer` field and this hook.
|
|
||||||
|
|
||||||
The same hook rewrites the request URL. The SDK builds it as
|
The same hook rewrites the request URL. The SDK builds it as
|
||||||
`scheme://host/path` from the concrete path, which on the receiver route is
|
`scheme://host/path` from the concrete path, which on the receiver route is
|
||||||
`/h/<uuid>` in full — and that UUID is a write capability, not an identifier:
|
`/h/<uuid>` in full — and that UUID is a write capability, not an identifier:
|
||||||
anyone holding it can post events this service accepts and its targets then
|
anyone holding it can post events this service accepts and its targets then
|
||||||
deliver. What is sent is the chi route pattern instead: `http://host/h/{uuid}`.
|
deliver. A tracker has its own retention, access control and deletion policy, so
|
||||||
A tracker has its own retention, access control and deletion policy, so the
|
the rule the access log follows above does not carry across that boundary. What
|
||||||
pattern is sent on every route, including the 2xx and 5xx responses outside the
|
is sent is the chi route pattern instead: `http://host/h/{uuid}`.
|
||||||
receiver for which the access log keeps the concrete path.
|
|
||||||
|
|
||||||
The scheme and the host are kept, and everything else in the URL is discarded
|
The scheme and the host are kept, and everything else in the URL is discarded
|
||||||
rather than edited, so a future SDK version that starts appending a query string
|
rather than edited, so a future SDK version that starts appending a query string
|
||||||
@@ -2553,9 +2526,8 @@ receiver route. What survives still names the failing route — scheme, host,
|
|||||||
route pattern, method — and `X-Request-Id` ties the event to the local access
|
route pattern, method — and `X-Request-Id` ties the event to the local access
|
||||||
log line that holds the rest. Nothing dropped is needed for the likeliest use,
|
log line that holds the rest. Nothing dropped is needed for the likeliest use,
|
||||||
debugging a CSRF rejection. Its three inputs are the TLS decision, `Origin` and
|
debugging a CSRF rejection. Its three inputs are the TLS decision, `Origin` and
|
||||||
`Referer`; the latter two are kept (`Referer` as its scheme and host, which is
|
`Referer`; the latter two are kept, and the first is the scheme of the retained
|
||||||
all the check compares), and the first is the scheme of the retained URL,
|
URL, because the SDK derives that scheme from
|
||||||
because the SDK derives that scheme from
|
|
||||||
`r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https"`. That predicate
|
`r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https"`. That predicate
|
||||||
is the SDK's own and is stricter than `internal/reqtls.IsTLS`, which this
|
is the SDK's own and is stricter than `internal/reqtls.IsTLS`, which this
|
||||||
service now uses everywhere it decides transport: the SDK reports `http` for the
|
service now uses everywhere it decides transport: the SDK reports `http` for the
|
||||||
@@ -2621,7 +2593,9 @@ than it:
|
|||||||
| ------------------------------------------ | ------- | ------------------- | ------------------------------------------ |
|
| ------------------------------------------ | ------- | ------------------- | ------------------------------------------ |
|
||||||
| `request body exceeds limit` (413) | `WARN` | path, method | yes — `MaxBodySize` precedes `RequireAuth` |
|
| `request body exceeds limit` (413) | `WARN` | path, method | yes — `MaxBodySize` precedes `RequireAuth` |
|
||||||
| `csrf: token validation failed` (403) | `WARN` | path, method | yes — `CSRF` precedes `RequireAuth` |
|
| `csrf: token validation failed` (403) | `WARN` | path, method | yes — `CSRF` precedes `RequireAuth` |
|
||||||
|
| `... rate limit exceeded` (429) | `WARN` | path | yes, on the receiver |
|
||||||
| `auth middleware: unauthenticated request` | `DEBUG` | path, method | yes, by definition |
|
| `auth middleware: unauthenticated request` | `DEBUG` | path, method | yes, by definition |
|
||||||
|
| `entrypoint not found` | `DEBUG` | entrypoint UUID | yes, on the receiver |
|
||||||
| `user not found` / `invalid password` | `DEBUG` | username | yes, on the sign-in form |
|
| `user not found` / `invalid password` | `DEBUG` | username | yes, on the sign-in form |
|
||||||
| `login failure limit exceeded` (429) | `WARN` | path | yes, on the sign-in form |
|
| `login failure limit exceeded` (429) | `WARN` | path | yes, on the sign-in form |
|
||||||
| `password verification capacity exhausted` | `WARN` | path | yes, on the sign-in form |
|
| `password verification capacity exhausted` | `WARN` | path | yes, on the sign-in form |
|
||||||
@@ -2649,12 +2623,13 @@ wider of the two handlers spends more on than the client spent sending them.
|
|||||||
Every case holds each line to the 2,560-byte ceiling. That per-line ceiling is
|
Every case holds each line to the 2,560-byte ceiling. That per-line ceiling is
|
||||||
what the figure above states, and every row establishes it.
|
what the figure above states, and every row establishes it.
|
||||||
|
|
||||||
Two of the sites go further and bound the whole flood's output — the total bytes
|
Three of the sites go further and bound the whole flood's output — the total
|
||||||
a run of distinct invented values wrote, which is the shape an operator sizing
|
bytes a run of distinct invented values wrote, which is the shape an operator
|
||||||
storage cares about. They are `request body exceeds limit`
|
sizing storage cares about. They are `request body exceeds limit`
|
||||||
(`TestMaxBodySize_FloodOfOversizePathsDoesNotGrowTheLog`) and `user not found`
|
(`TestMaxBodySize_FloodOfOversizePathsDoesNotGrowTheLog`),
|
||||||
(through `assertBoundedFlood`). The other rows carry no aggregate assertion; the
|
`entrypoint not found` and `user not found` (the last two through
|
||||||
per-line ceiling is what they establish.
|
`assertBoundedFlood`). The other rows carry no aggregate assertion; the per-line
|
||||||
|
ceiling is what they establish.
|
||||||
|
|
||||||
`internal/logfield/logfield_test.go` measures the per-rune charge against what
|
`internal/logfield/logfield_test.go` measures the per-rune charge against what
|
||||||
the handlers really emit, over roughly 3,000 code points on each, so an
|
the handlers really emit, over roughly 3,000 code points on each, so an
|
||||||
@@ -2672,7 +2647,7 @@ everything else, so its lines take the level the operator set and the handler
|
|||||||
`internal/logger` selected, and every value it emits is spent through the same
|
`internal/logger` selected, and every value it emits is spent through the same
|
||||||
`internal/logfield` budget. A record-not-found is not logged as an error: it is
|
`internal/logfield` budget. A record-not-found is not logged as an error: it is
|
||||||
the expected outcome on both of those paths, and each handler already records
|
the expected outcome on both of those paths, and each handler already records
|
||||||
its own miss at `DEBUG`, without the SQL and within the ceiling above. Slow
|
its own miss at `DEBUG` — bounded, per the table above — without the SQL. Slow
|
||||||
statements are kept, at `WARN`, above the same 200 ms threshold GORM used and
|
statements are kept, at `WARN`, above the same 200 ms threshold GORM used and
|
||||||
with the statement bounded, because that report is the one thing GORM's logger
|
with the statement bounded, because that report is the one thing GORM's logger
|
||||||
gave an operator that nothing else here does. The adapter orders its cases
|
gave an operator that nothing else here does. The adapter orders its cases
|
||||||
@@ -2997,12 +2972,6 @@ webhooker/
|
|||||||
├── cmd/webhooker/
|
├── cmd/webhooker/
|
||||||
│ └── main.go # Entry point: subcommand dispatch; no args locks DATA_DIR and wires fx
|
│ └── main.go # Entry point: subcommand dispatch; no args locks DATA_DIR and wires fx
|
||||||
├── internal/
|
├── internal/
|
||||||
│ ├── archive/
|
|
||||||
│ │ ├── archive.go # The archive writer of each database target, eviction, rename, and the rename lock
|
|
||||||
│ │ ├── writer.go # Archive file lifecycle and pruning
|
|
||||||
│ │ ├── rotation.go # Archive rotation and file names
|
|
||||||
│ │ ├── download.go # Archive download as gzipped JSON
|
|
||||||
│ │ └── sweeper.go # Periodic pruning of idle archives
|
|
||||||
│ ├── banner/
|
│ ├── banner/
|
||||||
│ │ └── banner.go # Ruled block for the one credential shown in the clear
|
│ │ └── banner.go # Ruled block for the one credential shown in the clear
|
||||||
│ ├── resetpw/
|
│ ├── resetpw/
|
||||||
@@ -3046,9 +3015,13 @@ webhooker/
|
|||||||
│ │ ├── target.go # Target interface, Task, Scheduler
|
│ │ ├── target.go # Target interface, Task, Scheduler
|
||||||
│ │ ├── target_http.go # HTTP target (retries, circuit breaker)
|
│ │ ├── target_http.go # HTTP target (retries, circuit breaker)
|
||||||
│ │ ├── target_slack.go # Slack/Mattermost incoming-webhook target
|
│ │ ├── target_slack.go # Slack/Mattermost incoming-webhook target
|
||||||
│ │ ├── target_database.go # Database archive target, writing through internal/archive
|
│ │ ├── target_database.go # Database archive target
|
||||||
|
│ │ ├── target_database_archive.go # Archive file lifecycle and pruning
|
||||||
|
│ │ ├── target_database_rotation.go # Archive rotation and file names
|
||||||
|
│ │ ├── target_database_export.go # Archive download as gzipped JSON
|
||||||
│ │ ├── target_log.go # Log target (stdout)
|
│ │ ├── target_log.go # Log target (stdout)
|
||||||
│ │ ├── target_config_view.go # Masked target config for templates
|
│ │ ├── target_config_view.go # Masked target config for templates
|
||||||
|
│ │ ├── archive_sweeper.go # Periodic pruning of idle archives
|
||||||
│ │ ├── queue_depth.go # Periodic sampler behind the queue-depth gauges
|
│ │ ├── queue_depth.go # Periodic sampler behind the queue-depth gauges
|
||||||
│ │ ├── url_mask.go # Strips credentials from *url.Error
|
│ │ ├── url_mask.go # Strips credentials from *url.Error
|
||||||
│ │ └── ssrf.go # SSRF prevention (IP validation, safe HTTP transport)
|
│ │ └── ssrf.go # SSRF prevention (IP validation, safe HTTP transport)
|
||||||
@@ -3089,8 +3062,6 @@ webhooker/
|
|||||||
│ │ ├── ratelimit.go # Per-IP rate limiting middleware (go-chi/httprate)
|
│ │ ├── ratelimit.go # Per-IP rate limiting middleware (go-chi/httprate)
|
||||||
│ │ ├── loginguard.go # Login failure counters and the Argon2id verification semaphore
|
│ │ ├── loginguard.go # Login failure counters and the Argon2id verification semaphore
|
||||||
│ │ └── middlewaretest/ # Test support: a Middleware for tests in other packages
|
│ │ └── middlewaretest/ # Test support: a Middleware for tests in other packages
|
||||||
│ ├── referer/
|
|
||||||
│ │ └── referer.go # SchemeAndHost: cuts a Referer down to its scheme and host, for the access log and the Sentry hook
|
|
||||||
│ ├── reqtls/
|
│ ├── reqtls/
|
||||||
│ │ └── reqtls.go # IsTLS: the one TLS predicate, r.TLS or X-Forwarded-Proto
|
│ │ └── reqtls.go # IsTLS: the one TLS predicate, r.TLS or X-Forwarded-Proto
|
||||||
│ ├── server/
|
│ ├── server/
|
||||||
@@ -3105,8 +3076,8 @@ webhooker/
|
|||||||
│ ├── static.go # //go:embed directive
|
│ ├── static.go # //go:embed directive
|
||||||
│ ├── css/input.css # Tailwind input, source for tailwind.css (make css)
|
│ ├── css/input.css # Tailwind input, source for tailwind.css (make css)
|
||||||
│ ├── css/tailwind.css # Generated stylesheet the pages load
|
│ ├── css/tailwind.css # Generated stylesheet the pages load
|
||||||
│ ├── css/style.css # Hand-written, loaded after tailwind.css: btn-small, the pointer cursor for input.css's buttons, the webhook list cards' focus outline, the rule hiding x-cloak elements
|
│ ├── css/style.css # Hand-written, loaded after tailwind.css: btn-small, the pointer cursor for input.css's buttons, the webhook list cards' focus outline
|
||||||
│ ├── js/app.js # Copy-to-clipboard, the delete prompts, and the Alpine.js components
|
│ ├── js/app.js # Copy-to-clipboard, and the Alpine.js components
|
||||||
│ └── js/alpine.min.js # Alpine.js CSP build, extracted from 3p/ by make assets, not committed
|
│ └── js/alpine.min.js # Alpine.js CSP build, extracted from 3p/ by make assets, not committed
|
||||||
├── templates/ # Go HTML templates (base, login, sources, etc.)
|
├── templates/ # Go HTML templates (base, login, sources, etc.)
|
||||||
├── script/ # Scripts to Rule Them All entrypoints
|
├── script/ # Scripts to Rule Them All entrypoints
|
||||||
@@ -3139,16 +3110,14 @@ Components are wired via Uber fx in this order:
|
|||||||
11. `metrics.New` — The delivery collectors, registered on that registry
|
11. `metrics.New` — The delivery collectors, registered on that registry
|
||||||
12. `middleware.New` — HTTP middleware
|
12. `middleware.New` — HTTP middleware
|
||||||
13. `delivery.New` — Event-driven delivery engine
|
13. `delivery.New` — Event-driven delivery engine
|
||||||
14. `archive.NewWriters` — The database targets' archive writers, and the lock
|
14. `delivery.NewArchiveSweeper` — Periodic pruning of idle archives
|
||||||
every rename of their files runs under
|
15. `delivery.Engine` → `delivery.Notifier` — interface bridge
|
||||||
15. `archive.NewSweeper` — Periodic pruning of idle archives
|
16. `delivery.Engine` → `delivery.Archives` — interface bridge so deleting or
|
||||||
16. `delivery.Engine` → `delivery.Notifier` — interface bridge
|
|
||||||
17. `archive.Writers` → `archive.Archives` — interface bridge so deleting or
|
|
||||||
renaming a webhook or target reaches its archive files
|
renaming a webhook or target reaches its archive files
|
||||||
18. `server.New` — HTTP server and router
|
17. `server.New` — HTTP server and router
|
||||||
|
|
||||||
The server starts via
|
The server starts via
|
||||||
`fx.Invoke(func(*server.Server, *delivery.Engine, *database.RetentionReaper, *archive.Sweeper) {})`,
|
`fx.Invoke(func(*server.Server, *delivery.Engine, *database.RetentionReaper, *delivery.ArchiveSweeper) {})`,
|
||||||
which triggers the fx lifecycle hooks in dependency order. The
|
which triggers the fx lifecycle hooks in dependency order. The
|
||||||
`delivery.Notifier` interface allows the webhook handler to send self-contained
|
`delivery.Notifier` interface allows the webhook handler to send self-contained
|
||||||
`delivery.Task` slices to the engine without a direct package dependency. Each
|
`delivery.Task` slices to the engine without a direct package dependency. Each
|
||||||
@@ -3162,9 +3131,8 @@ Applied to all routes in this order:
|
|||||||
|
|
||||||
1. **RequestID** — Generate unique request IDs (chi built-in)
|
1. **RequestID** — Generate unique request IDs (chi built-in)
|
||||||
2. **SecurityHeaders** — Production security headers on every response (HSTS
|
2. **SecurityHeaders** — Production security headers on every response (HSTS
|
||||||
unless `HSTS_MAX_AGE` is `0`, X-Content-Type-Options, X-Frame-Options, CSP
|
unless `HSTS_MAX_AGE` is `0`, X-Content-Type-Options, X-Frame-Options, CSP,
|
||||||
`default-src 'self'; script-src 'self'; style-src 'self'`, Referrer-Policy,
|
Referrer-Policy, Permissions-Policy)
|
||||||
Permissions-Policy)
|
|
||||||
3. **Logging** — Structured request logging (method, URL, status, latency,
|
3. **Logging** — Structured request logging (method, URL, status, latency,
|
||||||
remote IP, client IP, user agent, request ID)
|
remote IP, client IP, user agent, request ID)
|
||||||
4. **Metrics** — Prometheus HTTP metrics (if `METRICS_USERNAME` and
|
4. **Metrics** — Prometheus HTTP metrics (if `METRICS_USERNAME` and
|
||||||
@@ -3249,12 +3217,6 @@ credential check, see [The login endpoint](#the-login-endpoint).
|
|||||||
- Production security headers on all responses: HSTS (unless `HSTS_MAX_AGE` is
|
- Production security headers on all responses: HSTS (unless `HSTS_MAX_AGE` is
|
||||||
`0`), X-Content-Type-Options (`nosniff`), X-Frame-Options (`DENY`),
|
`0`), X-Content-Type-Options (`nosniff`), X-Frame-Options (`DENY`),
|
||||||
Content-Security-Policy, Referrer-Policy, and Permissions-Policy
|
Content-Security-Policy, Referrer-Policy, and Permissions-Policy
|
||||||
- The Content-Security-Policy is
|
|
||||||
`default-src 'self'; script-src 'self'; style-src 'self'`: a page takes
|
|
||||||
scripts and styles only from files webhooker serves, and the browser runs no
|
|
||||||
inline script or event handler and ignores any `<style>` element or `style`
|
|
||||||
attribute in a page. The pages need none: their scripts are in `static/js/`
|
|
||||||
and their styles in `static/css/`
|
|
||||||
- Request body size limits (1 MB) on all form POST endpoints, enforced by
|
- Request body size limits (1 MB) on all form POST endpoints, enforced by
|
||||||
middleware that runs before CSRF parses the form
|
middleware that runs before CSRF parses the form
|
||||||
- **CSRF protection** via [gorilla/csrf](https://github.com/gorilla/csrf) on all
|
- **CSRF protection** via [gorilla/csrf](https://github.com/gorilla/csrf) on all
|
||||||
@@ -3317,13 +3279,12 @@ order under a **5 second budget** (`fx.StopTimeout` in `cmd/webhooker/main.go`).
|
|||||||
That budget covers the whole sequence, not each hook. The order, read off the fx
|
That budget covers the whole sequence, not each hook. The order, read off the fx
|
||||||
stop-hook log:
|
stop-hook log:
|
||||||
|
|
||||||
1. `archive.Sweeper`
|
1. `ArchiveSweeper`
|
||||||
2. `RetentionReaper`
|
2. `RetentionReaper`
|
||||||
3. `server` — the HTTP drain, bounded by `server.ShutdownTimeout` (**3
|
3. `server` — the HTTP drain, bounded by `server.ShutdownTimeout` (**3
|
||||||
seconds**) and by what the hooks before it left, then a Sentry flush if
|
seconds**) and by what the hooks before it left, then a Sentry flush if
|
||||||
`SENTRY_DSN` is set
|
`SENTRY_DSN` is set
|
||||||
4. `delivery.Engine` — cuts off any `http` or `slack` request in flight, waits
|
4. `delivery.Engine` — waits for its workers, then closes the archive databases
|
||||||
for its workers, then closes the archive databases
|
|
||||||
5. `healthcheck`
|
5. `healthcheck`
|
||||||
6. `WebhookDBManager`
|
6. `WebhookDBManager`
|
||||||
7. the database close
|
7. the database close
|
||||||
@@ -3334,12 +3295,6 @@ by the stop context, so a wedge there consumes the 5 seconds before the HTTP
|
|||||||
server hook is ever entered. The hooks after the server are microsecond-scale in
|
server hook is ever entered. The hooks after the server are microsecond-scale in
|
||||||
normal operation.
|
normal operation.
|
||||||
|
|
||||||
The delivery engine cancels a delivery request in flight. That attempt is not
|
|
||||||
recorded or counted in the metrics, and the delivery keeps its `pending` or
|
|
||||||
`retrying` status, so startup recovery sends it again on the next start (see
|
|
||||||
Recovery paths under Bounded Worker Pool). If the receiver already had it, it
|
|
||||||
receives it twice: the at-least-once duplicate described under Rationale.
|
|
||||||
|
|
||||||
The HTTP drain budget is deliberately **shorter** than the sequence budget. Were
|
The HTTP drain budget is deliberately **shorter** than the sequence budget. Were
|
||||||
the two equal, a drain that used its whole budget would exhaust the sequence
|
the two equal, a drain that used its whole budget would exhaust the sequence
|
||||||
budget at the instant it finished, and every later hook — the delivery engine,
|
budget at the instant it finished, and every later hook — the delivery engine,
|
||||||
@@ -3360,7 +3315,7 @@ skipped when that leaves too little to be worth attempting — so a full-length
|
|||||||
drain means Sentry events are dropped rather than the database close being
|
drain means Sentry events are dropped rather than the database close being
|
||||||
skipped.
|
skipped.
|
||||||
|
|
||||||
This does not make the database close unconditional. A slow `archive.Sweeper` or
|
This does not make the database close unconditional. A slow `ArchiveSweeper` or
|
||||||
`RetentionReaper` is enough to cut the shutdown short, not only one that
|
`RetentionReaper` is enough to cut the shutdown short, not only one that
|
||||||
consumes the whole budget: what they spend comes out of the drain first, so
|
consumes the whole budget: what they spend comes out of the drain first, so
|
||||||
after 2 seconds of theirs a request still in flight gets 1 second to finish, and
|
after 2 seconds of theirs a request still in flight gets 1 second to finish, and
|
||||||
@@ -3462,12 +3417,16 @@ fixed independently of the compiler's:
|
|||||||
a step's log, and then fails. `make test` builds this stage alone.
|
a step's log, and then fails. `make test` builds this stage alone.
|
||||||
6. **Builder stage** (`golang:1.26.1-bookworm`) — depends on the lint and test
|
6. **Builder stage** (`golang:1.26.1-bookworm`) — depends on the lint and test
|
||||||
phases and the `css-check` and `markdown-check` stages passing (it copies a
|
phases and the `css-check` and `markdown-check` stages passing (it copies a
|
||||||
file from each), and runs `make build` (which extracts Alpine.js from `3p/`
|
file from each), runs `make build` (which extracts Alpine.js from `3p/`
|
||||||
first). `make build` compiles with cgo off, so the binary is statically
|
first), and then rebuilds the binary with `CGO_ENABLED=1` and static linking
|
||||||
linked and runs on the Alpine runtime image. The version is the `VERSION`
|
so it runs on musl, with the build tags `netgo`, `osusergo` and
|
||||||
build arg if one is given, otherwise derived from the `.git` in the context,
|
`sqlite_omit_load_extension` so it needs none of glibc's shared libraries at
|
||||||
and the stage fails if a context with `.git` would stamp an empty version,
|
run time. Both builds go through `make build`, the relink adding its
|
||||||
`dev` or `unknown` (see [Version stamping](#version-stamping)).
|
`-extldflags` via `GO_LDFLAGS` and its tags via `GO_TAGS`, so neither can
|
||||||
|
drop the `-X` that stamps the version. The version is the `VERSION` build arg
|
||||||
|
if one is given, otherwise derived from the `.git` in the context, and the
|
||||||
|
stage fails if a context with `.git` would stamp an empty version, `dev` or
|
||||||
|
`unknown` (see [Version stamping](#version-stamping)).
|
||||||
7. **Runtime stage** (`alpine:3.21`) — copies the static binary and
|
7. **Runtime stage** (`alpine:3.21`) — copies the static binary and
|
||||||
`deploy/docker-entrypoint.sh`, creates the `/var/lib/webhooker` directory for
|
`deploy/docker-entrypoint.sh`, creates the `/var/lib/webhooker` directory for
|
||||||
all SQLite databases, exposes port 8080, and includes a health check against
|
all SQLite databases, exposes port 8080, and includes a health check against
|
||||||
@@ -3479,6 +3438,11 @@ The lint and test phases invoke `gofmt`, `golangci-lint` and `go test` directly
|
|||||||
rather than `make fmt-check`, `make lint` and `make test`: those targets build
|
rather than `make fmt-check`, `make lint` and `make test`: those targets build
|
||||||
docker stages, which would need a docker daemon inside this build.
|
docker stages, which would need a docker daemon inside this build.
|
||||||
|
|
||||||
|
The lint phase and the Go stages use Debian rather than Alpine because
|
||||||
|
`gorm.io/driver/sqlite` pulls in `mattn/go-sqlite3`, which needs CGO and does
|
||||||
|
not compile against musl. Only the final binary is statically linked, which is
|
||||||
|
what lets it run on the Alpine runtime image.
|
||||||
|
|
||||||
`script/cibuild` is the CI gate: it runs `script/bootstrap`, then
|
`script/cibuild` is the CI gate: it runs `script/bootstrap`, then
|
||||||
`script/check`, then builds the image, whose build runs the lint and test phases
|
`script/check`, then builds the image, whose build runs the lint and test phases
|
||||||
and the stylesheet and Markdown checks again. A build that succeeds is a repo
|
and the stylesheet and Markdown checks again. A build that succeeds is a repo
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ import (
|
|||||||
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"go.uber.org/fx/fxevent"
|
"go.uber.org/fx/fxevent"
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/datadir"
|
"sneak.berlin/go/webhooker/internal/datadir"
|
||||||
@@ -50,7 +49,7 @@ import (
|
|||||||
// before the server may already have spent.
|
// before the server may already have spent.
|
||||||
//
|
//
|
||||||
// This does not make the database close unconditional: the
|
// This does not make the database close unconditional: the
|
||||||
// archive.Sweeper and RetentionReaper hooks run before the server.
|
// ArchiveSweeper and RetentionReaper hooks run before the server.
|
||||||
// What they spend comes out of the drain first, but past 3s it comes
|
// What they spend comes out of the drain first, but past 3s it comes
|
||||||
// out of the reserve, and they can consume the whole budget.
|
// out of the reserve, and they can consume the whole budget.
|
||||||
const stopTimeout = 5 * time.Second
|
const stopTimeout = 5 * time.Second
|
||||||
@@ -189,8 +188,8 @@ func newApp() *fx.App {
|
|||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
// The plain logger the session, the middleware, the
|
// The plain logger the session, the middleware and the
|
||||||
// webhook database manager and the archive writers take.
|
// webhook database manager take.
|
||||||
func(l *logger.Logger) *slog.Logger { return l.Get() },
|
func(l *logger.Logger) *slog.Logger { return l.Get() },
|
||||||
config.New,
|
config.New,
|
||||||
database.New,
|
database.New,
|
||||||
@@ -209,15 +208,14 @@ func newApp() *fx.App {
|
|||||||
// disagree about a destination.
|
// disagree about a destination.
|
||||||
delivery.NewGuard,
|
delivery.NewGuard,
|
||||||
delivery.New,
|
delivery.New,
|
||||||
archive.NewWriters,
|
delivery.NewArchiveSweeper,
|
||||||
archive.NewSweeper,
|
|
||||||
// Wire *delivery.Engine as delivery.Notifier so the
|
// Wire *delivery.Engine as delivery.Notifier so the
|
||||||
// webhook handler can notify the engine of new deliveries.
|
// webhook handler can notify the engine of new deliveries.
|
||||||
func(e *delivery.Engine) delivery.Notifier { return e },
|
func(e *delivery.Engine) delivery.Notifier { return e },
|
||||||
// Wire *archive.Writers as archive.Archives so deleting
|
// Wire *delivery.Engine as delivery.Archives so deleting
|
||||||
// or renaming a webhook or target reaches its archive
|
// or renaming a webhook or target reaches its archive
|
||||||
// files.
|
// files.
|
||||||
func(w *archive.Writers) archive.Archives { return w },
|
func(e *delivery.Engine) delivery.Archives { return e },
|
||||||
// Wire *delivery.Engine as delivery.CircuitBreakers so
|
// Wire *delivery.Engine as delivery.CircuitBreakers so
|
||||||
// the pages can show a target whose deliveries are
|
// the pages can show a target whose deliveries are
|
||||||
// paused.
|
// paused.
|
||||||
@@ -229,7 +227,7 @@ func newApp() *fx.App {
|
|||||||
*server.Server,
|
*server.Server,
|
||||||
*delivery.Engine,
|
*delivery.Engine,
|
||||||
*database.RetentionReaper,
|
*database.RetentionReaper,
|
||||||
*archive.Sweeper,
|
*delivery.ArchiveSweeper,
|
||||||
) {
|
) {
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -252,7 +252,7 @@ const tailHeadroom = 2 * time.Second
|
|||||||
// can produce, since a shorter drain leaves the flush more room and
|
// can produce, since a shorter drain leaves the flush more room and
|
||||||
// the worst case is not necessarily at either extreme.
|
// the worst case is not necessarily at either extreme.
|
||||||
//
|
//
|
||||||
// Nor does the hook start on a full budget: the archive.Sweeper and
|
// Nor does the hook start on a full budget: the ArchiveSweeper and
|
||||||
// RetentionReaper hooks run before it, and whatever they spent is
|
// RetentionReaper hooks run before it, and whatever they spent is
|
||||||
// gone. The outer sweep walks every amount they can spend. Once they
|
// gone. The outer sweep walks every amount they can spend. Once they
|
||||||
// have eaten into the headroom themselves, the hook must spend
|
// have eaten into the headroom themselves, the hook must spend
|
||||||
|
|||||||
@@ -1,454 +0,0 @@
|
|||||||
// Package archive manages the SQLite archive files the database target
|
|
||||||
// writes: how they are named and rotated, the writers that open, write,
|
|
||||||
// close and reopen them, pruning the rows past a target's expiry,
|
|
||||||
// renaming a target's files and moving them back when that fails, the
|
|
||||||
// gzipped JSON export, the idle sweep, and the locks that keep a rename
|
|
||||||
// apart from archive writes, from other edits and from exports.
|
|
||||||
package archive
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"log/slog"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"go.uber.org/fx"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// archiveNameMaxLen is how many characters of a webhook or target
|
|
||||||
// name an archive file name keeps.
|
|
||||||
const archiveNameMaxLen = 40
|
|
||||||
|
|
||||||
// Archives is how the handlers keep the database targets' archive
|
|
||||||
// files in step with the configuration. Deleting a webhook or a
|
|
||||||
// target releases the cached archive writers, whose open file
|
|
||||||
// handles would otherwise outlive them; renaming one renames the
|
|
||||||
// archive files, which are named for the webhook and the target
|
|
||||||
// (see FileName). RenameLock returns the lock every rename runs under
|
|
||||||
// (see Writers.RenameLock).
|
|
||||||
//
|
|
||||||
// It is an interface so that the handlers' tests can stand in for
|
|
||||||
// Writers.
|
|
||||||
//
|
|
||||||
// Neither eviction deletes an archive file. Both are idempotent
|
|
||||||
// and are no-ops for a webhook or target with no writer.
|
|
||||||
type Archives interface {
|
|
||||||
RenameLock() *sync.Mutex
|
|
||||||
EvictWebhook(webhookID string)
|
|
||||||
EvictTarget(targetID string)
|
|
||||||
Rename(targetID, webhookName, targetName string) error
|
|
||||||
}
|
|
||||||
|
|
||||||
// WritersParams holds the fx dependencies for Writers.
|
|
||||||
type WritersParams struct {
|
|
||||||
fx.In
|
|
||||||
|
|
||||||
Database *database.Database
|
|
||||||
DBManager *database.WebhookDBManager
|
|
||||||
Logger *slog.Logger
|
|
||||||
}
|
|
||||||
|
|
||||||
// Writers holds the archive writer of each database target, and the
|
|
||||||
// lock every rename of archive files runs under. The database target
|
|
||||||
// writes an event through it, the handlers rename and evict through
|
|
||||||
// it, the archive sweeper prunes through it, and the delivery engine
|
|
||||||
// evicts every writer when it stops. See archiveWriter for the
|
|
||||||
// close/reopen, auto-recreate, and expiry semantics.
|
|
||||||
type Writers struct {
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
log *slog.Logger
|
|
||||||
|
|
||||||
// writers holds one archive writer per database target, keyed
|
|
||||||
// by target ID.
|
|
||||||
mu sync.Mutex
|
|
||||||
writers map[string]*archiveWriter
|
|
||||||
|
|
||||||
// renameMu makes the webhook edit, the target edit and target
|
|
||||||
// creation run one at a time, each held from loading the stored
|
|
||||||
// names through the archive rename, the save and any move back.
|
|
||||||
// Interleaved, one could rename an archive between another's
|
|
||||||
// rename and save, leaving the file named for one edit and the
|
|
||||||
// stored names from the other. An archive download holds it while
|
|
||||||
// it reads the stored names and lists the files they give, and
|
|
||||||
// again for each file while it finds the file under the names
|
|
||||||
// stored then and opens it. RenameLock returns it.
|
|
||||||
renameMu sync.Mutex
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewWriters builds the archive writers. It opens no archive file:
|
|
||||||
// each target's writer is made on its first use.
|
|
||||||
func NewWriters(params WritersParams) *Writers {
|
|
||||||
return &Writers{
|
|
||||||
db: params.Database,
|
|
||||||
dbMgr: params.DBManager,
|
|
||||||
log: params.Logger,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// RenameLock implements Archives. It returns the lock every rename of
|
|
||||||
// archive files runs under: see renameMu for who holds it and for how
|
|
||||||
// long. Archive writes and Rename itself do not take it.
|
|
||||||
func (ws *Writers) RenameLock() *sync.Mutex {
|
|
||||||
return &ws.renameMu
|
|
||||||
}
|
|
||||||
|
|
||||||
// FileName returns the file name of a database target's
|
|
||||||
// archive: archive-WEBHOOKNAME-TARGETNAME-TARGETID.db, with both
|
|
||||||
// names passed through archiveNamePart. The target ID keeps the
|
|
||||||
// name unique when two targets' names come out the same.
|
|
||||||
func FileName(webhookName, targetName, targetID string) string {
|
|
||||||
return "archive-" + archiveNamePart(webhookName) + "-" +
|
|
||||||
archiveNamePart(targetName) + "-" + targetID + ".db"
|
|
||||||
}
|
|
||||||
|
|
||||||
// archiveNamePart makes a webhook or target name safe to put in a
|
|
||||||
// file name. It is lowercased; ASCII letters and digits are kept,
|
|
||||||
// every other run of characters becomes a single "-", and no "-" is
|
|
||||||
// left at either end. It is cut to archiveNameMaxLen characters, and
|
|
||||||
// a name with nothing left is "unnamed".
|
|
||||||
func archiveNamePart(name string) string {
|
|
||||||
var b strings.Builder
|
|
||||||
|
|
||||||
dash := false
|
|
||||||
|
|
||||||
for _, r := range strings.ToLower(name) {
|
|
||||||
if (r < 'a' || r > 'z') && (r < '0' || r > '9') {
|
|
||||||
dash = b.Len() > 0
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if dash {
|
|
||||||
b.WriteByte('-')
|
|
||||||
|
|
||||||
dash = false
|
|
||||||
}
|
|
||||||
|
|
||||||
b.WriteRune(r)
|
|
||||||
}
|
|
||||||
|
|
||||||
part := b.String()
|
|
||||||
if len(part) > archiveNameMaxLen {
|
|
||||||
part = strings.TrimRight(part[:archiveNameMaxLen], "-")
|
|
||||||
}
|
|
||||||
|
|
||||||
if part == "" {
|
|
||||||
return "unnamed"
|
|
||||||
}
|
|
||||||
|
|
||||||
return part
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write writes the full event of a delivery to a database target as
|
|
||||||
// a row into the target's archive database, honouring the optional
|
|
||||||
// per-target expiry and rotation parsed from the target config JSON.
|
|
||||||
// With rotation, the event goes to the file for the period of its
|
|
||||||
// receive time.
|
|
||||||
func (ws *Writers) Write(d *database.Delivery) error {
|
|
||||||
webhookID := d.Event.WebhookID
|
|
||||||
if webhookID == "" {
|
|
||||||
return errArchiveMissingWebhookID
|
|
||||||
}
|
|
||||||
|
|
||||||
expiry, err := ParseExpiry(d.Target.Config)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
rotation, err := ParseRotation(d.Target.Config)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// An event whose stored row was gone before its delivery ran has
|
|
||||||
// no receive time (see delivery's Engine.hydrateEvent), and goes
|
|
||||||
// to the file for now.
|
|
||||||
receivedAt := d.Event.CreatedAt
|
|
||||||
if receivedAt.IsZero() {
|
|
||||||
receivedAt = time.Now()
|
|
||||||
}
|
|
||||||
|
|
||||||
w, err := ws.writerFor(d.TargetID)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
row := archivedEvent{
|
|
||||||
EventID: d.Event.ID,
|
|
||||||
WebhookID: webhookID,
|
|
||||||
EntrypointID: d.Event.EntrypointID,
|
|
||||||
Method: d.Event.Method,
|
|
||||||
RawQuery: d.Event.RawQuery,
|
|
||||||
Headers: d.Event.Headers,
|
|
||||||
Body: d.Event.Body,
|
|
||||||
ContentType: d.Event.ContentType,
|
|
||||||
}
|
|
||||||
|
|
||||||
return w.write(row, expiry, archivePeriod(rotation, receivedAt))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rename implements Archives. It renames every one of a database
|
|
||||||
// target's archive files to FileName(webhookName, targetName,
|
|
||||||
// targetID), each keeping the period in its name. It goes through the
|
|
||||||
// target's writer, so the move holds the lock that the target's
|
|
||||||
// archive writes and the idle sweep take, and later writes use the
|
|
||||||
// new name. It never replaces a file: if one already has a new name,
|
|
||||||
// the error is ErrNameTaken.
|
|
||||||
//
|
|
||||||
// The writer is created if there is none, and it stays cached. The
|
|
||||||
// handlers rename before they save the new name, so until the save
|
|
||||||
// the main database still has the old one; a delivery in that window
|
|
||||||
// must find this writer rather than build one from the old name.
|
|
||||||
func (ws *Writers) Rename(
|
|
||||||
targetID, webhookName, targetName string,
|
|
||||||
) error {
|
|
||||||
w, err := ws.writerFor(targetID)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return w.rename(FileName(webhookName, targetName, targetID))
|
|
||||||
}
|
|
||||||
|
|
||||||
// EvictTarget implements Archives. It drops a database target's
|
|
||||||
// archive writer from the registry and closes its handle, so a
|
|
||||||
// deleted target does not leave a writer (and an open archive handle
|
|
||||||
// within its debounce window) alive for the process lifetime.
|
|
||||||
//
|
|
||||||
// The map entry is removed under the registry lock, which is
|
|
||||||
// then released before the handle is closed under the writer's
|
|
||||||
// own lock: that ordering keeps the registry available to other
|
|
||||||
// targets while an in-flight write on this one drains, and
|
|
||||||
// closing under the writer's lock means eviction can never race
|
|
||||||
// a write.
|
|
||||||
//
|
|
||||||
// Eviction is idempotent and silent for a target with no writer,
|
|
||||||
// which is the common case: only a database target that has
|
|
||||||
// received an event or been renamed has one. It never deletes the
|
|
||||||
// archive file.
|
|
||||||
func (ws *Writers) EvictTarget(targetID string) {
|
|
||||||
ws.mu.Lock()
|
|
||||||
|
|
||||||
w, ok := ws.writers[targetID]
|
|
||||||
if ok {
|
|
||||||
delete(ws.writers, targetID)
|
|
||||||
}
|
|
||||||
|
|
||||||
ws.mu.Unlock()
|
|
||||||
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
w.evict()
|
|
||||||
|
|
||||||
ws.log.Info(
|
|
||||||
"evicted archive writer",
|
|
||||||
"target_id", targetID,
|
|
||||||
"path", w.path,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// EvictWebhook implements Archives. It evicts, exactly as EvictTarget
|
|
||||||
// does, the writer of every database target of a webhook. The archive
|
|
||||||
// files themselves are left on disk — they are long-term storage the
|
|
||||||
// operator owns.
|
|
||||||
func (ws *Writers) EvictWebhook(webhookID string) {
|
|
||||||
ws.mu.Lock()
|
|
||||||
|
|
||||||
var gone []*archiveWriter
|
|
||||||
|
|
||||||
for targetID, w := range ws.writers {
|
|
||||||
if w.webhookID == webhookID {
|
|
||||||
delete(ws.writers, targetID)
|
|
||||||
|
|
||||||
gone = append(gone, w)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
ws.mu.Unlock()
|
|
||||||
|
|
||||||
for _, w := range gone {
|
|
||||||
w.evict()
|
|
||||||
|
|
||||||
ws.log.Info(
|
|
||||||
"evicted archive writer",
|
|
||||||
"webhook_id", webhookID,
|
|
||||||
"path", w.path,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// EvictAll evicts every cached archive writer, exactly as EvictTarget
|
|
||||||
// does for one target. The delivery engine calls it at shutdown, once
|
|
||||||
// its workers have returned. Closing the last handle on an archive
|
|
||||||
// moves the contents of its -wal into the .db and removes the
|
|
||||||
// -wal, so a clean stop leaves each archive as a single file.
|
|
||||||
func (ws *Writers) EvictAll() {
|
|
||||||
ws.mu.Lock()
|
|
||||||
|
|
||||||
writers := ws.writers
|
|
||||||
ws.writers = nil
|
|
||||||
|
|
||||||
ws.mu.Unlock()
|
|
||||||
|
|
||||||
for _, w := range writers {
|
|
||||||
w.evict()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// writerFor returns the archive writer for a database target,
|
|
||||||
// creating and caching it on first use. Each target has one writer
|
|
||||||
// so its close/reopen debounce state is shared across concurrent
|
|
||||||
// deliveries, and so a rename and the idle sweep take the same lock
|
|
||||||
// as its writes.
|
|
||||||
func (ws *Writers) writerFor(
|
|
||||||
targetID string,
|
|
||||||
) (*archiveWriter, error) {
|
|
||||||
ws.mu.Lock()
|
|
||||||
defer ws.mu.Unlock()
|
|
||||||
|
|
||||||
w, ok := ws.writers[targetID]
|
|
||||||
if !ok {
|
|
||||||
var err error
|
|
||||||
|
|
||||||
w, err = ws.newWriter(targetID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if ws.writers == nil {
|
|
||||||
ws.writers = make(map[string]*archiveWriter)
|
|
||||||
}
|
|
||||||
|
|
||||||
ws.writers[targetID] = w
|
|
||||||
}
|
|
||||||
|
|
||||||
// A delivery claims the entry: even if the idle sweep created
|
|
||||||
// it moments ago, it now belongs to the registry proper and
|
|
||||||
// the sweep must leave it in place when it finishes.
|
|
||||||
w.sweepOwned = false
|
|
||||||
|
|
||||||
return w, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// sweepWriterFor returns the archive writer the idle sweep should
|
|
||||||
// prune a target's archive through, together with whether the sweep
|
|
||||||
// itself created the registry entry.
|
|
||||||
//
|
|
||||||
// The sweep must route its prune through the registered writer so
|
|
||||||
// the writer's mutex orders it against concurrent writes, but it
|
|
||||||
// must never leave a registry entry behind: a sweep that ran
|
|
||||||
// concurrently with the target's deletion would otherwise
|
|
||||||
// re-create an entry that nothing will ever evict again, which is
|
|
||||||
// exactly the leak eviction exists to prevent. An entry the sweep
|
|
||||||
// creates is therefore marked sweep-owned and handed back to
|
|
||||||
// releaseSweepWriter when the sweep is done.
|
|
||||||
func (ws *Writers) sweepWriterFor(
|
|
||||||
targetID string,
|
|
||||||
) (*archiveWriter, bool, error) {
|
|
||||||
ws.mu.Lock()
|
|
||||||
defer ws.mu.Unlock()
|
|
||||||
|
|
||||||
w, ok := ws.writers[targetID]
|
|
||||||
if ok {
|
|
||||||
return w, false, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
w, err := ws.newWriter(targetID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if ws.writers == nil {
|
|
||||||
ws.writers = make(map[string]*archiveWriter)
|
|
||||||
}
|
|
||||||
|
|
||||||
w.sweepOwned = true
|
|
||||||
ws.writers[targetID] = w
|
|
||||||
|
|
||||||
return w, true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// releaseSweepWriter drops a registry entry that the idle sweep
|
|
||||||
// created, so a sweep leaves the registry exactly as it found it.
|
|
||||||
//
|
|
||||||
// The entry is removed only if it is still the very writer the
|
|
||||||
// sweep installed and no delivery has claimed it in the meantime
|
|
||||||
// (writerFor clears sweepOwned when it hands a writer to the
|
|
||||||
// write path). Both conditions are evaluated under the registry
|
|
||||||
// lock, so an eviction that raced the sweep — which removes the
|
|
||||||
// entry outright — simply finds nothing left to do here, and a
|
|
||||||
// delivery that adopted the writer keeps a registered, evictable
|
|
||||||
// one.
|
|
||||||
func (ws *Writers) releaseSweepWriter(
|
|
||||||
targetID string, w *archiveWriter,
|
|
||||||
) {
|
|
||||||
ws.mu.Lock()
|
|
||||||
defer ws.mu.Unlock()
|
|
||||||
|
|
||||||
cur, ok := ws.writers[targetID]
|
|
||||||
if !ok || cur != w || !cur.sweepOwned {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
delete(ws.writers, targetID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// newWriter builds the writer for a database target's archive. Its
|
|
||||||
// path is the one Path gives for the webhook and the target as
|
|
||||||
// the main database names them now; from then on only rename changes
|
|
||||||
// the name the writer uses. It does not touch the archive files.
|
|
||||||
func (ws *Writers) newWriter(
|
|
||||||
targetID string,
|
|
||||||
) (*archiveWriter, error) {
|
|
||||||
if ws.dbMgr == nil {
|
|
||||||
return nil, errArchiveNoDataDir
|
|
||||||
}
|
|
||||||
|
|
||||||
var target database.Target
|
|
||||||
|
|
||||||
err := ws.db.DB().
|
|
||||||
Preload("Webhook").
|
|
||||||
First(&target, "id = ?", targetID).Error
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"loading database target %s: %w", targetID, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
w := newArchiveWriter(
|
|
||||||
Path(ws.dbMgr, &target.Webhook, &target),
|
|
||||||
ws.log,
|
|
||||||
)
|
|
||||||
w.webhookID = target.WebhookID
|
|
||||||
|
|
||||||
return w, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// sweepArchive prunes one database target's archive of rows older
|
|
||||||
// than expiry, without requiring a write. A missing archive file is
|
|
||||||
// left missing (see sweepExpired), so a sweep never creates an
|
|
||||||
// archive for a target that has never received an event.
|
|
||||||
//
|
|
||||||
// It also never leaves a registry entry behind: an entry it had
|
|
||||||
// to create to reach the writer's mutex is released again once
|
|
||||||
// the prune is done, so a sweep racing a target deletion cannot
|
|
||||||
// resurrect the writer the eviction just dropped.
|
|
||||||
func (ws *Writers) sweepArchive(
|
|
||||||
targetID string, expiry time.Duration,
|
|
||||||
) error {
|
|
||||||
w, created, err := ws.sweepWriterFor(targetID)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if created {
|
|
||||||
defer ws.releaseSweepWriter(targetID, w)
|
|
||||||
}
|
|
||||||
|
|
||||||
return w.sweepExpired(expiry)
|
|
||||||
}
|
|
||||||
@@ -1,773 +0,0 @@
|
|||||||
package archive_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"database/sql"
|
|
||||||
"fmt"
|
|
||||||
"io/fs"
|
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/driver/sqlite"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// testContentType is the event content type used in tests.
|
|
||||||
testContentType = "application/json"
|
|
||||||
|
|
||||||
// eventQuery is the query string an archived event carries.
|
|
||||||
eventQuery = "a=1&b=2"
|
|
||||||
)
|
|
||||||
|
|
||||||
func archiveTestLogger() *slog.Logger {
|
|
||||||
return slog.New(slog.NewTextHandler(
|
|
||||||
os.Stderr,
|
|
||||||
&slog.HandlerOptions{Level: slog.LevelDebug},
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
// openArchiveDBForRead opens an archive file read-only so a
|
|
||||||
// test can inspect the rows the writer persisted.
|
|
||||||
func openArchiveDBForRead(
|
|
||||||
t *testing.T, path string,
|
|
||||||
) *gorm.DB {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
sqlDB, err := sql.Open(
|
|
||||||
"sqlite",
|
|
||||||
fmt.Sprintf("file:%s?mode=ro", path),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
|
||||||
|
|
||||||
gdb, err := gorm.Open(
|
|
||||||
sqlite.Dialector{Conn: sqlDB},
|
|
||||||
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
return gdb
|
|
||||||
}
|
|
||||||
|
|
||||||
// archiveFileSuffixes returns the archive file itself and the
|
|
||||||
// SQLite sidecars that accompany an open database. A test that
|
|
||||||
// asserts no archive was created has to check all of them.
|
|
||||||
func archiveFileSuffixes() []string {
|
|
||||||
return []string{"", "-wal", "-shm"}
|
|
||||||
}
|
|
||||||
|
|
||||||
// removeArchiveFiles simulates an operator moving the archive
|
|
||||||
// away by deleting the SQLite file and its sidecar files.
|
|
||||||
func removeArchiveFiles(t *testing.T, path string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for _, suffix := range []string{
|
|
||||||
"", "-wal", "-shm", "-journal",
|
|
||||||
} {
|
|
||||||
err := os.Remove(path + suffix)
|
|
||||||
if err != nil && !os.IsNotExist(err) {
|
|
||||||
t.Fatalf("removing %s%s: %v", path, suffix, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertStoredTime asserts that text, a timestamp as SQLite holds it,
|
|
||||||
// is in the form GORM stores created_at in: the layout of
|
|
||||||
// time.Time.String, with no " m=+…" monotonic clock reading after it.
|
|
||||||
func assertStoredTime(t *testing.T, text string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
_, err := time.Parse(
|
|
||||||
"2006-01-02 15:04:05.999999999 -0700 MST", text,
|
|
||||||
)
|
|
||||||
assert.NoError(t, err, "stored as %q", text)
|
|
||||||
}
|
|
||||||
|
|
||||||
// newEvent returns an event received now with the given body, as the
|
|
||||||
// webhook handler stores it.
|
|
||||||
func newEvent(body string) database.Event {
|
|
||||||
return database.Event{
|
|
||||||
BaseModel: database.BaseModel{
|
|
||||||
ID: uuid.New().String(),
|
|
||||||
CreatedAt: time.Now(),
|
|
||||||
},
|
|
||||||
WebhookID: uuid.New().String(),
|
|
||||||
EntrypointID: uuid.New().String(),
|
|
||||||
Method: http.MethodPost,
|
|
||||||
Headers: `{"Content-Type":["application/json"]}`,
|
|
||||||
Body: body,
|
|
||||||
ContentType: testContentType,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// databaseTargetDelivery returns a delivery of event to a database
|
|
||||||
// target, as the database target hands it to Writers.Write.
|
|
||||||
func databaseTargetDelivery(
|
|
||||||
event database.Event, tgt *database.Target,
|
|
||||||
) *database.Delivery {
|
|
||||||
return &database.Delivery{
|
|
||||||
EventID: event.ID,
|
|
||||||
TargetID: tgt.ID,
|
|
||||||
Event: event,
|
|
||||||
Target: database.Target{
|
|
||||||
Name: tgt.Name,
|
|
||||||
Type: database.TargetTypeDatabase,
|
|
||||||
Config: tgt.Config,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestArchiveWriter_WritesRow(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
|
||||||
w := archive.NewExportArchiveWriter(
|
|
||||||
path, archiveTestLogger(), 0,
|
|
||||||
)
|
|
||||||
|
|
||||||
row := archive.ExportArchivedEvent{
|
|
||||||
EventID: "ev-1",
|
|
||||||
WebhookID: "wh-1",
|
|
||||||
EntrypointID: "ep-1",
|
|
||||||
Method: "POST",
|
|
||||||
Headers: `{"X":"Y"}`,
|
|
||||||
Body: `{"hello":"world"}`,
|
|
||||||
ContentType: "application/json",
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, w.Write(row, 0))
|
|
||||||
assert.FileExists(t, path)
|
|
||||||
|
|
||||||
var got []archive.ExportArchivedEvent
|
|
||||||
|
|
||||||
require.NoError(t, w.DB().Find(&got).Error)
|
|
||||||
require.Len(t, got, 1)
|
|
||||||
assert.Equal(t, "ev-1", got[0].EventID)
|
|
||||||
assert.Equal(t, "wh-1", got[0].WebhookID)
|
|
||||||
assert.Equal(t, "ep-1", got[0].EntrypointID)
|
|
||||||
assert.Equal(t, row.Method, got[0].Method)
|
|
||||||
assert.Equal(t, row.ContentType, got[0].ContentType)
|
|
||||||
assert.JSONEq(t, `{"hello":"world"}`, got[0].Body)
|
|
||||||
assert.False(t, got[0].ArchivedAt.IsZero())
|
|
||||||
|
|
||||||
var archivedAt string
|
|
||||||
|
|
||||||
require.NoError(t, w.DB().Raw(
|
|
||||||
"SELECT CAST(archived_at AS TEXT) FROM archived_events",
|
|
||||||
).Row().Scan(&archivedAt))
|
|
||||||
assertStoredTime(t, archivedAt)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestArchiveWriter_RecreatesAfterRemoval(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
|
||||||
w := archive.NewExportArchiveWriter(
|
|
||||||
path, archiveTestLogger(), 0,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, w.Write(
|
|
||||||
archive.ExportArchivedEvent{EventID: "a"}, 0,
|
|
||||||
))
|
|
||||||
assert.FileExists(t, path)
|
|
||||||
|
|
||||||
// The operator moves the archive away while the handle is
|
|
||||||
// still open.
|
|
||||||
removeArchiveFiles(t, path)
|
|
||||||
require.NoFileExists(t, path)
|
|
||||||
|
|
||||||
// The next write recreates the file with a fresh schema and
|
|
||||||
// only the new row.
|
|
||||||
require.NoError(t, w.Write(
|
|
||||||
archive.ExportArchivedEvent{EventID: "b"}, 0,
|
|
||||||
))
|
|
||||||
assert.FileExists(t, path)
|
|
||||||
|
|
||||||
var got []archive.ExportArchivedEvent
|
|
||||||
|
|
||||||
require.NoError(t, w.DB().Find(&got).Error)
|
|
||||||
require.Len(t, got, 1)
|
|
||||||
assert.Equal(t, "b", got[0].EventID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveHandleIsWAL closes the last gap in the durability
|
|
||||||
// evidence: the main and per-webhook tiers each assert their journal
|
|
||||||
// mode on a live handle, and the archive tier gets its settings from
|
|
||||||
// the same code path but nothing checked the running file.
|
|
||||||
func TestArchiveHandleIsWAL(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
w := archive.NewExportArchiveWriter(
|
|
||||||
filepath.Join(t.TempDir(), "archive-wal.db"),
|
|
||||||
archiveTestLogger(), 0,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, w.Open(0))
|
|
||||||
|
|
||||||
var mode string
|
|
||||||
|
|
||||||
row := w.DB().Raw("pragma journal_mode").Row()
|
|
||||||
require.NoError(t, row.Scan(&mode))
|
|
||||||
assert.Equal(t, "wal", strings.ToLower(mode))
|
|
||||||
|
|
||||||
var busy string
|
|
||||||
|
|
||||||
row = w.DB().Raw("pragma busy_timeout").Row()
|
|
||||||
require.NoError(t, row.Scan(&busy))
|
|
||||||
assert.Equal(t, "10000", busy)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStatArchive proves Stat finds no file before the first
|
|
||||||
// write; after a write still held in the -wal, counts the -wal in the
|
|
||||||
// size and takes its later time as the last write; and finds no file
|
|
||||||
// again once the file has been moved away.
|
|
||||||
func TestStatArchive(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
|
||||||
|
|
||||||
_, err := archive.Stat(path)
|
|
||||||
require.ErrorIs(t, err, fs.ErrNotExist)
|
|
||||||
|
|
||||||
// With the clock stopped, the reopen debounce never passes, so
|
|
||||||
// the handle stays open after the write.
|
|
||||||
stopped := time.Now()
|
|
||||||
w := archive.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
|
||||||
w.SetNow(func() time.Time { return stopped })
|
|
||||||
|
|
||||||
require.NoError(t, w.Write(archive.ExportArchivedEvent{EventID: "a"}, 0))
|
|
||||||
|
|
||||||
written := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
|
||||||
earlier := written.Add(-time.Hour)
|
|
||||||
require.NoError(t, os.Chtimes(path, earlier, earlier))
|
|
||||||
require.NoError(t, os.Chtimes(path+"-wal", written, written))
|
|
||||||
|
|
||||||
file, err := os.Stat(path)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
wal, err := os.Stat(path + "-wal")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Positive(t, wal.Size())
|
|
||||||
|
|
||||||
got, err := archive.Stat(path)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, 1, got.Files)
|
|
||||||
assert.Equal(t, file.Size()+wal.Size(), got.Size)
|
|
||||||
assert.True(t, written.Equal(got.Written), got.Written)
|
|
||||||
|
|
||||||
removeArchiveFiles(t, path)
|
|
||||||
|
|
||||||
_, err = archive.Stat(path)
|
|
||||||
require.ErrorIs(t, err, fs.ErrNotExist)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestArchiveWriter_ReopenDebounce(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const debounce = 2 * time.Second
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
|
||||||
w := archive.NewExportArchiveWriter(
|
|
||||||
path, archiveTestLogger(), debounce,
|
|
||||||
)
|
|
||||||
|
|
||||||
// The writer measures its reopen debounce on this clock, which
|
|
||||||
// only the test moves, so how long the host takes between
|
|
||||||
// writes cannot change the result.
|
|
||||||
now := time.Now()
|
|
||||||
|
|
||||||
w.SetNow(func() time.Time { return now })
|
|
||||||
|
|
||||||
require.NoError(t, w.Write(
|
|
||||||
archive.ExportArchivedEvent{EventID: "a"}, 0,
|
|
||||||
))
|
|
||||||
require.NoError(t, w.Write(
|
|
||||||
archive.ExportArchivedEvent{EventID: "b"}, 0,
|
|
||||||
))
|
|
||||||
|
|
||||||
// Two writes inside the debounce window trigger only the
|
|
||||||
// initial open — no extra close/reopen.
|
|
||||||
assert.Equal(t, 1, w.Reopens())
|
|
||||||
|
|
||||||
now = now.Add(debounce)
|
|
||||||
|
|
||||||
require.NoError(t, w.Write(
|
|
||||||
archive.ExportArchivedEvent{EventID: "c"}, 0,
|
|
||||||
))
|
|
||||||
|
|
||||||
// A write after the window elapses closes and reopens once.
|
|
||||||
assert.Equal(t, 2, w.Reopens())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestArchiveWriter_ExpiryPrune(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
|
||||||
w := archive.NewExportArchiveWriter(
|
|
||||||
path, archiveTestLogger(), 0,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, w.Open(0))
|
|
||||||
|
|
||||||
old := archive.ExportArchivedEvent{
|
|
||||||
EventID: "old",
|
|
||||||
ArchivedAt: time.Now().Add(-2 * time.Hour),
|
|
||||||
}
|
|
||||||
fresh := archive.ExportArchivedEvent{
|
|
||||||
EventID: "fresh",
|
|
||||||
ArchivedAt: time.Now(),
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, w.DB().Create(&old).Error)
|
|
||||||
require.NoError(t, w.DB().Create(&fresh).Error)
|
|
||||||
|
|
||||||
// Reopening with a one-hour expiry prunes the old row.
|
|
||||||
require.NoError(t, w.Reopen(time.Hour))
|
|
||||||
|
|
||||||
var got []archive.ExportArchivedEvent
|
|
||||||
|
|
||||||
require.NoError(t, w.DB().Find(&got).Error)
|
|
||||||
require.Len(t, got, 1)
|
|
||||||
assert.Equal(t, "fresh", got[0].EventID)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParseArchiveExpiry(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
in string
|
|
||||||
want time.Duration
|
|
||||||
wantErr bool
|
|
||||||
}{
|
|
||||||
{"empty config", "", 0, false},
|
|
||||||
{"explicit never", `{"expiry":"never"}`, 0, false},
|
|
||||||
{"empty expiry", `{"expiry":""}`, 0, false},
|
|
||||||
{"duration", `{"expiry":"1h"}`, time.Hour, false},
|
|
||||||
{"unparseable", `{"expiry":"nonsense"}`, 0, true},
|
|
||||||
{"zero duration", `{"expiry":"0s"}`, 0, true},
|
|
||||||
{"negative duration", `{"expiry":"-5h"}`, 0, true},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
got, err := archive.ParseExpiry(tc.in)
|
|
||||||
if tc.wantErr {
|
|
||||||
require.Error(t, err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, tc.want, got)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateArchiveExpiry(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
valid := []string{"", "never", "1h", "720h", "30m"}
|
|
||||||
for _, in := range valid {
|
|
||||||
require.NoError(t,
|
|
||||||
archive.ValidateExpiry(in),
|
|
||||||
"expiry %q should be accepted", in,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
invalid := []string{"nonsense", "7d", "-5h", "0s", "0"}
|
|
||||||
for _, in := range invalid {
|
|
||||||
require.Error(t,
|
|
||||||
archive.ValidateExpiry(in),
|
|
||||||
"expiry %q should be rejected", in,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveFileName pins the archive file name and the rules
|
|
||||||
// that make a webhook or target name safe to put in it.
|
|
||||||
func TestArchiveFileName(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const id = "3f2a1c9e-8d4b-4c1a-9e2f-0a1b2c3d4e5f"
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
webhook string
|
|
||||||
target string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
"plain names", "orders", "archive",
|
|
||||||
"archive-orders-archive-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"lowercased", "Orders", "Main Archive",
|
|
||||||
"archive-orders-main-archive-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"a run of other characters is one dash",
|
|
||||||
`a /\..b`, "c__--d",
|
|
||||||
"archive-a-b-c-d-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"no dash at either end", " --orders!! ", "(archive)",
|
|
||||||
"archive-orders-archive-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"path separators", "../../etc/passwd", "a/b",
|
|
||||||
"archive-etc-passwd-a-b-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"letters outside ASCII are dropped",
|
|
||||||
"Bestellungen Größe", "café",
|
|
||||||
"archive-bestellungen-gr-e-caf-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"nothing left is unnamed", "", "!!!",
|
|
||||||
"archive-unnamed-unnamed-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cut to 40 characters", strings.Repeat("a", 50), "x",
|
|
||||||
"archive-" + strings.Repeat("a", 40) + "-x-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"no dash left by the cut",
|
|
||||||
strings.Repeat("a", 39) + " b", "x",
|
|
||||||
"archive-" + strings.Repeat("a", 39) + "-x-" + id + ".db",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t, tc.want,
|
|
||||||
archive.FileName(tc.webhook, tc.target, id),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWrite_EachTargetHasItsOwnArchive proves two
|
|
||||||
// database targets of one webhook archive into separate files.
|
|
||||||
func TestWrite_EachTargetHasItsOwnArchive(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
first := env.seedDatabaseTarget(t, "")
|
|
||||||
second := env.addDatabaseTarget(t, first.WebhookID, "")
|
|
||||||
|
|
||||||
event := newEvent(`{"n":1}`)
|
|
||||||
|
|
||||||
for _, tgt := range []*database.Target{first, second} {
|
|
||||||
require.NoError(t, env.writers.Write(
|
|
||||||
databaseTargetDelivery(event, tgt),
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NotEqual(
|
|
||||||
t, env.archivePath(first), env.archivePath(second),
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t, []string{event.ID},
|
|
||||||
archivedEventIDs(t, env.archivePath(first)),
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t, []string{event.ID},
|
|
||||||
archivedEventIDs(t, env.archivePath(second)),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRename_MovesTheFile proves a rename moves the archive, rows
|
|
||||||
// and all, and that later writes go to the new name.
|
|
||||||
func TestRename_MovesTheFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, "")
|
|
||||||
oldPath := env.archivePath(tgt)
|
|
||||||
|
|
||||||
first := newEvent(`{"n":1}`)
|
|
||||||
require.NoError(t, env.writers.Write(databaseTargetDelivery(first, tgt)))
|
|
||||||
require.FileExists(t, oldPath)
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t, env.writers.Rename(tgt.ID, "Orders", "Long Term"),
|
|
||||||
)
|
|
||||||
|
|
||||||
newPath := filepath.Join(
|
|
||||||
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.NoFileExists(t, oldPath)
|
|
||||||
assert.Equal(t, []string{first.ID}, archivedEventIDs(t, newPath))
|
|
||||||
|
|
||||||
second := newEvent(`{"n":2}`)
|
|
||||||
require.NoError(t, env.writers.Write(databaseTargetDelivery(second, tgt)))
|
|
||||||
|
|
||||||
assert.ElementsMatch(
|
|
||||||
t, []string{first.ID, second.ID},
|
|
||||||
archivedEventIDs(t, newPath),
|
|
||||||
)
|
|
||||||
assert.NoFileExists(
|
|
||||||
t, oldPath, "a write after the rename must use the new name",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRename_NeverReplacesAFile plants a file at the new name, once
|
|
||||||
// the .db alone, once a lone -wal and once a lone -shm, and proves
|
|
||||||
// each time that the rename is refused, the planted file survives,
|
|
||||||
// and the archive keeps its name and its rows.
|
|
||||||
func TestRename_NeverReplacesAFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, suffix := range archiveFileSuffixes() {
|
|
||||||
t.Run("planted .db"+suffix, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, "")
|
|
||||||
oldPath := env.archivePath(tgt)
|
|
||||||
|
|
||||||
first := newEvent(`{"n":1}`)
|
|
||||||
require.NoError(
|
|
||||||
t, env.writers.Write(databaseTargetDelivery(first, tgt)),
|
|
||||||
)
|
|
||||||
|
|
||||||
newPath := filepath.Join(
|
|
||||||
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
|
||||||
)
|
|
||||||
plantedPath := newPath + suffix
|
|
||||||
require.NoError(
|
|
||||||
t, os.WriteFile(plantedPath, []byte("planted"), 0o600),
|
|
||||||
)
|
|
||||||
|
|
||||||
require.ErrorIs(
|
|
||||||
t, env.writers.Rename(tgt.ID, "Orders", "Long Term"),
|
|
||||||
archive.ErrNameTaken,
|
|
||||||
)
|
|
||||||
|
|
||||||
//nolint:gosec // reads the file the test planted under t.TempDir()
|
|
||||||
planted, err := os.ReadFile(plantedPath)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "planted", string(planted))
|
|
||||||
|
|
||||||
second := newEvent(`{"n":2}`)
|
|
||||||
require.NoError(
|
|
||||||
t, env.writers.Write(databaseTargetDelivery(second, tgt)),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.ElementsMatch(
|
|
||||||
t, []string{first.ID, second.ID},
|
|
||||||
archivedEventIDs(t, oldPath),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRename_BeforeTheNameIsSaved covers the order the handlers
|
|
||||||
// use: they rename before they save the new name, so a delivery in
|
|
||||||
// between must write under the new name although the main database
|
|
||||||
// still has the old one. It also shows that renaming an archive that
|
|
||||||
// does not exist yet is not an error.
|
|
||||||
func TestRename_BeforeTheNameIsSaved(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, "")
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t, env.writers.Rename(tgt.ID, "Orders", "Archive"),
|
|
||||||
)
|
|
||||||
|
|
||||||
event := newEvent(`{"n":1}`)
|
|
||||||
require.NoError(t, env.writers.Write(databaseTargetDelivery(event, tgt)))
|
|
||||||
|
|
||||||
assert.FileExists(
|
|
||||||
t,
|
|
||||||
filepath.Join(
|
|
||||||
env.dataDir, "archive-orders-archive-"+tgt.ID+".db",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
assert.NoFileExists(t, env.archivePath(tgt))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveWriter_RenameMovesSidecars proves a rename carries
|
|
||||||
// the -wal and -shm a crash can leave beside an archive no handle
|
|
||||||
// has opened since. SQLite finds them by name, so a -wal left
|
|
||||||
// behind would lose the transactions it holds.
|
|
||||||
func TestArchiveWriter_RenameMovesSidecars(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
oldPath := filepath.Join(dir, "archive-old.db")
|
|
||||||
newPath := filepath.Join(dir, "archive-new.db")
|
|
||||||
|
|
||||||
for _, suffix := range archiveFileSuffixes() {
|
|
||||||
require.NoError(
|
|
||||||
t, os.WriteFile(oldPath+suffix, []byte(suffix), 0o600),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
w := archive.NewExportArchiveWriter(
|
|
||||||
oldPath, archiveTestLogger(), 0,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, w.Rename("archive-new.db"))
|
|
||||||
|
|
||||||
for _, suffix := range archiveFileSuffixes() {
|
|
||||||
assert.NoFileExists(t, oldPath+suffix)
|
|
||||||
assert.FileExists(t, newPath+suffix)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, newPath, w.Path())
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveWriter_RenameMovesBackOnFailure makes the -wal fail to
|
|
||||||
// move after the .db has moved, and proves the .db is moved back, so
|
|
||||||
// the archive is never split across two names. The new name is 255
|
|
||||||
// bytes, the longest a file name may be, so the .db can take it but
|
|
||||||
// the -wal, four bytes longer, cannot.
|
|
||||||
func TestArchiveWriter_RenameMovesBackOnFailure(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
oldPath := filepath.Join(dir, "archive-old.db")
|
|
||||||
newName := strings.Repeat("a", 252) + ".db"
|
|
||||||
|
|
||||||
for _, suffix := range archiveFileSuffixes() {
|
|
||||||
require.NoError(
|
|
||||||
t, os.WriteFile(oldPath+suffix, []byte(suffix), 0o600),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
w := archive.NewExportArchiveWriter(
|
|
||||||
oldPath, archiveTestLogger(), 0,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.Error(t, w.Rename(newName))
|
|
||||||
|
|
||||||
for _, suffix := range archiveFileSuffixes() {
|
|
||||||
assert.FileExists(t, oldPath+suffix)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.NoFileExists(t, filepath.Join(dir, newName))
|
|
||||||
assert.Equal(t, oldPath, w.Path())
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveWriter_RenameMovesBackEveryFile renames a target with a
|
|
||||||
// file without a period and a file for a month, and proves that when
|
|
||||||
// the month's file fails to move, the file already moved is moved back:
|
|
||||||
// both files are under the old name with their rows, and nothing is
|
|
||||||
// under the new name. The new name is 251 bytes, so the file without a
|
|
||||||
// period, with its -wal and -shm, can take it, but the month's file,
|
|
||||||
// eight bytes longer, cannot.
|
|
||||||
func TestArchiveWriter_RenameMovesBackEveryFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
oldPath := filepath.Join(dir, "archive-old.db")
|
|
||||||
monthPath := filepath.Join(dir, "archive-old-2026-03.db")
|
|
||||||
w := archive.NewExportArchiveWriter(oldPath, archiveTestLogger(), 0)
|
|
||||||
|
|
||||||
require.NoError(t, w.WritePeriod(
|
|
||||||
archive.ExportArchivedEvent{EventID: "in-none"}, 0, "",
|
|
||||||
))
|
|
||||||
require.NoError(t, w.WritePeriod(
|
|
||||||
archive.ExportArchivedEvent{EventID: "in-month"}, 0, "2026-03",
|
|
||||||
))
|
|
||||||
|
|
||||||
require.Error(t, w.Rename(strings.Repeat("a", 248)+".db"))
|
|
||||||
|
|
||||||
assert.Equal(t, []string{"in-none"}, archivedEventIDs(t, oldPath))
|
|
||||||
assert.Equal(t, []string{"in-month"}, archivedEventIDs(t, monthPath))
|
|
||||||
|
|
||||||
entries, err := os.ReadDir(dir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
for _, entry := range entries {
|
|
||||||
assert.True(t, strings.HasPrefix(entry.Name(), "archive-old"),
|
|
||||||
"%s is not under the old name", entry.Name())
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, oldPath, w.Path())
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRenameLock_HeldByOneEditAtATime holds the rename lock, as an edit
|
|
||||||
// does from loading the stored names until it has saved them. While it
|
|
||||||
// is held, the edit's rename and an archive write still finish, since
|
|
||||||
// neither takes it, and the event ends up under the new name whichever
|
|
||||||
// ran first. A second edit waits for the lock until the first lets go.
|
|
||||||
func TestRenameLock_HeldByOneEditAtATime(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, "")
|
|
||||||
event := newEvent(`{"n":1}`)
|
|
||||||
|
|
||||||
env.writers.RenameLock().Lock()
|
|
||||||
|
|
||||||
finished := make(chan error, 2)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
finished <- env.writers.Rename(tgt.ID, "Orders", "Long Term")
|
|
||||||
}()
|
|
||||||
go func() {
|
|
||||||
finished <- env.writers.Write(databaseTargetDelivery(event, tgt))
|
|
||||||
}()
|
|
||||||
|
|
||||||
for range 2 {
|
|
||||||
select {
|
|
||||||
case err := <-finished:
|
|
||||||
require.NoError(t, err)
|
|
||||||
case <-time.After(10 * time.Second):
|
|
||||||
t.Fatal("a rename or a write waited for the rename lock")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
newPath := filepath.Join(
|
|
||||||
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
|
||||||
)
|
|
||||||
assert.Equal(t, []string{event.ID}, archivedEventIDs(t, newPath))
|
|
||||||
|
|
||||||
secondEdit := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
env.writers.RenameLock().Lock()
|
|
||||||
close(secondEdit)
|
|
||||||
env.writers.RenameLock().Unlock()
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-secondEdit:
|
|
||||||
t.Fatal("a second edit took the rename lock while it was held")
|
|
||||||
case <-time.After(100 * time.Millisecond):
|
|
||||||
}
|
|
||||||
|
|
||||||
env.writers.RenameLock().Unlock()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-secondEdit:
|
|
||||||
case <-time.After(10 * time.Second):
|
|
||||||
t.Fatal("a second edit did not get the rename lock once released")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,280 +0,0 @@
|
|||||||
package archive
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"log/slog"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"go.uber.org/fx"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ErrExportArchiveWriterEvicted exposes the sentinel returned by
|
|
||||||
// an evicted archive writer. It carries the Err prefix rather
|
|
||||||
// than this file's usual Export one because it is a sentinel
|
|
||||||
// error.
|
|
||||||
var ErrExportArchiveWriterEvicted = errArchiveWriterEvicted
|
|
||||||
|
|
||||||
// ExportArchivedEvent aliases the archive row type so black-box
|
|
||||||
// tests can construct and read archive rows.
|
|
||||||
type ExportArchivedEvent = archivedEvent
|
|
||||||
|
|
||||||
// ExportArchiveWriter wraps an archiveWriter so black-box tests
|
|
||||||
// can exercise the archive file mechanics.
|
|
||||||
type ExportArchiveWriter struct {
|
|
||||||
w *archiveWriter
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewExportArchiveWriter builds an archive writer for tests,
|
|
||||||
// optionally overriding the reopen debounce (a non-positive
|
|
||||||
// debounce keeps the production default).
|
|
||||||
func NewExportArchiveWriter(
|
|
||||||
path string, log *slog.Logger, debounce time.Duration,
|
|
||||||
) *ExportArchiveWriter {
|
|
||||||
w := newArchiveWriter(path, log)
|
|
||||||
if debounce > 0 {
|
|
||||||
w.debounce = debounce
|
|
||||||
}
|
|
||||||
|
|
||||||
return &ExportArchiveWriter{w: w}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Write archives a row through the writer, into the file named
|
|
||||||
// without a period.
|
|
||||||
func (e *ExportArchiveWriter) Write(
|
|
||||||
row ExportArchivedEvent, expiry time.Duration,
|
|
||||||
) error {
|
|
||||||
return e.w.write(row, expiry, "")
|
|
||||||
}
|
|
||||||
|
|
||||||
// WritePeriod archives a row through the writer, into the file for
|
|
||||||
// period.
|
|
||||||
func (e *ExportArchiveWriter) WritePeriod(
|
|
||||||
row ExportArchivedEvent, expiry time.Duration, period string,
|
|
||||||
) error {
|
|
||||||
return e.w.write(row, expiry, period)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Open opens the archive file, pruning when expiry is positive.
|
|
||||||
func (e *ExportArchiveWriter) Open(expiry time.Duration) error {
|
|
||||||
return e.w.open(e.w.path, expiry)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reopen closes and reopens the archive file.
|
|
||||||
func (e *ExportArchiveWriter) Reopen(
|
|
||||||
expiry time.Duration,
|
|
||||||
) error {
|
|
||||||
return e.w.reopen(e.w.path, expiry)
|
|
||||||
}
|
|
||||||
|
|
||||||
// SetNow replaces the clock the writer measures its reopen
|
|
||||||
// debounce on.
|
|
||||||
func (e *ExportArchiveWriter) SetNow(now func() time.Time) {
|
|
||||||
e.w.now = now
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reopens reports how many times the file has been opened.
|
|
||||||
func (e *ExportArchiveWriter) Reopens() int {
|
|
||||||
return e.w.reopens
|
|
||||||
}
|
|
||||||
|
|
||||||
// DB returns the writer's current open handle for row
|
|
||||||
// inspection in tests.
|
|
||||||
func (e *ExportArchiveWriter) DB() *gorm.DB {
|
|
||||||
return e.w.db
|
|
||||||
}
|
|
||||||
|
|
||||||
// Path returns the archive file the writer owns.
|
|
||||||
func (e *ExportArchiveWriter) Path() string {
|
|
||||||
return e.w.path
|
|
||||||
}
|
|
||||||
|
|
||||||
// OpenExisting opens the archive without permitting creation,
|
|
||||||
// the way the idle sweep does.
|
|
||||||
func (e *ExportArchiveWriter) OpenExisting(
|
|
||||||
expiry time.Duration,
|
|
||||||
) error {
|
|
||||||
return e.w.openMode(e.w.path, archiveModeExisting, expiry)
|
|
||||||
}
|
|
||||||
|
|
||||||
// SweepExpired runs an idle sweep of the archive.
|
|
||||||
func (e *ExportArchiveWriter) SweepExpired(
|
|
||||||
expiry time.Duration,
|
|
||||||
) error {
|
|
||||||
return e.w.sweepExpired(expiry)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Evict marks the writer evicted and closes its handle, exactly
|
|
||||||
// as leaving the registry does.
|
|
||||||
func (e *ExportArchiveWriter) Evict() {
|
|
||||||
e.w.evict()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rename gives the archive file a new name in the same directory,
|
|
||||||
// as a rename of the webhook or target does.
|
|
||||||
func (e *ExportArchiveWriter) Rename(name string) error {
|
|
||||||
return e.w.rename(name)
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleOpen reports whether the writer currently holds an open
|
|
||||||
// archive handle.
|
|
||||||
func (e *ExportArchiveWriter) HandleOpen() bool {
|
|
||||||
e.w.mu.Lock()
|
|
||||||
defer e.w.mu.Unlock()
|
|
||||||
|
|
||||||
return e.w.db != nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Same reports whether both wrappers refer to the very same
|
|
||||||
// underlying archive writer, so a test can prove a registry entry
|
|
||||||
// is the writer it was handed rather than a replacement.
|
|
||||||
func (e *ExportArchiveWriter) Same(
|
|
||||||
other *ExportArchiveWriter,
|
|
||||||
) bool {
|
|
||||||
return other != nil && e.w == other.w
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportArchiveWriterFor returns the archive writer the registry
|
|
||||||
// currently caches for a database target, or nil when none is
|
|
||||||
// cached. It never creates one, so a test can hold a reference to
|
|
||||||
// the very writer an eviction is about to detach.
|
|
||||||
func (ws *Writers) ExportArchiveWriterFor(
|
|
||||||
targetID string,
|
|
||||||
) *ExportArchiveWriter {
|
|
||||||
ws.mu.Lock()
|
|
||||||
defer ws.mu.Unlock()
|
|
||||||
|
|
||||||
w, ok := ws.writers[targetID]
|
|
||||||
if !ok {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return &ExportArchiveWriter{w: w}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportHasArchiveWriter reports whether the registry currently
|
|
||||||
// caches an archive writer for a target.
|
|
||||||
func (ws *Writers) ExportHasArchiveWriter(
|
|
||||||
targetID string,
|
|
||||||
) bool {
|
|
||||||
ws.mu.Lock()
|
|
||||||
defer ws.mu.Unlock()
|
|
||||||
|
|
||||||
_, ok := ws.writers[targetID]
|
|
||||||
|
|
||||||
return ok
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportArchiveHandleOpen reports whether the cached archive
|
|
||||||
// writer for a target holds an open database handle. It
|
|
||||||
// returns false when no writer is cached.
|
|
||||||
func (ws *Writers) ExportArchiveHandleOpen(
|
|
||||||
targetID string,
|
|
||||||
) bool {
|
|
||||||
ws.mu.Lock()
|
|
||||||
w, ok := ws.writers[targetID]
|
|
||||||
ws.mu.Unlock()
|
|
||||||
|
|
||||||
if !ok {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
w.mu.Lock()
|
|
||||||
defer w.mu.Unlock()
|
|
||||||
|
|
||||||
return w.db != nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportEnsureArchiveWriter creates (if needed) and returns the
|
|
||||||
// archive file path of the cached writer for a target, so a
|
|
||||||
// test can prime the registry the way a delivery would.
|
|
||||||
func (ws *Writers) ExportEnsureArchiveWriter(
|
|
||||||
targetID string,
|
|
||||||
) (string, error) {
|
|
||||||
w, err := ws.writerFor(targetID)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
return w.path, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportSweepWriterFor takes a target's registry writer exactly
|
|
||||||
// as the idle sweep does, reporting whether the sweep had to
|
|
||||||
// create the entry. It lets a test drive the registry through the
|
|
||||||
// sweep's own entry point instead of choreographing goroutines.
|
|
||||||
func (ws *Writers) ExportSweepWriterFor(
|
|
||||||
targetID string,
|
|
||||||
) (*ExportArchiveWriter, bool, error) {
|
|
||||||
w, created, err := ws.sweepWriterFor(targetID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return &ExportArchiveWriter{w: w}, created, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportReleaseSweepWriter releases a sweep-created registry entry
|
|
||||||
// exactly as a finished sweep does.
|
|
||||||
func (ws *Writers) ExportReleaseSweepWriter(
|
|
||||||
targetID string, w *ExportArchiveWriter,
|
|
||||||
) {
|
|
||||||
ws.releaseSweepWriter(targetID, w.w)
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewTestSweeper builds a Sweeper backed by the given main
|
|
||||||
// database and archive writers, without the fx lifecycle.
|
|
||||||
// Intended for tests.
|
|
||||||
func NewTestSweeper(
|
|
||||||
db *database.Database,
|
|
||||||
writers *Writers,
|
|
||||||
log *slog.Logger,
|
|
||||||
) *Sweeper {
|
|
||||||
return &Sweeper{
|
|
||||||
db: db,
|
|
||||||
writers: writers,
|
|
||||||
log: log,
|
|
||||||
interval: time.Hour,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportSweep runs a single archive sweep synchronously for
|
|
||||||
// tests.
|
|
||||||
func (s *Sweeper) ExportSweep(ctx context.Context) {
|
|
||||||
s.sweep(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportStart starts the sweeper's background loop for tests.
|
|
||||||
func (s *Sweeper) ExportStart() {
|
|
||||||
s.start()
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportRegisterHooks registers the sweeper's real fx lifecycle
|
|
||||||
// hooks on a lifecycle supplied by a test, so a test can drive
|
|
||||||
// the exact OnStart/OnStop functions the application runs and
|
|
||||||
// hand OnStart the kind of context fx actually supplies.
|
|
||||||
func (s *Sweeper) ExportRegisterHooks(lc fx.Lifecycle) {
|
|
||||||
s.registerHooks(lc)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportStop stops the sweeper's background loop for tests.
|
|
||||||
func (s *Sweeper) ExportStop(ctx context.Context) error {
|
|
||||||
return s.stop(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportWedgeLoop adds a goroutine to the sweeper's WaitGroup
|
|
||||||
// that never observes cancellation and returns only when release
|
|
||||||
// is closed. It stands in for a prune stuck on a locked archive.
|
|
||||||
func (s *Sweeper) ExportWedgeLoop(
|
|
||||||
release <-chan struct{},
|
|
||||||
) {
|
|
||||||
s.wg.Go(func() {
|
|
||||||
<-release
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportSetInterval overrides the sweep interval for tests.
|
|
||||||
func (s *Sweeper) ExportSetInterval(d time.Duration) {
|
|
||||||
s.interval = d
|
|
||||||
}
|
|
||||||
@@ -29,7 +29,6 @@ func TestEventDBCreateOmitsAssociations(t *testing.T) {
|
|||||||
Name: "leaky-target",
|
Name: "leaky-target",
|
||||||
Type: database.TargetTypeSlack,
|
Type: database.TargetTypeSlack,
|
||||||
Config: `{"webhookUrl":"https://hooks.example/secret"}`,
|
Config: `{"webhookUrl":"https://hooks.example/secret"}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
}
|
}
|
||||||
target.ID = uuid.New().String()
|
target.ID = uuid.New().String()
|
||||||
|
|
||||||
|
|||||||
@@ -25,7 +25,6 @@ func TestPreloadedModelsMarshalWithoutTheirParent(t *testing.T) {
|
|||||||
Targets: []database.Target{{
|
Targets: []database.Target{{
|
||||||
Name: "log",
|
Name: "log",
|
||||||
Type: database.TargetTypeLog,
|
Type: database.TargetTypeLog,
|
||||||
MaxAttempts: 1,
|
|
||||||
}},
|
}},
|
||||||
}
|
}
|
||||||
require.NoError(t, db.Create(&stored).Error)
|
require.NoError(t, db.Create(&stored).Error)
|
||||||
|
|||||||
@@ -49,7 +49,6 @@ func TestModelsDoNotMarshalTheirSecrets(t *testing.T) {
|
|||||||
Name: keptField,
|
Name: keptField,
|
||||||
Type: database.TargetTypeSlack,
|
Type: database.TargetTypeSlack,
|
||||||
Config: `{"webhookUrl":"https://h/s/` + marker + `"}`,
|
Config: `{"webhookUrl":"https://h/s/` + marker + `"}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -100,7 +99,6 @@ func TestWebhookMarshalsNoTargetConfig(t *testing.T) {
|
|||||||
Targets: []database.Target{{
|
Targets: []database.Target{{
|
||||||
Name: "slack",
|
Name: "slack",
|
||||||
Config: `{"webhookUrl":"https://h/s/` + marker + `"}`,
|
Config: `{"webhookUrl":"https://h/s/` + marker + `"}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
}},
|
}},
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
@@ -29,12 +29,9 @@ type Target struct {
|
|||||||
// handler that marshals the model itself.
|
// handler that marshals the model itself.
|
||||||
Config string `gorm:"type:text" json:"-"` // JSON configuration
|
Config string `gorm:"type:text" json:"-"` // JSON configuration
|
||||||
|
|
||||||
// MaxAttempts is how many times an HTTP or Slack target attempts
|
// For HTTP targets (max_retries=0 means fire-and-forget,
|
||||||
// each delivery in all, at least 1. 1 is a single attempt with no
|
// >0 enables retries with backoff)
|
||||||
// circuit breaker; above 1, a failed attempt is retried with
|
MaxRetries int `json:"maxRetries,omitempty"`
|
||||||
// backoff behind the breaker. The table's check constraint refuses
|
|
||||||
// a target saved with less than 1, whatever its type.
|
|
||||||
MaxAttempts int `gorm:"not null;check:max_attempts >= 1" json:"maxAttempts,omitempty"`
|
|
||||||
|
|
||||||
// Relations. No model marshals the record it belongs to:
|
// Relations. No model marshals the record it belongs to:
|
||||||
// Webhook.Targets leads back here, and the JSON could loop.
|
// Webhook.Targets leads back here, and the JSON could loop.
|
||||||
|
|||||||
@@ -1,44 +0,0 @@
|
|||||||
package database_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// newHTTPTarget returns an http target that makes maxAttempts attempts
|
|
||||||
// at each delivery.
|
|
||||||
func newHTTPTarget(maxAttempts int) *database.Target {
|
|
||||||
return &database.Target{
|
|
||||||
WebhookID: uuid.New().String(),
|
|
||||||
Name: "target",
|
|
||||||
Type: database.TargetTypeHTTP,
|
|
||||||
Config: `{"url":"https://example.com/"}`,
|
|
||||||
MaxAttempts: maxAttempts,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTargetCreate_RefusesZeroAttempts relies on the targets table
|
|
||||||
// alone: nothing in the model checks the number, so the check
|
|
||||||
// constraint is what keeps a target with 0 attempts out of the
|
|
||||||
// database.
|
|
||||||
func TestTargetCreate_RefusesZeroAttempts(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := startedTestDB(t)
|
|
||||||
|
|
||||||
err := db.Create(newHTTPTarget(0)).Error
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), "CHECK constraint failed")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTargetCreate_AcceptsOneAttempt(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := startedTestDB(t)
|
|
||||||
|
|
||||||
require.NoError(t, db.Create(newHTTPTarget(1)).Error)
|
|
||||||
}
|
|
||||||
@@ -4,13 +4,27 @@ import (
|
|||||||
"math"
|
"math"
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"gorm.io/gorm"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// DefaultRetentionDays is the event retention period the new
|
// DefaultRetentionDays is the event retention period applied to a
|
||||||
// webhook form applies when its retention field is left blank.
|
// webhook created without an explicit retention value. It is the
|
||||||
|
// single source of truth for that policy and must stay in sync
|
||||||
|
// with the `gorm:"default:30"` column default on
|
||||||
|
// Webhook.RetentionDays below; a struct tag cannot reference a
|
||||||
|
// constant, so a test asserts the two agree.
|
||||||
DefaultRetentionDays = 30
|
DefaultRetentionDays = 30
|
||||||
|
|
||||||
|
// RetentionForeverDays is the sentinel RetentionDays value meaning
|
||||||
|
// "retain events forever". Users express that intent as 0, which
|
||||||
|
// Webhook.BeforeSave rewrites to this value: the column default
|
||||||
|
// substitutes DefaultRetentionDays for a zero value at insert
|
||||||
|
// time, so a zero can never survive a round trip to the database.
|
||||||
|
// Nothing outside this file may hardcode the number.
|
||||||
|
RetentionForeverDays = 365 * 1000
|
||||||
|
|
||||||
// MaxFiniteRetentionDays is the largest finite retention period the
|
// MaxFiniteRetentionDays is the largest finite retention period the
|
||||||
// reaper's cutoff arithmetic can represent, and therefore the
|
// reaper's cutoff arithmetic can represent, and therefore the
|
||||||
// largest one a caller may request. It is derived from that
|
// largest one a caller may request. It is derived from that
|
||||||
@@ -24,7 +38,9 @@ const (
|
|||||||
// turns the cutoff into a timestamp in the far future that matches
|
// turns the cutoff into a timestamp in the far future that matches
|
||||||
// every row in the webhook's database. That is why this bound is
|
// every row in the webhook's database. That is why this bound is
|
||||||
// enforced on input and why retentionCutoff saturates underneath
|
// enforced on input and why retentionCutoff saturates underneath
|
||||||
// it.
|
// it. Note that RetentionForeverDays deliberately sits above this
|
||||||
|
// ceiling: such webhooks are skipped before any cutoff is
|
||||||
|
// computed, and never reach the arithmetic at all.
|
||||||
MaxFiniteRetentionDays = int(
|
MaxFiniteRetentionDays = int(
|
||||||
math.MaxInt64 / int64(time.Hour) / hoursPerDay,
|
math.MaxInt64 / int64(time.Hour) / hoursPerDay,
|
||||||
)
|
)
|
||||||
@@ -32,10 +48,12 @@ const (
|
|||||||
|
|
||||||
// Webhook represents a webhook processing unit that groups entrypoints and targets
|
// Webhook represents a webhook processing unit that groups entrypoints and targets
|
||||||
//
|
//
|
||||||
// Every method below takes a pointer receiver, so handlers put a
|
// Every method below takes a pointer receiver. BeforeSave has to,
|
||||||
// *Webhook into template data: html/template cannot call a pointer
|
// because it mutates the record and GORM only invokes hooks declared
|
||||||
// method on a value held in a map, because a map element is not
|
// that way; the display helpers follow suit so the receiver kinds do
|
||||||
// addressable.
|
// not mix. Handlers therefore put a *Webhook into template data:
|
||||||
|
// html/template cannot call a pointer method on a value held in a map,
|
||||||
|
// because a map element is not addressable.
|
||||||
type Webhook struct {
|
type Webhook struct {
|
||||||
BaseModel
|
BaseModel
|
||||||
|
|
||||||
@@ -43,9 +61,10 @@ type Webhook struct {
|
|||||||
Name string `gorm:"not null" json:"name"`
|
Name string `gorm:"not null" json:"name"`
|
||||||
Description string `json:"description"`
|
Description string `json:"description"`
|
||||||
|
|
||||||
// RetentionDays is the number of days to retain events, and 0
|
// RetentionDays is the number of days to retain events. A value of
|
||||||
// means retain them forever.
|
// RetentionForeverDays means retain forever. The column default
|
||||||
RetentionDays int `json:"retentionDays"`
|
// must equal DefaultRetentionDays.
|
||||||
|
RetentionDays int `gorm:"default:30" json:"retentionDays"`
|
||||||
|
|
||||||
// Relations. No model marshals the record it belongs to:
|
// Relations. No model marshals the record it belongs to:
|
||||||
// User.Webhooks leads back here, and the JSON could loop.
|
// User.Webhooks leads back here, and the JSON could loop.
|
||||||
@@ -54,17 +73,36 @@ type Webhook struct {
|
|||||||
Targets []Target `json:"targets,omitempty"`
|
Targets []Target `json:"targets,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// retainsForever reports whether a RetentionDays value means "keep
|
// BeforeSave normalises RetentionDays on every insert and update. A
|
||||||
// events indefinitely", which 0 does. It is the single definition of
|
// non-positive value is the user's way of asking for "retain forever",
|
||||||
// that question, shared by Webhook.RetainsForever and by the reaper's
|
// which is stored as the RetentionForeverDays sentinel.
|
||||||
|
//
|
||||||
|
// This has to happen in a hook rather than at the call sites. GORM
|
||||||
|
// substitutes the column default (DefaultRetentionDays) for a zero
|
||||||
|
// value while building the insert statement, which runs after
|
||||||
|
// BeforeSave; rewriting any later than this loses that race and the
|
||||||
|
// row lands at 30 days. Living on the model also means a future call
|
||||||
|
// site, such as a REST API or a fixture, cannot bypass it.
|
||||||
|
func (w *Webhook) BeforeSave(_ *gorm.DB) error {
|
||||||
|
if w.RetentionDays <= 0 {
|
||||||
|
w.RetentionDays = RetentionForeverDays
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// retainsForever reports whether a stored RetentionDays value means
|
||||||
|
// "keep events indefinitely". It is the single definition of that
|
||||||
|
// question, shared by Webhook.RetainsForever and by the reaper's
|
||||||
// cutoff computation so the two cannot disagree about which webhooks
|
// cutoff computation so the two cannot disagree about which webhooks
|
||||||
// are exempt from reaping.
|
// are exempt from reaping.
|
||||||
//
|
//
|
||||||
// A negative value, which the forms refuse, is read the same way: as a
|
// It accepts the RetentionForeverDays sentinel written by BeforeSave
|
||||||
// day count it would put the cutoff in the future, where it matches
|
// and, defensively, any non-positive value, which BeforeSave treats
|
||||||
// every event the webhook has.
|
// as a request to retain forever.
|
||||||
func retainsForever(retentionDays int) bool {
|
func retainsForever(retentionDays int) bool {
|
||||||
return retentionDays <= 0
|
return retentionDays <= 0 ||
|
||||||
|
retentionDays >= RetentionForeverDays
|
||||||
}
|
}
|
||||||
|
|
||||||
// RetainsForever reports whether this webhook's events are kept
|
// RetainsForever reports whether this webhook's events are kept
|
||||||
@@ -81,8 +119,7 @@ func (w *Webhook) RetentionCutoff(now time.Time) (time.Time, bool) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// RetentionLabel returns the webhook's retention policy as display
|
// RetentionLabel returns the webhook's retention policy as display
|
||||||
// text: "forever" rather than "0 days" for a webhook that keeps its
|
// text, so that no template has to know about the sentinel value.
|
||||||
// events.
|
|
||||||
func (w *Webhook) RetentionLabel() string {
|
func (w *Webhook) RetentionLabel() string {
|
||||||
if w.RetainsForever() {
|
if w.RetainsForever() {
|
||||||
return "forever"
|
return "forever"
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ package database_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"reflect"
|
||||||
|
"strconv"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -45,7 +47,8 @@ func storedRetention(t *testing.T, db *gorm.DB, id string) int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// newWebhookWithRetention creates a webhook through the ordinary Create
|
// newWebhookWithRetention creates a webhook through the ordinary Create
|
||||||
// path, as the new webhook form does.
|
// path, so the BeforeSave hook and the GORM column default both apply
|
||||||
|
// exactly as they do in production.
|
||||||
func newWebhookWithRetention(
|
func newWebhookWithRetention(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
db *gorm.DB,
|
db *gorm.DB,
|
||||||
@@ -64,7 +67,7 @@ func newWebhookWithRetention(
|
|||||||
return wh.ID
|
return wh.ID
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWebhookRetentionDays_ZeroIsStoredAsZero(t *testing.T) {
|
func TestWebhookBeforeSave_ZeroBecomesForeverSentinel(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
db := startedTestDB(t)
|
db := startedTestDB(t)
|
||||||
@@ -72,10 +75,31 @@ func TestWebhookRetentionDays_ZeroIsStoredAsZero(t *testing.T) {
|
|||||||
wh := &database.Webhook{RetentionDays: 0}
|
wh := &database.Webhook{RetentionDays: 0}
|
||||||
id := newWebhookWithRetention(t, db, wh)
|
id := newWebhookWithRetention(t, db, wh)
|
||||||
|
|
||||||
assert.Equal(t, 0, storedRetention(t, db, id))
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
database.RetentionForeverDays,
|
||||||
|
storedRetention(t, db, id),
|
||||||
|
"a zero retention must be stored as the sentinel, "+
|
||||||
|
"not replaced by the column default",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWebhookRetentionDays_DayCountIsStored(t *testing.T) {
|
func TestWebhookBeforeSave_NegativeBecomesForeverSentinel(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
db := startedTestDB(t)
|
||||||
|
|
||||||
|
wh := &database.Webhook{RetentionDays: -5}
|
||||||
|
id := newWebhookWithRetention(t, db, wh)
|
||||||
|
|
||||||
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
database.RetentionForeverDays,
|
||||||
|
storedRetention(t, db, id),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWebhookBeforeSave_PositiveIsPreserved(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
db := startedTestDB(t)
|
db := startedTestDB(t)
|
||||||
@@ -86,9 +110,10 @@ func TestWebhookRetentionDays_DayCountIsStored(t *testing.T) {
|
|||||||
assert.Equal(t, 7, storedRetention(t, db, id))
|
assert.Equal(t, 7, storedRetention(t, db, id))
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestWebhookRetentionDays_UpdateToZeroIsStoredAsZero changes a day
|
// TestWebhookBeforeSave_UpdateToZeroBecomesSentinel proves the hook
|
||||||
// count to 0 through the same Save call the edit handler makes.
|
// fires on update as well as insert, via the same Save call the edit
|
||||||
func TestWebhookRetentionDays_UpdateToZeroIsStoredAsZero(t *testing.T) {
|
// handler makes.
|
||||||
|
func TestWebhookBeforeSave_UpdateToZeroBecomesSentinel(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
db := startedTestDB(t)
|
db := startedTestDB(t)
|
||||||
@@ -100,7 +125,28 @@ func TestWebhookRetentionDays_UpdateToZeroIsStoredAsZero(t *testing.T) {
|
|||||||
wh.RetentionDays = 0
|
wh.RetentionDays = 0
|
||||||
require.NoError(t, db.Omit(clause.Associations).Save(wh).Error)
|
require.NoError(t, db.Omit(clause.Associations).Save(wh).Error)
|
||||||
|
|
||||||
assert.Equal(t, 0, storedRetention(t, db, id))
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
database.RetentionForeverDays,
|
||||||
|
storedRetention(t, db, id),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestWebhookRetentionColumnDefaultMatchesConstant guards the one place
|
||||||
|
// the default lives twice: a struct tag cannot reference a constant, so
|
||||||
|
// this asserts the tag and DefaultRetentionDays agree.
|
||||||
|
func TestWebhookRetentionColumnDefaultMatchesConstant(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
field, ok := reflect.TypeFor[database.Webhook]().
|
||||||
|
FieldByName("RetentionDays")
|
||||||
|
require.True(t, ok, "Webhook.RetentionDays must exist")
|
||||||
|
|
||||||
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
"default:"+strconv.Itoa(database.DefaultRetentionDays),
|
||||||
|
field.Tag.Get("gorm"),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestMaxFiniteRetentionDaysIsTheOverflowCeiling asserts that the
|
// TestMaxFiniteRetentionDaysIsTheOverflowCeiling asserts that the
|
||||||
@@ -130,6 +176,14 @@ func TestMaxFiniteRetentionDaysIsTheOverflowCeiling(t *testing.T) {
|
|||||||
time.Duration(overCeiling*hoursPerDay)*time.Hour,
|
time.Duration(overCeiling*hoursPerDay)*time.Hour,
|
||||||
"one day past the ceiling must overflow",
|
"one day past the ceiling must overflow",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
assert.Less(
|
||||||
|
t,
|
||||||
|
database.MaxFiniteRetentionDays,
|
||||||
|
database.RetentionForeverDays,
|
||||||
|
"the sentinel sits above the ceiling and is only safe "+
|
||||||
|
"because retain-forever webhooks skip the arithmetic",
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWebhookRetainsForeverAndLabel(t *testing.T) {
|
func TestWebhookRetainsForeverAndLabel(t *testing.T) {
|
||||||
@@ -141,8 +195,16 @@ func TestWebhookRetainsForeverAndLabel(t *testing.T) {
|
|||||||
forever bool
|
forever bool
|
||||||
label string
|
label string
|
||||||
}{
|
}{
|
||||||
{"zero", 0, true, testForeverLabel},
|
{
|
||||||
{"negative", -1, true, testForeverLabel},
|
"sentinel",
|
||||||
|
database.RetentionForeverDays, true, testForeverLabel,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"above sentinel",
|
||||||
|
database.RetentionForeverDays + 1, true, testForeverLabel,
|
||||||
|
},
|
||||||
|
{"legacy zero", 0, true, testForeverLabel},
|
||||||
|
{"legacy negative", -1, true, testForeverLabel},
|
||||||
{"default", database.DefaultRetentionDays, false, "30 days"},
|
{"default", database.DefaultRetentionDays, false, "30 days"},
|
||||||
{"one day", 1, false, "1 day"},
|
{"one day", 1, false, "1 day"},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -243,8 +243,7 @@ func (r *RetentionReaper) reapWebhook(
|
|||||||
|
|
||||||
// retentionCutoff returns the timestamp before which a webhook's
|
// retentionCutoff returns the timestamp before which a webhook's
|
||||||
// events have expired, and whether any cutoff applies at all. It
|
// events have expired, and whether any cutoff applies at all. It
|
||||||
// reports false for a retention of 0, which means retain forever, so
|
// reports false for a retain-forever policy, so no DELETE is issued.
|
||||||
// no DELETE is issued.
|
|
||||||
//
|
//
|
||||||
// The day count is clamped to MaxFiniteRetentionDays first. This is
|
// The day count is clamped to MaxFiniteRetentionDays first. This is
|
||||||
// defense in depth rather than decoration: a time.Duration is an int64
|
// defense in depth rather than decoration: a time.Duration is an int64
|
||||||
|
|||||||
@@ -85,6 +85,41 @@ func createWebhook(
|
|||||||
db.Omit(clause.Associations).Create(wh).Error,
|
db.Omit(clause.Associations).Create(wh).Error,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Webhook.BeforeSave rewrites a non-positive RetentionDays to the
|
||||||
|
// retain-forever sentinel, and the column's GORM default would
|
||||||
|
// otherwise substitute 30. Force the requested value with a
|
||||||
|
// column-level update so tests can plant legacy rows that predate
|
||||||
|
// the sentinel and still carry a literal 0 or negative value.
|
||||||
|
require.NoError(
|
||||||
|
t,
|
||||||
|
db.Model(wh).
|
||||||
|
Update("retention_days", retentionDays).Error,
|
||||||
|
)
|
||||||
|
|
||||||
|
return wh.ID
|
||||||
|
}
|
||||||
|
|
||||||
|
// createWebhookNormally inserts a webhook through the ordinary Create
|
||||||
|
// path, with no column-level forcing, so Webhook.BeforeSave applies
|
||||||
|
// exactly as it does in production. Passing 0 therefore yields a row
|
||||||
|
// holding the RetentionForeverDays sentinel.
|
||||||
|
func createWebhookNormally(
|
||||||
|
t *testing.T,
|
||||||
|
db *gorm.DB,
|
||||||
|
retentionDays int,
|
||||||
|
) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
wh := &database.Webhook{
|
||||||
|
UserID: uuid.New().String(),
|
||||||
|
Name: testWebhookName,
|
||||||
|
RetentionDays: retentionDays,
|
||||||
|
}
|
||||||
|
require.NoError(
|
||||||
|
t,
|
||||||
|
db.Omit(clause.Associations).Create(wh).Error,
|
||||||
|
)
|
||||||
|
|
||||||
return wh.ID
|
return wh.ID
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -246,19 +281,19 @@ func TestRetentionReaper_ReapsExpiredKeepsRecent(t *testing.T) {
|
|||||||
assertChainPresent(t, db, recent)
|
assertChainPresent(t, db, recent)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRetentionReaper_SkipsZeroReapsFiniteInSameSweep covers the
|
// TestRetentionReaper_SkipsSentinelReapsFiniteInSameSweep covers the
|
||||||
// end-to-end retain-forever path: a webhook created with a retention
|
// end-to-end retain-forever path: a webhook created the normal way with
|
||||||
// of 0 reads back 0, and the reaper leaves its ancient events alone
|
// a requested retention of 0 lands on the RetentionForeverDays
|
||||||
// while still reaping a finite-retention webhook in the very same
|
// sentinel, and the reaper leaves its ancient events alone while still
|
||||||
// sweep.
|
// reaping a finite-retention webhook in the very same sweep.
|
||||||
func TestRetentionReaper_SkipsZeroReapsFiniteInSameSweep(
|
func TestRetentionReaper_SkipsSentinelReapsFiniteInSameSweep(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupRetentionTest(t)
|
env := setupRetentionTest(t)
|
||||||
|
|
||||||
foreverID := createWebhook(t, env.mainDB.DB(), 0)
|
foreverID := createWebhookNormally(t, env.mainDB.DB(), 0)
|
||||||
|
|
||||||
var stored database.Webhook
|
var stored database.Webhook
|
||||||
|
|
||||||
@@ -267,12 +302,14 @@ func TestRetentionReaper_SkipsZeroReapsFiniteInSameSweep(
|
|||||||
env.mainDB.DB().Where("id = ?", foreverID).
|
env.mainDB.DB().Where("id = ?", foreverID).
|
||||||
First(&stored).Error,
|
First(&stored).Error,
|
||||||
)
|
)
|
||||||
require.Zero(
|
require.Equal(
|
||||||
t, stored.RetentionDays,
|
t,
|
||||||
"a retention of 0 must be stored as 0",
|
database.RetentionForeverDays,
|
||||||
|
stored.RetentionDays,
|
||||||
|
"a requested retention of 0 must persist as the sentinel",
|
||||||
)
|
)
|
||||||
|
|
||||||
finiteID := createWebhook(t, env.mainDB.DB(), 30)
|
finiteID := createWebhookNormally(t, env.mainDB.DB(), 30)
|
||||||
|
|
||||||
foreverDB, err := env.mgr.GetDB(foreverID)
|
foreverDB, err := env.mgr.GetDB(foreverID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -298,10 +335,14 @@ func TestRetentionReaper_SkipsZeroReapsFiniteInSameSweep(
|
|||||||
// negative, so subtracting it moves the cutoff into the far future,
|
// negative, so subtracting it moves the cutoff into the far future,
|
||||||
// where "created_at < cutoff" matches every row: an event created a
|
// where "created_at < cutoff" matches every row: an event created a
|
||||||
// moment ago, and its delivery and delivery result, were all deleted
|
// moment ago, and its delivery and delivery result, were all deleted
|
||||||
// on the first sweep. 200000 is inside that band.
|
// on the first sweep. 200000 is inside that band and below the
|
||||||
|
// retain-forever sentinel, so it is treated as a finite policy and
|
||||||
|
// really does reach the arithmetic.
|
||||||
//
|
//
|
||||||
// The form refuses such a value; the point of the test is that a row
|
// The row is planted at the column level because such a value can no
|
||||||
// written by any other path still cannot trigger the wipe.
|
// longer be submitted through the form; the point of the test is that
|
||||||
|
// a row from an older version, or a future call site, still cannot
|
||||||
|
// trigger the wipe.
|
||||||
func TestRetentionReaper_HugeFiniteRetentionRetainsRecentEvents(
|
func TestRetentionReaper_HugeFiniteRetentionRetainsRecentEvents(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
@@ -317,6 +358,12 @@ func TestRetentionReaper_HugeFiniteRetentionRetainsRecentEvents(
|
|||||||
database.MaxFiniteRetentionDays,
|
database.MaxFiniteRetentionDays,
|
||||||
"the test value must exceed what the cutoff can represent",
|
"the test value must exceed what the cutoff can represent",
|
||||||
)
|
)
|
||||||
|
require.Less(
|
||||||
|
t,
|
||||||
|
overflowingRetentionDays,
|
||||||
|
database.RetentionForeverDays,
|
||||||
|
"the test value must not be treated as retain-forever",
|
||||||
|
)
|
||||||
|
|
||||||
webhookID := createWebhook(
|
webhookID := createWebhook(
|
||||||
t, env.mainDB.DB(), overflowingRetentionDays,
|
t, env.mainDB.DB(), overflowingRetentionDays,
|
||||||
@@ -332,15 +379,14 @@ func TestRetentionReaper_HugeFiniteRetentionRetainsRecentEvents(
|
|||||||
assertChainPresent(t, db, fresh)
|
assertChainPresent(t, db, fresh)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRetentionReaper_RetainsForeverWhenNegative checks that a negative
|
func TestRetentionReaper_RetainsForeverWhenNonPositive(t *testing.T) {
|
||||||
// retention, which the forms refuse, keeps every event rather than
|
|
||||||
// putting the cutoff in the future, where it would match them all.
|
|
||||||
func TestRetentionReaper_RetainsForeverWhenNegative(t *testing.T) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupRetentionTest(t)
|
env := setupRetentionTest(t)
|
||||||
|
|
||||||
webhookID := createWebhook(t, env.mainDB.DB(), -1)
|
// A legacy row written before the sentinel existed still carries a
|
||||||
|
// literal 0; the <= 0 guard must keep honouring it.
|
||||||
|
webhookID := createWebhook(t, env.mainDB.DB(), 0)
|
||||||
|
|
||||||
db, err := env.mgr.GetDB(webhookID)
|
db, err := env.mgr.GetDB(webhookID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|||||||
@@ -93,7 +93,7 @@ func TestMainSchemaHoldsEveryModel(t *testing.T) {
|
|||||||
Type: database.TargetTypeHTTP,
|
Type: database.TargetTypeHTTP,
|
||||||
Active: true,
|
Active: true,
|
||||||
Config: `{"url":"https://example.com/"}`,
|
Config: `{"url":"https://example.com/"}`,
|
||||||
MaxAttempts: 3,
|
MaxRetries: 3,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -137,7 +137,7 @@ func TestPerWebhookEventDatabaseFilesAreOwnerOnly(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveDatabaseFilesAreOwnerOnly covers the archive-*.db tier.
|
// TestArchiveDatabaseFilesAreOwnerOnly covers the archive-*.db tier.
|
||||||
// internal/archive builds that path and opens it through OpenSQLite,
|
// internal/delivery builds that path and opens it through OpenSQLite,
|
||||||
// the same single open path exercised here, so the mode is settled for
|
// the same single open path exercised here, so the mode is settled for
|
||||||
// all three tiers in one place.
|
// all three tiers in one place.
|
||||||
func TestArchiveDatabaseFilesAreOwnerOnly(t *testing.T) {
|
func TestArchiveDatabaseFilesAreOwnerOnly(t *testing.T) {
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
-- Migration 001: the schema of a database target's archive file. The
|
-- Migration 001: the schema of a database target's archive file. The
|
||||||
-- archivedEvent model in internal/archive must agree with it.
|
-- archivedEvent model in internal/delivery must agree with it.
|
||||||
|
|
||||||
CREATE TABLE archived_events (
|
CREATE TABLE archived_events (
|
||||||
id integer PRIMARY KEY AUTOINCREMENT,
|
id integer PRIMARY KEY AUTOINCREMENT,
|
||||||
|
|||||||
@@ -46,7 +46,7 @@ CREATE TABLE webhooks (
|
|||||||
user_id uuid NOT NULL,
|
user_id uuid NOT NULL,
|
||||||
name text NOT NULL,
|
name text NOT NULL,
|
||||||
description text,
|
description text,
|
||||||
retention_days integer,
|
retention_days integer DEFAULT 30,
|
||||||
PRIMARY KEY (id),
|
PRIMARY KEY (id),
|
||||||
CONSTRAINT fk_users_webhooks FOREIGN KEY (user_id) REFERENCES users (id)
|
CONSTRAINT fk_users_webhooks FOREIGN KEY (user_id) REFERENCES users (id)
|
||||||
);
|
);
|
||||||
@@ -80,11 +80,10 @@ CREATE TABLE targets (
|
|||||||
type text NOT NULL,
|
type text NOT NULL,
|
||||||
active numeric DEFAULT true,
|
active numeric DEFAULT true,
|
||||||
config text,
|
config text,
|
||||||
max_attempts integer NOT NULL,
|
max_retries integer,
|
||||||
PRIMARY KEY (id),
|
PRIMARY KEY (id),
|
||||||
CONSTRAINT fk_webhooks_targets
|
CONSTRAINT fk_webhooks_targets
|
||||||
FOREIGN KEY (webhook_id) REFERENCES webhooks (id),
|
FOREIGN KEY (webhook_id) REFERENCES webhooks (id)
|
||||||
CONSTRAINT chk_targets_max_attempts CHECK (max_attempts >= 1)
|
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE INDEX idx_targets_deleted_at ON targets (deleted_at);
|
CREATE INDEX idx_targets_deleted_at ON targets (deleted_at);
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
package archive
|
package delivery
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
@@ -15,18 +15,18 @@ import (
|
|||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
// SweeperParams holds the fx dependencies for the
|
// ArchiveSweeperParams holds the fx dependencies for the
|
||||||
// Sweeper.
|
// ArchiveSweeper.
|
||||||
type SweeperParams struct {
|
type ArchiveSweeperParams struct {
|
||||||
fx.In
|
fx.In
|
||||||
|
|
||||||
Config *config.Config
|
Config *config.Config
|
||||||
Database *database.Database
|
Database *database.Database
|
||||||
Writers *Writers
|
Engine *Engine
|
||||||
Logger *logger.Logger
|
Logger *logger.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
// Sweeper periodically prunes expired rows from the
|
// ArchiveSweeper periodically prunes expired rows from the
|
||||||
// archive databases of database targets that carry a positive
|
// archive databases of database targets that carry a positive
|
||||||
// expiry.
|
// expiry.
|
||||||
//
|
//
|
||||||
@@ -40,9 +40,9 @@ type SweeperParams struct {
|
|||||||
// It reuses Config.RetentionSweepInterval rather than
|
// It reuses Config.RetentionSweepInterval rather than
|
||||||
// introducing a second interval: this is a retention sweep with
|
// introducing a second interval: this is a retention sweep with
|
||||||
// the same semantics as the event retention reaper.
|
// the same semantics as the event retention reaper.
|
||||||
type Sweeper struct {
|
type ArchiveSweeper struct {
|
||||||
db *database.Database
|
db *database.Database
|
||||||
writers *Writers
|
eng *Engine
|
||||||
log *slog.Logger
|
log *slog.Logger
|
||||||
interval time.Duration
|
interval time.Duration
|
||||||
|
|
||||||
@@ -54,16 +54,16 @@ type Sweeper struct {
|
|||||||
wg sync.WaitGroup
|
wg sync.WaitGroup
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewSweeper creates the archive sweeper and registers
|
// NewArchiveSweeper creates the archive sweeper and registers
|
||||||
// its fx lifecycle hooks. The background sweep loop starts on
|
// its fx lifecycle hooks. The background sweep loop starts on
|
||||||
// OnStart and stops cleanly on OnStop via context cancellation.
|
// OnStart and stops cleanly on OnStop via context cancellation.
|
||||||
func NewSweeper(
|
func NewArchiveSweeper(
|
||||||
lc fx.Lifecycle,
|
lc fx.Lifecycle,
|
||||||
params SweeperParams,
|
params ArchiveSweeperParams,
|
||||||
) *Sweeper {
|
) *ArchiveSweeper {
|
||||||
s := &Sweeper{
|
s := &ArchiveSweeper{
|
||||||
db: params.Database,
|
db: params.Database,
|
||||||
writers: params.Writers,
|
eng: params.Engine,
|
||||||
log: params.Logger.Get(),
|
log: params.Logger.Get(),
|
||||||
interval: params.Config.RetentionSweepInterval,
|
interval: params.Config.RetentionSweepInterval,
|
||||||
}
|
}
|
||||||
@@ -77,7 +77,7 @@ func NewSweeper(
|
|||||||
// lifecycle. The start hook's context is deliberately ignored
|
// lifecycle. The start hook's context is deliberately ignored
|
||||||
// (see start for why the background loop must not inherit it);
|
// (see start for why the background loop must not inherit it);
|
||||||
// the stop hook's context is honoured (see stop).
|
// the stop hook's context is honoured (see stop).
|
||||||
func (s *Sweeper) registerHooks(lc fx.Lifecycle) {
|
func (s *ArchiveSweeper) registerHooks(lc fx.Lifecycle) {
|
||||||
lc.Append(fx.Hook{
|
lc.Append(fx.Hook{
|
||||||
//nolint:contextcheck // Not passing the hook context is
|
//nolint:contextcheck // Not passing the hook context is
|
||||||
// the point: see start.
|
// the point: see start.
|
||||||
@@ -103,7 +103,7 @@ func (s *Sweeper) registerHooks(lc fx.Lifecycle) {
|
|||||||
// goroutine must outlive the startup phase, so its lifetime is
|
// goroutine must outlive the startup phase, so its lifetime is
|
||||||
// bounded by OnStop instead: stop cancels this context and waits
|
// bounded by OnStop instead: stop cancels this context and waits
|
||||||
// on the WaitGroup.
|
// on the WaitGroup.
|
||||||
func (s *Sweeper) start() {
|
func (s *ArchiveSweeper) start() {
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
s.cancel = cancel
|
s.cancel = cancel
|
||||||
|
|
||||||
@@ -121,7 +121,7 @@ func (s *Sweeper) start() {
|
|||||||
// exit, bounded by the stop hook's context: a prune wedged on a
|
// exit, bounded by the stop hook's context: a prune wedged on a
|
||||||
// locked archive must not hang the process past fx's stop
|
// locked archive must not hang the process past fx's stop
|
||||||
// timeout.
|
// timeout.
|
||||||
func (s *Sweeper) stop(ctx context.Context) error {
|
func (s *ArchiveSweeper) stop(ctx context.Context) error {
|
||||||
s.log.Info("archive sweeper stopping")
|
s.log.Info("archive sweeper stopping")
|
||||||
|
|
||||||
if s.cancel != nil {
|
if s.cancel != nil {
|
||||||
@@ -140,7 +140,7 @@ func (s *Sweeper) stop(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Sweeper) run(ctx context.Context) {
|
func (s *ArchiveSweeper) run(ctx context.Context) {
|
||||||
defer s.wg.Done()
|
defer s.wg.Done()
|
||||||
|
|
||||||
ticker := time.NewTicker(s.interval)
|
ticker := time.NewTicker(s.interval)
|
||||||
@@ -164,7 +164,7 @@ func (s *Sweeper) run(ctx context.Context) {
|
|||||||
// A failure for one target is logged and the sweep continues,
|
// A failure for one target is logged and the sweep continues,
|
||||||
// matching how the write path already treats a prune error as
|
// matching how the write path already treats a prune error as
|
||||||
// non-fatal.
|
// non-fatal.
|
||||||
func (s *Sweeper) sweep(ctx context.Context) {
|
func (s *ArchiveSweeper) sweep(ctx context.Context) {
|
||||||
var targets []database.Target
|
var targets []database.Target
|
||||||
|
|
||||||
err := s.db.DB().
|
err := s.db.DB().
|
||||||
@@ -202,8 +202,8 @@ func (s *Sweeper) sweep(ctx context.Context) {
|
|||||||
// sweepTarget prunes the archive of a single database target.
|
// sweepTarget prunes the archive of a single database target.
|
||||||
// A missing, empty, or "never" expiry parses as a zero duration
|
// A missing, empty, or "never" expiry parses as a zero duration
|
||||||
// and is skipped entirely, so those archives are never pruned.
|
// and is skipped entirely, so those archives are never pruned.
|
||||||
func (s *Sweeper) sweepTarget(target *database.Target) {
|
func (s *ArchiveSweeper) sweepTarget(target *database.Target) {
|
||||||
expiry, err := ParseExpiry(target.Config)
|
expiry, err := parseArchiveExpiry(target.Config)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.log.Error(
|
s.log.Error(
|
||||||
"archive sweep: invalid database target config",
|
"archive sweep: invalid database target config",
|
||||||
@@ -219,11 +219,11 @@ func (s *Sweeper) sweepTarget(target *database.Target) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if s.writers == nil {
|
if s.eng == nil || s.eng.dbTarget == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
err = s.writers.sweepArchive(target.ID, expiry)
|
err = s.eng.dbTarget.sweepArchive(target.ID, expiry)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package archive_test
|
package delivery_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
@@ -16,15 +16,14 @@ import (
|
|||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/fx"
|
|
||||||
"gorm.io/driver/sqlite"
|
"gorm.io/driver/sqlite"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"gorm.io/gorm/clause"
|
"gorm.io/gorm/clause"
|
||||||
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
||||||
"sneak.berlin/go/webhooker/internal/db/migrations"
|
"sneak.berlin/go/webhooker/internal/db/migrations"
|
||||||
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -38,18 +37,6 @@ const (
|
|||||||
// sweepConcurrentWrites is how many deliveries the
|
// sweepConcurrentWrites is how many deliveries the
|
||||||
// concurrent write-plus-sweep test races against the sweep.
|
// concurrent write-plus-sweep test races against the sweep.
|
||||||
sweepConcurrentWrites = 20
|
sweepConcurrentWrites = 20
|
||||||
|
|
||||||
// hookStopTimeout bounds how long a lifecycle test waits for
|
|
||||||
// the sweeper's OnStop hook to return before declaring the
|
|
||||||
// shutdown hung.
|
|
||||||
hookStopTimeout = 10 * time.Second
|
|
||||||
|
|
||||||
// wedgeStopTimeout is the stop timeout a wedged-shutdown test
|
|
||||||
// hands OnStop, standing in for fx's StopTimeout. The test
|
|
||||||
// asserts only that the hook returns at all, and allows it
|
|
||||||
// hookStopTimeout — forty times this budget — to do so, so no
|
|
||||||
// assertion here races the wall clock.
|
|
||||||
wedgeStopTimeout = 250 * time.Millisecond
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// archiveTestWebhookName is the name of every webhook
|
// archiveTestWebhookName is the name of every webhook
|
||||||
@@ -58,12 +45,12 @@ const (
|
|||||||
const archiveTestWebhookName = "Sweep Test!"
|
const archiveTestWebhookName = "Sweep Test!"
|
||||||
|
|
||||||
// archiveEnv bundles the pieces an archive test drives: a main
|
// archiveEnv bundles the pieces an archive test drives: a main
|
||||||
// configuration database holding webhooks and targets, the archive
|
// configuration database holding webhooks and targets, a delivery
|
||||||
// writers, the archive sweeper, and the data directory the archive
|
// engine owning the archive writer registry, the archive sweeper,
|
||||||
// files live in.
|
// and the data directory the archive files live in.
|
||||||
type archiveEnv struct {
|
type archiveEnv struct {
|
||||||
sweeper *archive.Sweeper
|
sweeper *delivery.ArchiveSweeper
|
||||||
writers *archive.Writers
|
eng *delivery.Engine
|
||||||
mainDB *database.Database
|
mainDB *database.Database
|
||||||
dataDir string
|
dataDir string
|
||||||
}
|
}
|
||||||
@@ -79,28 +66,30 @@ func setupArchiveTest(t *testing.T) *archiveEnv {
|
|||||||
|
|
||||||
t.Cleanup(func() { _ = mainDB.Close() })
|
t.Cleanup(func() { _ = mainDB.Close() })
|
||||||
|
|
||||||
writers := archive.NewWriters(archive.WritersParams{
|
eng := delivery.NewTestEngineWithDB(
|
||||||
Database: mainDB,
|
mainDB,
|
||||||
DBManager: databasetest.NewWebhookDBManager(t, dataDir),
|
databasetest.NewWebhookDBManager(t, dataDir),
|
||||||
Logger: log,
|
log,
|
||||||
})
|
&http.Client{Timeout: 5 * time.Second},
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
|
||||||
return &archiveEnv{
|
return &archiveEnv{
|
||||||
sweeper: archive.NewTestSweeper(
|
sweeper: delivery.NewTestArchiveSweeper(
|
||||||
mainDB, writers, log,
|
mainDB, eng, log,
|
||||||
),
|
),
|
||||||
writers: writers,
|
eng: eng,
|
||||||
mainDB: mainDB,
|
mainDB: mainDB,
|
||||||
dataDir: dataDir,
|
dataDir: dataDir,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// archivePath returns where the writers keep a database target's
|
// archivePath returns where the engine keeps a database target's
|
||||||
// archive file, for the names seedDatabaseTarget gave it.
|
// archive file, for the names seedDatabaseTarget gave it.
|
||||||
func (env *archiveEnv) archivePath(tgt *database.Target) string {
|
func (env *archiveEnv) archivePath(tgt *database.Target) string {
|
||||||
return filepath.Join(
|
return filepath.Join(
|
||||||
env.dataDir,
|
env.dataDir,
|
||||||
archive.FileName(
|
delivery.ArchiveFileName(
|
||||||
archiveTestWebhookName, tgt.Name, tgt.ID,
|
archiveTestWebhookName, tgt.Name, tgt.ID,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -140,7 +129,6 @@ func (env *archiveEnv) addDatabaseTarget(
|
|||||||
Type: database.TargetTypeDatabase,
|
Type: database.TargetTypeDatabase,
|
||||||
Active: true,
|
Active: true,
|
||||||
Config: configJSON,
|
Config: configJSON,
|
||||||
MaxAttempts: 1,
|
|
||||||
}
|
}
|
||||||
require.NoError(
|
require.NoError(
|
||||||
t,
|
t,
|
||||||
@@ -188,7 +176,7 @@ func seedArchiveFile(
|
|||||||
require.NoError(t, migrations.Apply(gdb, migrations.Archive))
|
require.NoError(t, migrations.Apply(gdb, migrations.Archive))
|
||||||
|
|
||||||
for i, at := range archivedAt {
|
for i, at := range archivedAt {
|
||||||
row := archive.ExportArchivedEvent{
|
row := delivery.ExportArchivedEvent{
|
||||||
EventID: fmt.Sprintf("ev-%d", i),
|
EventID: fmt.Sprintf("ev-%d", i),
|
||||||
WebhookID: webhookID,
|
WebhookID: webhookID,
|
||||||
Method: http.MethodPost,
|
Method: http.MethodPost,
|
||||||
@@ -208,7 +196,7 @@ func archivedEventIDs(
|
|||||||
) []string {
|
) []string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
var rows []archive.ExportArchivedEvent
|
var rows []delivery.ExportArchivedEvent
|
||||||
|
|
||||||
rdb := openArchiveDBForRead(t, path)
|
rdb := openArchiveDBForRead(t, path)
|
||||||
require.NoError(t, rdb.Order("event_id").Find(&rows).Error)
|
require.NoError(t, rdb.Order("event_id").Find(&rows).Error)
|
||||||
@@ -245,7 +233,7 @@ func countArchivedRows(path string) (int64, error) {
|
|||||||
|
|
||||||
var count int64
|
var count int64
|
||||||
|
|
||||||
err = gdb.Model(&archive.ExportArchivedEvent{}).
|
err = gdb.Model(&delivery.ExportArchivedEvent{}).
|
||||||
Count(&count).Error
|
Count(&count).Error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
@@ -254,55 +242,6 @@ func countArchivedRows(path string) (int64, error) {
|
|||||||
return count, nil
|
return count, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// recordingLifecycle is a minimal fx.Lifecycle that records the
|
|
||||||
// hooks a component registers, so a test can invoke the real
|
|
||||||
// OnStart/OnStop functions with a context of its choosing.
|
|
||||||
type recordingLifecycle struct {
|
|
||||||
hooks []fx.Hook
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *recordingLifecycle) Append(h fx.Hook) {
|
|
||||||
l.hooks = append(l.hooks, h)
|
|
||||||
}
|
|
||||||
|
|
||||||
// requireStopHookExpires drives hook.OnStop with a stop context
|
|
||||||
// that expires while a wedged goroutine is still running, and
|
|
||||||
// requires the hook to return the deadline error naming
|
|
||||||
// component instead of blocking on the WaitGroup forever.
|
|
||||||
func requireStopHookExpires(
|
|
||||||
t *testing.T, hook fx.Hook, component string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
stopCtx, cancel := context.WithTimeout(
|
|
||||||
context.Background(), wedgeStopTimeout,
|
|
||||||
)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
var stopErr error
|
|
||||||
|
|
||||||
stopped := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
defer close(stopped)
|
|
||||||
|
|
||||||
stopErr = hook.OnStop(stopCtx)
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-stopped:
|
|
||||||
case <-time.After(hookStopTimeout):
|
|
||||||
t.Fatal(
|
|
||||||
"OnStop did not return: it discarded the stop " +
|
|
||||||
"context and is waiting on a wedged goroutine " +
|
|
||||||
"that will never observe cancellation",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
require.ErrorIs(t, stopErr, context.DeadlineExceeded)
|
|
||||||
require.ErrorContains(t, stopErr, component)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveSweeper_LoopOutlivesStartHookContext is the
|
// TestArchiveSweeper_LoopOutlivesStartHookContext is the
|
||||||
// regression test for a sweeper that never swept. fx calls
|
// regression test for a sweeper that never swept. fx calls
|
||||||
// OnStart with a context carrying the application's start
|
// OnStart with a context carrying the application's start
|
||||||
@@ -386,16 +325,16 @@ func TestArchiveSweep_DoesNotResurrectEvictedWriter(
|
|||||||
// the deletion path does. The target row is deliberately left
|
// the deletion path does. The target row is deliberately left
|
||||||
// in place: this is the tick that listed the target before
|
// in place: this is the tick that listed the target before
|
||||||
// the deletion committed.
|
// the deletion committed.
|
||||||
_, err := env.writers.ExportEnsureArchiveWriter(tgt.ID)
|
_, err := env.eng.ExportEnsureArchiveWriter(tgt.ID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
env.writers.EvictTarget(tgt.ID)
|
env.eng.EvictTarget(tgt.ID)
|
||||||
require.False(t, env.writers.ExportHasArchiveWriter(tgt.ID))
|
require.False(t, env.eng.ExportHasArchiveWriter(tgt.ID))
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
|
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"a sweep must never re-register a writer for a target "+
|
"a sweep must never re-register a writer for a target "+
|
||||||
"whose registry entry has already been released",
|
"whose registry entry has already been released",
|
||||||
)
|
)
|
||||||
@@ -418,7 +357,7 @@ func TestArchiveSweep_LeavesNoRegistryEntry(t *testing.T) {
|
|||||||
time.Now().Add(-time.Minute),
|
time.Now().Add(-time.Minute),
|
||||||
)
|
)
|
||||||
|
|
||||||
require.False(t, env.writers.ExportHasArchiveWriter(tgt.ID))
|
require.False(t, env.eng.ExportHasArchiveWriter(tgt.ID))
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
|
|
||||||
@@ -427,7 +366,7 @@ func TestArchiveSweep_LeavesNoRegistryEntry(t *testing.T) {
|
|||||||
"the sweep must still prune an idle archive",
|
"the sweep must still prune an idle archive",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"the sweep must release the registry entry it created",
|
"the sweep must release the registry entry it created",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -449,22 +388,24 @@ func TestArchiveSweep_KeepsWriterAdoptedByDelivery(
|
|||||||
t, tgt, time.Now().Add(-48*time.Hour),
|
t, tgt, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
d := databaseTargetDelivery(newEvent(`{"n":1}`), tgt)
|
webhookDB := testWebhookDB(t)
|
||||||
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
|
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
require.False(t, env.writers.ExportHasArchiveWriter(tgt.ID))
|
require.False(t, env.eng.ExportHasArchiveWriter(tgt.ID))
|
||||||
|
|
||||||
require.NoError(t, env.writers.Write(d))
|
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
|
|
||||||
assert.True(
|
assert.True(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"a delivery's writer must stay registered",
|
"a delivery's writer must stay registered",
|
||||||
)
|
)
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
|
|
||||||
assert.True(
|
assert.True(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"a sweep must not drop a writer a delivery owns",
|
"a sweep must not drop a writer a delivery owns",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -503,7 +444,7 @@ func TestArchiveSweep_KeepsWriterAdoptedDuringSweep(
|
|||||||
t, tgt, time.Now().Add(-48*time.Hour),
|
t, tgt, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
sweepWriter, created, err := env.writers.ExportSweepWriterFor(
|
sweepWriter, created, err := env.eng.ExportSweepWriterFor(
|
||||||
tgt.ID,
|
tgt.ID,
|
||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -513,34 +454,36 @@ func TestArchiveSweep_KeepsWriterAdoptedDuringSweep(
|
|||||||
)
|
)
|
||||||
|
|
||||||
// The delivery lands mid-sweep and adopts the entry.
|
// The delivery lands mid-sweep and adopts the entry.
|
||||||
d := databaseTargetDelivery(newEvent(`{"n":1}`), tgt)
|
webhookDB := testWebhookDB(t)
|
||||||
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
|
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
|
||||||
|
|
||||||
require.NoError(t, env.writers.Write(d))
|
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
|
|
||||||
adopted := env.writers.ExportArchiveWriterFor(tgt.ID)
|
adopted := env.eng.ExportArchiveWriterFor(tgt.ID)
|
||||||
require.NotNil(t, adopted)
|
require.NotNil(t, adopted)
|
||||||
require.True(
|
require.True(
|
||||||
t, sweepWriter.Same(adopted),
|
t, sweepWriter.Same(adopted),
|
||||||
"the delivery must have adopted the sweep's writer",
|
"the delivery must have adopted the sweep's writer",
|
||||||
)
|
)
|
||||||
require.True(
|
require.True(
|
||||||
t, env.writers.ExportArchiveHandleOpen(tgt.ID),
|
t, env.eng.ExportArchiveHandleOpen(tgt.ID),
|
||||||
"the delivery leaves the archive handle open",
|
"the delivery leaves the archive handle open",
|
||||||
)
|
)
|
||||||
|
|
||||||
// The sweep finishes.
|
// The sweep finishes.
|
||||||
env.writers.ExportReleaseSweepWriter(tgt.ID, sweepWriter)
|
env.eng.ExportReleaseSweepWriter(tgt.ID, sweepWriter)
|
||||||
|
|
||||||
require.True(
|
require.True(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"a writer adopted by a delivery during a sweep must "+
|
"a writer adopted by a delivery during a sweep must "+
|
||||||
"stay registered, or its open handle is unreachable",
|
"stay registered, or its open handle is unreachable",
|
||||||
)
|
)
|
||||||
|
|
||||||
env.writers.EvictTarget(tgt.ID)
|
env.eng.EvictTarget(tgt.ID)
|
||||||
|
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"the adopted writer must still be evictable",
|
"the adopted writer must still be evictable",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
@@ -604,7 +547,7 @@ func TestArchiveSweep_OpenExistingDoesNotCreateFile(
|
|||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
path := filepath.Join(dir, "archive-absent.db")
|
path := filepath.Join(dir, "archive-absent.db")
|
||||||
|
|
||||||
w := archive.NewExportArchiveWriter(
|
w := delivery.NewExportArchiveWriter(
|
||||||
path, archiveTestLogger(), 0,
|
path, archiveTestLogger(), 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -675,7 +618,7 @@ func TestArchiveSweep_LeavesArchiveClosed(t *testing.T) {
|
|||||||
t, tgt, time.Now().Add(-48*time.Hour),
|
t, tgt, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
w := archive.NewExportArchiveWriter(
|
w := delivery.NewExportArchiveWriter(
|
||||||
path, archiveTestLogger(), 0,
|
path, archiveTestLogger(), 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -715,23 +658,25 @@ func TestArchiveSweep_ClosesHandleOfRegisteredWriter(
|
|||||||
t, tgt, time.Now().Add(-48*time.Hour),
|
t, tgt, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
d := databaseTargetDelivery(newEvent(`{"n":1}`), tgt)
|
webhookDB := testWebhookDB(t)
|
||||||
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
|
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
|
||||||
|
|
||||||
require.NoError(t, env.writers.Write(d))
|
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
|
|
||||||
require.True(
|
require.True(
|
||||||
t, env.writers.ExportArchiveHandleOpen(tgt.ID),
|
t, env.eng.ExportArchiveHandleOpen(tgt.ID),
|
||||||
"the delivery must leave the archive handle open",
|
"the delivery must leave the archive handle open",
|
||||||
)
|
)
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
|
|
||||||
require.True(
|
require.True(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"the delivery's registry entry must survive the sweep",
|
"the delivery's registry entry must survive the sweep",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.writers.ExportArchiveHandleOpen(tgt.ID),
|
t, env.eng.ExportArchiveHandleOpen(tgt.ID),
|
||||||
"the sweep must leave the archive closed",
|
"the sweep must leave the archive closed",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -747,7 +692,7 @@ func TestArchiveSweep_ClosesHandleBeforeReopening(t *testing.T) {
|
|||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive.db")
|
path := filepath.Join(t.TempDir(), "archive.db")
|
||||||
|
|
||||||
w := archive.NewExportArchiveWriter(
|
w := delivery.NewExportArchiveWriter(
|
||||||
path, archiveTestLogger(), 0,
|
path, archiveTestLogger(), 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -775,8 +720,8 @@ func TestArchiveSweep_CancelledSweepLogsNoError(t *testing.T) {
|
|||||||
|
|
||||||
var errorLines bytes.Buffer
|
var errorLines bytes.Buffer
|
||||||
|
|
||||||
sweeper := archive.NewTestSweeper(
|
sweeper := delivery.NewTestArchiveSweeper(
|
||||||
env.mainDB, env.writers,
|
env.mainDB, env.eng,
|
||||||
slog.New(slog.NewTextHandler(
|
slog.New(slog.NewTextHandler(
|
||||||
&errorLines,
|
&errorLines,
|
||||||
&slog.HandlerOptions{Level: slog.LevelError},
|
&slog.HandlerOptions{Level: slog.LevelError},
|
||||||
@@ -820,7 +765,7 @@ func TestArchiveSweep_NeverExpiryUntouched(t *testing.T) {
|
|||||||
"config %q must keep rows forever", configJSON,
|
"config %q must keep rows forever", configJSON,
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"config %q must leave no registry entry behind",
|
"config %q must leave no registry entry behind",
|
||||||
configJSON,
|
configJSON,
|
||||||
)
|
)
|
||||||
@@ -885,7 +830,7 @@ func archiveTableExists(t *testing.T, path string) bool {
|
|||||||
|
|
||||||
return openArchiveDBForRead(t, path).
|
return openArchiveDBForRead(t, path).
|
||||||
Migrator().
|
Migrator().
|
||||||
HasTable(&archive.ExportArchivedEvent{})
|
HasTable(&delivery.ExportArchivedEvent{})
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_DoesNotCreateArchiveFile proves the sweep
|
// TestArchiveSweep_DoesNotCreateArchiveFile proves the sweep
|
||||||
@@ -912,7 +857,7 @@ func TestArchiveSweep_DoesNotCreateArchiveFile(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"the sweep must leave no registry entry behind",
|
"the sweep must leave no registry entry behind",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -930,7 +875,7 @@ func TestArchiveSweep_DoesNotCreateAfterWriterExists(
|
|||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
|
|
||||||
path, err := env.writers.ExportEnsureArchiveWriter(tgt.ID)
|
path, err := env.eng.ExportEnsureArchiveWriter(tgt.ID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NoFileExists(t, path)
|
require.NoFileExists(t, path)
|
||||||
|
|
||||||
@@ -977,14 +922,21 @@ func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
|||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
|
|
||||||
|
webhookDB := testWebhookDB(t)
|
||||||
|
|
||||||
|
// The deliveries are seeded up front, on the test's own
|
||||||
|
// goroutine: the seed helpers assert, and testify assertions
|
||||||
|
// must not run off the test goroutine.
|
||||||
deliveries := make(
|
deliveries := make(
|
||||||
[]*database.Delivery, 0, sweepConcurrentWrites,
|
[]*database.Delivery, 0, sweepConcurrentWrites,
|
||||||
)
|
)
|
||||||
|
|
||||||
for range sweepConcurrentWrites {
|
for range sweepConcurrentWrites {
|
||||||
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
|
|
||||||
deliveries = append(
|
deliveries = append(
|
||||||
deliveries,
|
deliveries,
|
||||||
databaseTargetDelivery(newEvent(`{"n":1}`), tgt),
|
seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -996,7 +948,7 @@ func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
|||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
|
|
||||||
for _, d := range deliveries {
|
for _, d := range deliveries {
|
||||||
_ = env.writers.Write(d)
|
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
+72
-15
@@ -13,7 +13,6 @@ import (
|
|||||||
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/globals"
|
"sneak.berlin/go/webhooker/internal/globals"
|
||||||
"sneak.berlin/go/webhooker/internal/lifecycle"
|
"sneak.berlin/go/webhooker/internal/lifecycle"
|
||||||
@@ -108,7 +107,7 @@ type Task struct {
|
|||||||
TargetName string
|
TargetName string
|
||||||
TargetType database.TargetType
|
TargetType database.TargetType
|
||||||
TargetConfig string
|
TargetConfig string
|
||||||
MaxAttempts int
|
MaxRetries int
|
||||||
|
|
||||||
Method string
|
Method string
|
||||||
RawQuery string
|
RawQuery string
|
||||||
@@ -125,11 +124,31 @@ type Notifier interface {
|
|||||||
Notify(tasks []Task)
|
Notify(tasks []Task)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Archives is how the handlers keep the database targets' archive
|
||||||
|
// files in step with the configuration. Deleting a webhook or a
|
||||||
|
// target releases the cached archive writers, whose open file
|
||||||
|
// handles would otherwise outlive them; renaming one renames the
|
||||||
|
// archive files, which are named for the webhook and the target
|
||||||
|
// (see ArchiveFileName).
|
||||||
|
//
|
||||||
|
// It is deliberately separate from Notifier: archiving lifecycle
|
||||||
|
// is not notification, and a small interface keeps the handlers
|
||||||
|
// package free of any dependency on the engine's internals while
|
||||||
|
// staying trivially fakeable in tests.
|
||||||
|
//
|
||||||
|
// Neither eviction deletes an archive file. Both are idempotent
|
||||||
|
// and are no-ops for a webhook or target with no engine state.
|
||||||
|
type Archives interface {
|
||||||
|
EvictWebhook(webhookID string)
|
||||||
|
EvictTarget(targetID string)
|
||||||
|
Rename(targetID, webhookName, targetName string) error
|
||||||
|
}
|
||||||
|
|
||||||
// CircuitBreakers is how the handlers read a target's circuit
|
// CircuitBreakers is how the handlers read a target's circuit
|
||||||
// breaker, so the webhook page and the event log can say that
|
// breaker, so the webhook page and the event log can say that
|
||||||
// deliveries to the target are paused and until when. It keeps the
|
// deliveries to the target are paused and until when. Like Archives,
|
||||||
// handlers free of the engine's internals and is trivially faked in
|
// it keeps the handlers free of the engine's internals and is
|
||||||
// tests.
|
// trivially faked in tests.
|
||||||
type CircuitBreakers interface {
|
type CircuitBreakers interface {
|
||||||
StateAndCooldown(targetID string) (CircuitState, time.Duration)
|
StateAndCooldown(targetID string) (CircuitState, time.Duration)
|
||||||
}
|
}
|
||||||
@@ -145,7 +164,6 @@ type EngineParams struct {
|
|||||||
Logger *logger.Logger
|
Logger *logger.Logger
|
||||||
SSRFGuard *Guard
|
SSRFGuard *Guard
|
||||||
Metrics *metrics.Set
|
Metrics *metrics.Set
|
||||||
Archives *archive.Writers
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Engine processes queued deliveries in the background
|
// Engine processes queued deliveries in the background
|
||||||
@@ -184,10 +202,9 @@ type Engine struct {
|
|||||||
httpTarget *httpTarget
|
httpTarget *httpTarget
|
||||||
slackTarget *slackTarget
|
slackTarget *slackTarget
|
||||||
|
|
||||||
// archives is the database targets' archive writers. The
|
// dbTarget is retained so the engine can reach the archive
|
||||||
// database target writes each event through them, and stop
|
// writer registry for eviction, renames and the idle sweep.
|
||||||
// closes them.
|
dbTarget *databaseTarget
|
||||||
archives *archive.Writers
|
|
||||||
|
|
||||||
// inflight is the set of deliveries this engine currently owns.
|
// inflight is the set of deliveries this engine currently owns.
|
||||||
// Recovery and the sweeps re-dispatch only what it does not
|
// Recovery and the sweeps re-dispatch only what it does not
|
||||||
@@ -211,7 +228,6 @@ func New(
|
|||||||
workers: defaultWorkers,
|
workers: defaultWorkers,
|
||||||
version: params.Globals.Version,
|
version: params.Globals.Version,
|
||||||
mtr: params.Metrics,
|
mtr: params.Metrics,
|
||||||
archives: params.Archives,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
e.initTargets(&http.Client{
|
e.initTargets(&http.Client{
|
||||||
@@ -256,6 +272,47 @@ func (e *Engine) Notify(tasks []Task) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// EvictWebhook implements Archives. The cached archive writer of
|
||||||
|
// every database target of the webhook is dropped from the
|
||||||
|
// registry and its file handle closed. The archive files
|
||||||
|
// themselves are left on disk — they are long-term storage the
|
||||||
|
// operator owns.
|
||||||
|
func (e *Engine) EvictWebhook(webhookID string) {
|
||||||
|
if e.dbTarget == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
e.dbTarget.evictWebhook(webhookID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// EvictTarget implements Archives. It is EvictWebhook for a single
|
||||||
|
// database target, and leaves the archive file on disk the same
|
||||||
|
// way.
|
||||||
|
func (e *Engine) EvictTarget(targetID string) {
|
||||||
|
if e.dbTarget == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
e.dbTarget.evict(targetID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rename implements Archives. It renames every one of a database
|
||||||
|
// target's archive files to ArchiveFileName(webhookName, targetName,
|
||||||
|
// targetID), each keeping the period in its name, under the lock the
|
||||||
|
// target's archive writes and the idle sweep take. It never replaces
|
||||||
|
// a file: if one already has a new name, the error is
|
||||||
|
// ErrArchiveNameTaken. The caller renames before it saves the new
|
||||||
|
// name: see databaseTarget.rename.
|
||||||
|
func (e *Engine) Rename(
|
||||||
|
targetID, webhookName, targetName string,
|
||||||
|
) error {
|
||||||
|
if e.dbTarget == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return e.dbTarget.rename(targetID, webhookName, targetName)
|
||||||
|
}
|
||||||
|
|
||||||
// StateAndCooldown implements CircuitBreakers. It returns the state of
|
// StateAndCooldown implements CircuitBreakers. It returns the state of
|
||||||
// the target's circuit breaker and, while the breaker is open, what is
|
// the target's circuit breaker and, while the breaker is open, what is
|
||||||
// left of its cooldown; the cooldown is zero once that has passed and
|
// left of its cooldown; the cooldown is zero once that has passed and
|
||||||
@@ -409,7 +466,7 @@ func (e *Engine) stop(ctx context.Context) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
e.archives.EvictAll()
|
e.dbTarget.evictAll()
|
||||||
|
|
||||||
e.log.Info("delivery engine stopped")
|
e.log.Info("delivery engine stopped")
|
||||||
|
|
||||||
@@ -1715,7 +1772,7 @@ func buildTargetFromTask(task *Task) database.Target {
|
|||||||
Name: task.TargetName,
|
Name: task.TargetName,
|
||||||
Type: task.TargetType,
|
Type: task.TargetType,
|
||||||
Config: task.TargetConfig,
|
Config: task.TargetConfig,
|
||||||
MaxAttempts: task.MaxAttempts,
|
MaxRetries: task.MaxRetries,
|
||||||
}
|
}
|
||||||
|
|
||||||
target.ID = task.TargetID
|
target.ID = task.TargetID
|
||||||
@@ -1971,7 +2028,7 @@ func (e *Engine) countAttemptsBatch(
|
|||||||
// One delivery id per recorded attempt, tallied here rather than
|
// One delivery id per recorded attempt, tallied here rather than
|
||||||
// grouped in SQL: internal/gormlog forbids (*gorm.DB).Scan, which
|
// grouped in SQL: internal/gormlog forbids (*gorm.DB).Scan, which
|
||||||
// a GROUP BY into a struct would need, and an attempt row per
|
// a GROUP BY into a struct would need, and an attempt row per
|
||||||
// delivery is bounded by the target's MaxAttempts.
|
// delivery is bounded by the target's MaxRetries.
|
||||||
var attemptIDs []string
|
var attemptIDs []string
|
||||||
|
|
||||||
err := webhookDB.
|
err := webhookDB.
|
||||||
@@ -2049,7 +2106,7 @@ func buildRecoveryTask(
|
|||||||
TargetName: target.Name,
|
TargetName: target.Name,
|
||||||
TargetType: target.Type,
|
TargetType: target.Type,
|
||||||
TargetConfig: target.Config,
|
TargetConfig: target.Config,
|
||||||
MaxAttempts: target.MaxAttempts,
|
MaxRetries: target.MaxRetries,
|
||||||
Method: event.Method,
|
Method: event.Method,
|
||||||
RawQuery: event.RawQuery,
|
RawQuery: event.RawQuery,
|
||||||
Headers: event.Headers,
|
Headers: event.Headers,
|
||||||
|
|||||||
@@ -170,7 +170,7 @@ func iTask(
|
|||||||
d database.Delivery,
|
d database.Delivery,
|
||||||
event database.Event,
|
event database.Event,
|
||||||
webhookID, targetID, name, config string,
|
webhookID, targetID, name, config string,
|
||||||
maxAttempts, attemptNum int,
|
maxRetries, attemptNum int,
|
||||||
body *string,
|
body *string,
|
||||||
) delivery.Task {
|
) delivery.Task {
|
||||||
return delivery.Task{
|
return delivery.Task{
|
||||||
@@ -181,7 +181,7 @@ func iTask(
|
|||||||
TargetName: name,
|
TargetName: name,
|
||||||
TargetType: database.TargetTypeHTTP,
|
TargetType: database.TargetTypeHTTP,
|
||||||
TargetConfig: config,
|
TargetConfig: config,
|
||||||
MaxAttempts: maxAttempts,
|
MaxRetries: maxRetries,
|
||||||
Method: event.Method,
|
Method: event.Method,
|
||||||
Headers: event.Headers,
|
Headers: event.Headers,
|
||||||
ContentType: event.ContentType,
|
ContentType: event.ContentType,
|
||||||
@@ -242,7 +242,7 @@ func TestProcessNewTask_InlineBody(t *testing.T) {
|
|||||||
cfg := iHTTPConfig(ts.URL)
|
cfg := iHTTPConfig(ts.URL)
|
||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"test-target", cfg, 1, 1, &bodyStr,
|
"test-target", cfg, 0, 1, &bodyStr,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(
|
s.Engine.ExportProcessNewTask(
|
||||||
@@ -293,7 +293,7 @@ func TestProcessNewTask_LargeBody_FetchFromDB(
|
|||||||
cfg := iHTTPConfig(ts.URL)
|
cfg := iHTTPConfig(ts.URL)
|
||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"test-large", cfg, 1, 1, nil,
|
"test-large", cfg, 0, 1, nil,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(
|
s.Engine.ExportProcessNewTask(
|
||||||
@@ -320,7 +320,7 @@ func TestProcessNewTask_InvalidWebhookID(t *testing.T) {
|
|||||||
TargetName: "test",
|
TargetName: "test",
|
||||||
TargetType: database.TargetTypeHTTP,
|
TargetType: database.TargetTypeHTTP,
|
||||||
TargetConfig: iHTTPConfig("http://localhost:9999"),
|
TargetConfig: iHTTPConfig("http://localhost:9999"),
|
||||||
MaxAttempts: 1,
|
MaxRetries: 0,
|
||||||
Body: nil,
|
Body: nil,
|
||||||
AttemptNum: 1,
|
AttemptNum: 1,
|
||||||
}
|
}
|
||||||
@@ -506,7 +506,7 @@ func TestWorkerLifecycle_StartStop(t *testing.T) {
|
|||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"lifecycle-test", "",
|
"lifecycle-test", "",
|
||||||
1, 1, &bodyStr,
|
0, 1, &bodyStr,
|
||||||
)
|
)
|
||||||
task.TargetType = database.TargetTypeLog
|
task.TargetType = database.TargetTypeLog
|
||||||
|
|
||||||
@@ -645,7 +645,7 @@ func TestRecoverPendingDeliveries(t *testing.T) {
|
|||||||
|
|
||||||
iCreateTarget(t, s.MainDB, targetID,
|
iCreateTarget(t, s.MainDB, targetID,
|
||||||
s.WebhookID, "recovery-target",
|
s.WebhookID, "recovery-target",
|
||||||
database.TargetTypeLog, "", 1,
|
database.TargetTypeLog, "", 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
iSeedPendingDeliveries(
|
iSeedPendingDeliveries(
|
||||||
@@ -686,7 +686,7 @@ func iCreateTarget(
|
|||||||
targetID, webhookID, name string,
|
targetID, webhookID, name string,
|
||||||
targetType database.TargetType,
|
targetType database.TargetType,
|
||||||
config string,
|
config string,
|
||||||
maxAttempts int,
|
maxRetries int,
|
||||||
) {
|
) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -696,7 +696,7 @@ func iCreateTarget(
|
|||||||
Type: targetType,
|
Type: targetType,
|
||||||
Active: true,
|
Active: true,
|
||||||
Config: config,
|
Config: config,
|
||||||
MaxAttempts: maxAttempts,
|
MaxRetries: maxRetries,
|
||||||
}
|
}
|
||||||
target.ID = targetID
|
target.ID = targetID
|
||||||
|
|
||||||
@@ -1077,7 +1077,7 @@ func TestRecoverInFlight_WithPendingDeliveries(
|
|||||||
|
|
||||||
iCreateTarget(t, s.MainDB, targetID,
|
iCreateTarget(t, s.MainDB, targetID,
|
||||||
s.WebhookID, "recover-target",
|
s.WebhookID, "recover-target",
|
||||||
database.TargetTypeLog, "", 1,
|
database.TargetTypeLog, "", 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
event := iSeedEvent(
|
event := iSeedEvent(
|
||||||
@@ -1232,7 +1232,7 @@ func TestDeliverHTTP_CustomTargetHeaders(t *testing.T) {
|
|||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"auth-target", string(cfgJSON),
|
"auth-target", string(cfgJSON),
|
||||||
1, 1, &bodyStr,
|
0, 1, &bodyStr,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(
|
s.Engine.ExportProcessNewTask(
|
||||||
@@ -1277,7 +1277,7 @@ func TestDeliverHTTP_TargetTimeout(t *testing.T) {
|
|||||||
|
|
||||||
task, d := iHTTPTaskAndDelivery(
|
task, d := iHTTPTaskAndDelivery(
|
||||||
event, del, "timeout-target",
|
event, del, "timeout-target",
|
||||||
string(cfgJSON), 1, 1,
|
string(cfgJSON), 0, 1,
|
||||||
)
|
)
|
||||||
|
|
||||||
e.ExportDeliverHTTP(context.TODO(), db, d, task)
|
e.ExportDeliverHTTP(context.TODO(), db, d, task)
|
||||||
@@ -1339,7 +1339,7 @@ func TestDeliverHTTP_CutsStoredResponseAtMaxBodyLog(
|
|||||||
bodyStr := event.Body
|
bodyStr := event.Body
|
||||||
task := iTask(
|
task := iTask(
|
||||||
del, event, s.WebhookID, targetID,
|
del, event, s.WebhookID, targetID,
|
||||||
"cut-target", cfgJSON, 1, 1, &bodyStr,
|
"cut-target", cfgJSON, 0, 1, &bodyStr,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(context.TODO(), &task)
|
s.Engine.ExportProcessNewTask(context.TODO(), &task)
|
||||||
@@ -1403,7 +1403,7 @@ func iHTTPTaskAndDelivery(
|
|||||||
event database.Event,
|
event database.Event,
|
||||||
del database.Delivery,
|
del database.Delivery,
|
||||||
name, config string,
|
name, config string,
|
||||||
maxAttempts, attemptNum int,
|
maxRetries, attemptNum int,
|
||||||
) (*delivery.Task, *database.Delivery) {
|
) (*delivery.Task, *database.Delivery) {
|
||||||
task := &delivery.Task{
|
task := &delivery.Task{
|
||||||
DeliveryID: del.ID,
|
DeliveryID: del.ID,
|
||||||
@@ -1413,7 +1413,7 @@ func iHTTPTaskAndDelivery(
|
|||||||
TargetName: name,
|
TargetName: name,
|
||||||
TargetType: database.TargetTypeHTTP,
|
TargetType: database.TargetTypeHTTP,
|
||||||
TargetConfig: config,
|
TargetConfig: config,
|
||||||
MaxAttempts: maxAttempts,
|
MaxRetries: maxRetries,
|
||||||
AttemptNum: attemptNum,
|
AttemptNum: attemptNum,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1426,7 +1426,6 @@ func iHTTPTaskAndDelivery(
|
|||||||
Name: name,
|
Name: name,
|
||||||
Type: database.TargetTypeHTTP,
|
Type: database.TargetTypeHTTP,
|
||||||
Config: config,
|
Config: config,
|
||||||
MaxAttempts: maxAttempts,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
d.ID = del.ID
|
d.ID = del.ID
|
||||||
@@ -1467,7 +1466,7 @@ func TestDeliverHTTP_InvalidConfig(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
task, d := iHTTPTaskAndDelivery(
|
task, d := iHTTPTaskAndDelivery(
|
||||||
event, del, "bad-config", `not-json`, 1, 1,
|
event, del, "bad-config", `not-json`, 0, 1,
|
||||||
)
|
)
|
||||||
|
|
||||||
e.ExportDeliverHTTP(context.TODO(), db, d, task)
|
e.ExportDeliverHTTP(context.TODO(), db, d, task)
|
||||||
@@ -1491,7 +1490,7 @@ func TestDeliverHTTP_InvalidConfigUnrecordedStaysPending(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
task, d := iHTTPTaskAndDelivery(
|
task, d := iHTTPTaskAndDelivery(
|
||||||
event, del, "bad-config", `not-json`, 1, 1,
|
event, del, "bad-config", `not-json`, 0, 1,
|
||||||
)
|
)
|
||||||
|
|
||||||
require.NoError(t, db.Exec("drop table delivery_results").Error)
|
require.NoError(t, db.Exec("drop table delivery_results").Error)
|
||||||
|
|||||||
@@ -2,16 +2,11 @@ package delivery_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sync"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"gorm.io/gorm/clause"
|
"gorm.io/gorm/clause"
|
||||||
@@ -149,7 +144,7 @@ func seedLogTask(
|
|||||||
bodyStr := event.Body
|
bodyStr := event.Body
|
||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"hook-context-test", "", 1, 1, &bodyStr,
|
"hook-context-test", "", 0, 1, &bodyStr,
|
||||||
)
|
)
|
||||||
task.TargetType = database.TargetTypeLog
|
task.TargetType = database.TargetTypeLog
|
||||||
|
|
||||||
@@ -277,159 +272,11 @@ func TestEngine_StopHookHonoursStopTimeout(t *testing.T) {
|
|||||||
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
||||||
}
|
}
|
||||||
|
|
||||||
// seedHeldDelivery gives the setup's webhook an http target with
|
|
||||||
// maxAttempts, pointed at a receiver that keeps every request open until
|
|
||||||
// the client gives up on it, and seeds one delivery to that target at
|
|
||||||
// status. It returns the task that sends attempt attemptNum of the
|
|
||||||
// delivery, and a channel that is closed when a request reaches the
|
|
||||||
// receiver.
|
|
||||||
func seedHeldDelivery(
|
|
||||||
t *testing.T,
|
|
||||||
s iSetup,
|
|
||||||
maxAttempts int,
|
|
||||||
status database.DeliveryStatus,
|
|
||||||
attemptNum int,
|
|
||||||
) (delivery.Task, <-chan struct{}) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
arrived := make(chan struct{})
|
|
||||||
|
|
||||||
var once sync.Once
|
|
||||||
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(_ http.ResponseWriter, r *http.Request) {
|
|
||||||
// The server notices the client hanging up only once the
|
|
||||||
// body has been read; until then r.Context() never ends.
|
|
||||||
_, _ = io.Copy(io.Discard, r.Body)
|
|
||||||
|
|
||||||
once.Do(func() { close(arrived) })
|
|
||||||
|
|
||||||
<-r.Context().Done()
|
|
||||||
},
|
|
||||||
))
|
|
||||||
t.Cleanup(ts.Close)
|
|
||||||
|
|
||||||
cfg := iHTTPConfig(ts.URL)
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
|
|
||||||
iCreateWebhook(t, s.MainDB, s.WebhookID, "hook")
|
|
||||||
iCreateTarget(
|
|
||||||
t, s.MainDB, targetID, s.WebhookID, "held",
|
|
||||||
database.TargetTypeHTTP, cfg, maxAttempts,
|
|
||||||
)
|
|
||||||
|
|
||||||
event := iSeedEvent(t, s.WebhookDB, s.WebhookID, `{"held":true}`)
|
|
||||||
d := iSeedDelivery(t, s.WebhookDB, event.ID, targetID, status)
|
|
||||||
|
|
||||||
bodyStr := event.Body
|
|
||||||
|
|
||||||
return iTask(
|
|
||||||
d, event, s.WebhookID, targetID, "held", cfg,
|
|
||||||
maxAttempts, attemptNum, &bodyStr,
|
|
||||||
), arrived
|
|
||||||
}
|
|
||||||
|
|
||||||
// stopDuringAttempt waits until the receiver holds an attempt open, then
|
|
||||||
// stops the engine through its real stop hook.
|
|
||||||
func stopDuringAttempt(
|
|
||||||
t *testing.T, lc *recordingLifecycle, arrived <-chan struct{},
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-arrived:
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("the delivery attempt never reached the receiver")
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, lc.hooks[0].OnStop(context.Background()))
|
|
||||||
}
|
|
||||||
|
|
||||||
// requireRequeued runs restart recovery on the stopped engine and
|
|
||||||
// requires it to queue task's attempt again on ch.
|
|
||||||
func requireRequeued(
|
|
||||||
t *testing.T, s iSetup, ch <-chan delivery.Task, task delivery.Task,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
s.Engine.ExportRecoverWebhookDeliveries(
|
|
||||||
context.Background(), s.WebhookID,
|
|
||||||
)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case got := <-ch:
|
|
||||||
assert.Equal(t, task.DeliveryID, got.DeliveryID)
|
|
||||||
assert.Equal(t, task.AttemptNum, got.AttemptNum)
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("restart recovery did not queue the delivery again")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEngine_StopLeavesFireAndForgetAttemptPending is the regression
|
|
||||||
// test for a stop that turned an attempt in flight into a terminal
|
|
||||||
// failure. Stopping the engine cancels the request; for a target with
|
|
||||||
// no retries that was the delivery's only attempt, recorded as failed
|
|
||||||
// with "context canceled", and nothing ever sent it again. The
|
|
||||||
// delivery must instead stay pending, with no result and no count in
|
|
||||||
// the attempt metrics, for restart recovery to send.
|
|
||||||
func TestEngine_StopLeavesFireAndForgetAttemptPending(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
reg := mIsolate(t, s)
|
|
||||||
lc := startEngineViaHook(t, s.Engine)
|
|
||||||
|
|
||||||
task, arrived := seedHeldDelivery(
|
|
||||||
t, s, 1, database.DeliveryStatusPending, 1,
|
|
||||||
)
|
|
||||||
|
|
||||||
s.Engine.Notify([]delivery.Task{task})
|
|
||||||
stopDuringAttempt(t, lc, arrived)
|
|
||||||
|
|
||||||
iAssertStatus(
|
|
||||||
t, s.WebhookDB, task.DeliveryID, database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
assert.Empty(t, iResults(t, s.WebhookDB, task.DeliveryID))
|
|
||||||
assert.Zero(t, mCounter(t, reg, mAttempts, mTypeHTTP))
|
|
||||||
assert.Zero(t, mHTTPDurations(t, reg))
|
|
||||||
|
|
||||||
requireRequeued(t, s, s.Engine.ExportDeliveryCh(), task)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEngine_StopLeavesLastRetryAttemptRetrying is the same regression
|
|
||||||
// for a target with retries, on its last attempt: the cancelled attempt
|
|
||||||
// used to exhaust the retries and end the delivery failed. It must stay
|
|
||||||
// retrying with only its earlier attempt recorded, and restart recovery
|
|
||||||
// must send that last attempt again.
|
|
||||||
func TestEngine_StopLeavesLastRetryAttemptRetrying(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const maxAttempts = 2
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
lc := startEngineViaHook(t, s.Engine)
|
|
||||||
|
|
||||||
task, arrived := seedHeldDelivery(
|
|
||||||
t, s, maxAttempts, database.DeliveryStatusRetrying, maxAttempts,
|
|
||||||
)
|
|
||||||
iSeedFailedResult(t, s.WebhookDB, task.DeliveryID)
|
|
||||||
|
|
||||||
s.Engine.ExportScheduleRetry(task, 0)
|
|
||||||
stopDuringAttempt(t, lc, arrived)
|
|
||||||
|
|
||||||
iAssertStatus(
|
|
||||||
t, s.WebhookDB, task.DeliveryID, database.DeliveryStatusRetrying,
|
|
||||||
)
|
|
||||||
assert.Len(t, iResults(t, s.WebhookDB, task.DeliveryID), 1)
|
|
||||||
|
|
||||||
requireRequeued(t, s, s.Engine.ExportRetryCh(), task)
|
|
||||||
}
|
|
||||||
|
|
||||||
// deliverToArchive gives the setup's webhook a database target,
|
// deliverToArchive gives the setup's webhook a database target,
|
||||||
// runs one delivery to it through the running engine, and returns
|
// runs one delivery to it through the running engine, and returns
|
||||||
// the target's archive file path. The archive writer holds the file
|
// the target's ID and archive file path. The archive writer holds
|
||||||
// open afterwards.
|
// the file open afterwards.
|
||||||
func deliverToArchive(t *testing.T, s iSetup) string {
|
func deliverToArchive(t *testing.T, s iSetup) (string, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
iCreateWebhook(t, s.MainDB, s.WebhookID, "hook")
|
iCreateWebhook(t, s.MainDB, s.WebhookID, "hook")
|
||||||
@@ -438,7 +285,6 @@ func deliverToArchive(t *testing.T, s iSetup) string {
|
|||||||
WebhookID: s.WebhookID,
|
WebhookID: s.WebhookID,
|
||||||
Name: "archive",
|
Name: "archive",
|
||||||
Type: database.TargetTypeDatabase,
|
Type: database.TargetTypeDatabase,
|
||||||
MaxAttempts: 1,
|
|
||||||
}
|
}
|
||||||
require.NoError(
|
require.NoError(
|
||||||
t, s.MainDB.Omit(clause.Associations).Create(tgt).Error,
|
t, s.MainDB.Omit(clause.Associations).Create(tgt).Error,
|
||||||
@@ -452,7 +298,7 @@ func deliverToArchive(t *testing.T, s iSetup) string {
|
|||||||
|
|
||||||
iWaitForDelivered(t, s.WebhookDB, deliveryID)
|
iWaitForDelivered(t, s.WebhookDB, deliveryID)
|
||||||
|
|
||||||
return filepath.Join(
|
return tgt.ID, filepath.Join(
|
||||||
filepath.Dir(s.DBMgr.DBPath(s.WebhookID)),
|
filepath.Dir(s.DBMgr.DBPath(s.WebhookID)),
|
||||||
"archive-hook-archive-"+tgt.ID+".db",
|
"archive-hook-archive-"+tgt.ID+".db",
|
||||||
)
|
)
|
||||||
@@ -471,7 +317,7 @@ func TestEngine_StopHookClosesArchives(t *testing.T) {
|
|||||||
|
|
||||||
lc := startEngineViaHook(t, s.Engine)
|
lc := startEngineViaHook(t, s.Engine)
|
||||||
|
|
||||||
path := deliverToArchive(t, s)
|
_, path := deliverToArchive(t, s)
|
||||||
require.FileExists(
|
require.FileExists(
|
||||||
t, path+"-wal",
|
t, path+"-wal",
|
||||||
"an open archive should have a -wal for the stop to remove",
|
"an open archive should have a -wal for the stop to remove",
|
||||||
@@ -505,23 +351,21 @@ func TestEngine_StopHookTimeoutLeavesArchivesOpen(t *testing.T) {
|
|||||||
|
|
||||||
lc := startEngineViaHook(t, s.Engine)
|
lc := startEngineViaHook(t, s.Engine)
|
||||||
|
|
||||||
path := deliverToArchive(t, s)
|
targetID, _ := deliverToArchive(t, s)
|
||||||
|
|
||||||
release := make(chan struct{})
|
release := make(chan struct{})
|
||||||
|
|
||||||
t.Cleanup(func() {
|
t.Cleanup(func() {
|
||||||
close(release)
|
close(release)
|
||||||
s.Engine.ExportArchives().EvictWebhook(s.WebhookID)
|
s.Engine.EvictWebhook(s.WebhookID)
|
||||||
})
|
})
|
||||||
|
|
||||||
s.Engine.ExportWedgeWorker(release)
|
s.Engine.ExportWedgeWorker(release)
|
||||||
|
|
||||||
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
||||||
|
|
||||||
// Closing the last handle on an archive removes its -wal, so a
|
require.True(
|
||||||
// -wal still there is a writer still open.
|
t, s.Engine.ExportArchiveHandleOpen(targetID),
|
||||||
require.FileExists(
|
|
||||||
t, path+"-wal",
|
|
||||||
"a stop that timed out must not close archive writers",
|
"a stop that timed out must not close archive writers",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -142,7 +142,7 @@ func buildHTTPFixture(
|
|||||||
dlv database.Delivery,
|
dlv database.Delivery,
|
||||||
event database.Event,
|
event database.Event,
|
||||||
targetID, name, config string,
|
targetID, name, config string,
|
||||||
maxAttempts, attemptNum int,
|
maxRetries, attemptNum int,
|
||||||
) httpDeliveryFixture {
|
) httpDeliveryFixture {
|
||||||
task := &delivery.Task{
|
task := &delivery.Task{
|
||||||
DeliveryID: dlv.ID,
|
DeliveryID: dlv.ID,
|
||||||
@@ -152,7 +152,7 @@ func buildHTTPFixture(
|
|||||||
TargetName: name,
|
TargetName: name,
|
||||||
TargetType: database.TargetTypeHTTP,
|
TargetType: database.TargetTypeHTTP,
|
||||||
TargetConfig: config,
|
TargetConfig: config,
|
||||||
MaxAttempts: maxAttempts,
|
MaxRetries: maxRetries,
|
||||||
AttemptNum: attemptNum,
|
AttemptNum: attemptNum,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -165,7 +165,7 @@ func buildHTTPFixture(
|
|||||||
Name: name,
|
Name: name,
|
||||||
Type: database.TargetTypeHTTP,
|
Type: database.TargetTypeHTTP,
|
||||||
Config: config,
|
Config: config,
|
||||||
MaxAttempts: maxAttempts,
|
MaxRetries: maxRetries,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
d.ID = dlv.ID
|
d.ID = dlv.ID
|
||||||
@@ -277,7 +277,7 @@ func TestDeliverHTTP_Success(t *testing.T) {
|
|||||||
|
|
||||||
cfg := newHTTPTargetConfig(ts.URL)
|
cfg := newHTTPTargetConfig(ts.URL)
|
||||||
fix := buildHTTPFixture(
|
fix := buildHTTPFixture(
|
||||||
dlv, event, targetID, "test-http", cfg, 1, 1,
|
dlv, event, targetID, "test-http", cfg, 0, 1,
|
||||||
)
|
)
|
||||||
|
|
||||||
e.ExportDeliverHTTP(
|
e.ExportDeliverHTTP(
|
||||||
@@ -324,7 +324,7 @@ func TestDeliverHTTP_Failure(t *testing.T) {
|
|||||||
cfg := newHTTPTargetConfig(ts.URL)
|
cfg := newHTTPTargetConfig(ts.URL)
|
||||||
fix := buildHTTPFixture(
|
fix := buildHTTPFixture(
|
||||||
dlv, event, targetID,
|
dlv, event, targetID,
|
||||||
"test-http-fail", cfg, 1, 1,
|
"test-http-fail", cfg, 0, 1,
|
||||||
)
|
)
|
||||||
|
|
||||||
e.ExportDeliverHTTP(
|
e.ExportDeliverHTTP(
|
||||||
@@ -473,23 +473,14 @@ func TestDeliverHTTP_WithRetries_Success(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestDeliverHTTP_MaxAttemptsExhausted proves a target with three
|
func TestDeliverHTTP_MaxRetriesExhausted(t *testing.T) {
|
||||||
// delivery attempts makes at most three: the first two failures are
|
|
||||||
// retried, the third is final, and each attempt goes through the
|
|
||||||
// target's circuit breaker.
|
|
||||||
func TestDeliverHTTP_MaxAttemptsExhausted(t *testing.T) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const maxAttempts = 3
|
|
||||||
|
|
||||||
db := testWebhookDB(t)
|
db := testWebhookDB(t)
|
||||||
|
|
||||||
var requests atomic.Int32
|
|
||||||
|
|
||||||
ts := httptest.NewServer(
|
ts := httptest.NewServer(
|
||||||
http.HandlerFunc(
|
http.HandlerFunc(
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
func(w http.ResponseWriter, _ *http.Request) {
|
||||||
requests.Add(1)
|
|
||||||
w.WriteHeader(http.StatusBadGateway)
|
w.WriteHeader(http.StatusBadGateway)
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -498,103 +489,26 @@ func TestDeliverHTTP_MaxAttemptsExhausted(t *testing.T) {
|
|||||||
|
|
||||||
e := testEngine(t, 1)
|
e := testEngine(t, 1)
|
||||||
targetID := uuid.New().String()
|
targetID := uuid.New().String()
|
||||||
|
|
||||||
// It opens on the last failure only if every attempt reached it.
|
|
||||||
cb := delivery.NewTestCircuitBreaker(maxAttempts, time.Minute)
|
|
||||||
e.ExportSetCircuitBreaker(targetID, cb)
|
|
||||||
|
|
||||||
event := seedEvent(t, db, `{"retry":"exhaust"}`)
|
event := seedEvent(t, db, `{"retry":"exhaust"}`)
|
||||||
|
|
||||||
dlv := seedDelivery(
|
dlv := seedDelivery(
|
||||||
t, db, event.ID, targetID,
|
t, db, event.ID, targetID,
|
||||||
database.DeliveryStatusPending,
|
database.DeliveryStatusRetrying,
|
||||||
)
|
|
||||||
|
|
||||||
cfg := newHTTPTargetConfig(ts.URL)
|
|
||||||
sched := &recordingScheduler{}
|
|
||||||
|
|
||||||
for attempt := 1; attempt <= maxAttempts; attempt++ {
|
|
||||||
fix := buildHTTPFixture(
|
|
||||||
dlv, event, targetID,
|
|
||||||
"test-http-exhaust", cfg, maxAttempts, attempt,
|
|
||||||
)
|
|
||||||
|
|
||||||
e.ExportDeliverHTTPWithScheduler(
|
|
||||||
context.TODO(), db, fix.Delivery, fix.Task, sched,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, int32(maxAttempts), requests.Load())
|
|
||||||
assert.Len(t, sched.delays, maxAttempts-1,
|
|
||||||
"every failure but the last should be retried",
|
|
||||||
)
|
|
||||||
assertDeliveryStatus(t, db, dlv.ID,
|
|
||||||
database.DeliveryStatusFailed,
|
|
||||||
)
|
|
||||||
assert.Equal(t, delivery.CircuitOpen, cb.State(),
|
|
||||||
"every attempt should be counted by the circuit breaker",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDeliverHTTP_SingleAttemptSkipsCircuitBreaker proves a target
|
|
||||||
// with one delivery attempt makes exactly that attempt, even while
|
|
||||||
// its circuit breaker is open, and that a failure is final.
|
|
||||||
func TestDeliverHTTP_SingleAttemptSkipsCircuitBreaker(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := testWebhookDB(t)
|
|
||||||
|
|
||||||
var requests atomic.Int32
|
|
||||||
|
|
||||||
ts := httptest.NewServer(
|
|
||||||
http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
requests.Add(1)
|
|
||||||
w.WriteHeader(http.StatusBadGateway)
|
|
||||||
},
|
|
||||||
),
|
|
||||||
)
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
e := testEngine(t, 1)
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
|
|
||||||
cb := e.ExportGetCircuitBreaker(targetID)
|
|
||||||
|
|
||||||
for range delivery.ExportDefaultFailureThreshold {
|
|
||||||
cb.RecordFailure()
|
|
||||||
}
|
|
||||||
|
|
||||||
require.Equal(t, delivery.CircuitOpen, cb.State())
|
|
||||||
|
|
||||||
event := seedEvent(t, db, `{"retry":"none"}`)
|
|
||||||
|
|
||||||
dlv := seedDelivery(
|
|
||||||
t, db, event.ID, targetID,
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
cfg := newHTTPTargetConfig(ts.URL)
|
cfg := newHTTPTargetConfig(ts.URL)
|
||||||
fix := buildHTTPFixture(
|
fix := buildHTTPFixture(
|
||||||
dlv, event, targetID,
|
dlv, event, targetID,
|
||||||
"test-http-single", cfg, 1, 1,
|
"test-http-exhaust", cfg, 3, 3,
|
||||||
)
|
|
||||||
sched := &recordingScheduler{}
|
|
||||||
|
|
||||||
e.ExportDeliverHTTPWithScheduler(
|
|
||||||
context.TODO(), db, fix.Delivery, fix.Task, sched,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Equal(t, int32(1), requests.Load(),
|
e.ExportDeliverHTTP(
|
||||||
"the open circuit breaker should not hold the attempt back",
|
context.TODO(), db, fix.Delivery, fix.Task,
|
||||||
)
|
)
|
||||||
assert.Empty(t, sched.delays, "no retry should be scheduled")
|
|
||||||
assertDeliveryStatus(t, db, dlv.ID,
|
assertDeliveryStatus(t, db, dlv.ID,
|
||||||
database.DeliveryStatusFailed,
|
database.DeliveryStatusFailed,
|
||||||
)
|
)
|
||||||
assertDeliveryResult(
|
|
||||||
t, db, dlv.ID, false, http.StatusBadGateway,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDeliverHTTP_SchedulesRetryOnFailure(
|
func TestDeliverHTTP_SchedulesRetryOnFailure(
|
||||||
@@ -816,7 +730,6 @@ func seedConcurrencyTasks(
|
|||||||
Name: fmt.Sprintf("task-%d", i),
|
Name: fmt.Sprintf("task-%d", i),
|
||||||
Type: database.TargetTypeHTTP,
|
Type: database.TargetTypeHTTP,
|
||||||
Config: targetCfg,
|
Config: targetCfg,
|
||||||
MaxAttempts: 1,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
tasks[i].ID = dlv.ID
|
tasks[i].ID = dlv.ID
|
||||||
@@ -828,7 +741,7 @@ func seedConcurrencyTasks(
|
|||||||
TargetName: tasks[i].Target.Name,
|
TargetName: tasks[i].Target.Name,
|
||||||
TargetType: tasks[i].Target.Type,
|
TargetType: tasks[i].Target.Type,
|
||||||
TargetConfig: tasks[i].Target.Config,
|
TargetConfig: tasks[i].Target.Config,
|
||||||
MaxAttempts: 1,
|
MaxRetries: 0,
|
||||||
AttemptNum: 1,
|
AttemptNum: 1,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1778,7 +1691,6 @@ func buildSlackDelivery(
|
|||||||
Name: name,
|
Name: name,
|
||||||
Type: database.TargetTypeSlack,
|
Type: database.TargetTypeSlack,
|
||||||
Config: slackCfg,
|
Config: slackCfg,
|
||||||
MaxAttempts: 1,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
d.ID = dlv.ID
|
d.ID = dlv.ID
|
||||||
@@ -1965,7 +1877,6 @@ func TestDeliverSlack_InvalidConfig(t *testing.T) {
|
|||||||
Name: "test-slack-bad",
|
Name: "test-slack-bad",
|
||||||
Type: database.TargetTypeSlack,
|
Type: database.TargetTypeSlack,
|
||||||
Config: `{"not_webhook_url":"missing"}`,
|
Config: `{"not_webhook_url":"missing"}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
d.ID = dlv.ID
|
d.ID = dlv.ID
|
||||||
@@ -2024,7 +1935,6 @@ func TestProcessDelivery_RoutesToSlack(t *testing.T) {
|
|||||||
task := &delivery.Task{
|
task := &delivery.Task{
|
||||||
DeliveryID: dlv.ID,
|
DeliveryID: dlv.ID,
|
||||||
TargetType: database.TargetTypeSlack,
|
TargetType: database.TargetTypeSlack,
|
||||||
MaxAttempts: 1,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
e.ExportProcessDelivery(
|
e.ExportProcessDelivery(
|
||||||
@@ -2144,7 +2054,7 @@ func buildSlackRetryDelivery(
|
|||||||
Name: "test-slack-retry",
|
Name: "test-slack-retry",
|
||||||
Type: database.TargetTypeSlack,
|
Type: database.TargetTypeSlack,
|
||||||
Config: cfg,
|
Config: cfg,
|
||||||
MaxAttempts: 5,
|
MaxRetries: 5,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
d.ID = dlv.ID
|
d.ID = dlv.ID
|
||||||
@@ -2182,7 +2092,7 @@ func TestDeliverSlack_WithRetries_SchedulesRetry(
|
|||||||
DeliveryID: dlv.ID,
|
DeliveryID: dlv.ID,
|
||||||
TargetID: targetID,
|
TargetID: targetID,
|
||||||
TargetType: database.TargetTypeSlack,
|
TargetType: database.TargetTypeSlack,
|
||||||
MaxAttempts: 5,
|
MaxRetries: 5,
|
||||||
AttemptNum: 1,
|
AttemptNum: 1,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -152,7 +152,7 @@ func TestEventDBHoldsNoTargetRowsOnFailedDelivery(t *testing.T) {
|
|||||||
)
|
)
|
||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"leaky-target", cfg, 1, 1, &body,
|
"leaky-target", cfg, 0, 1, &body,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(context.TODO(), &task)
|
s.Engine.ExportProcessNewTask(context.TODO(), &task)
|
||||||
|
|||||||
@@ -152,7 +152,6 @@ func tsSeedTarget(
|
|||||||
Type: database.TargetTypeSlack,
|
Type: database.TargetTypeSlack,
|
||||||
Config: config,
|
Config: config,
|
||||||
Active: true,
|
Active: true,
|
||||||
MaxAttempts: 1,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
require.NoError(t, mainDB.Create(&target).Error)
|
require.NoError(t, mainDB.Create(&target).Error)
|
||||||
@@ -177,7 +176,7 @@ func tsTask(
|
|||||||
TargetName: target.Name,
|
TargetName: target.Name,
|
||||||
TargetType: database.TargetTypeSlack,
|
TargetType: database.TargetTypeSlack,
|
||||||
TargetConfig: target.Config,
|
TargetConfig: target.Config,
|
||||||
MaxAttempts: 1,
|
MaxRetries: 0,
|
||||||
Method: event.Method,
|
Method: event.Method,
|
||||||
Headers: event.Headers,
|
Headers: event.Headers,
|
||||||
ContentType: event.ContentType,
|
ContentType: event.ContentType,
|
||||||
|
|||||||
@@ -12,11 +12,16 @@ import (
|
|||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/metrics"
|
"sneak.berlin/go/webhooker/internal/metrics"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ErrExportArchiveWriterEvicted exposes the sentinel returned by
|
||||||
|
// an evicted archive writer. It carries the Err prefix rather
|
||||||
|
// than this file's usual Export one because it is a sentinel
|
||||||
|
// error.
|
||||||
|
var ErrExportArchiveWriterEvicted = errArchiveWriterEvicted
|
||||||
|
|
||||||
// Exported constants for test access.
|
// Exported constants for test access.
|
||||||
const (
|
const (
|
||||||
ExportDeliveryChannelSize = deliveryChannelSize
|
ExportDeliveryChannelSize = deliveryChannelSize
|
||||||
@@ -139,7 +144,6 @@ func (e *Engine) ExportDeliverSlack(
|
|||||||
task := &Task{
|
task := &Task{
|
||||||
DeliveryID: d.ID,
|
DeliveryID: d.ID,
|
||||||
TargetID: d.TargetID,
|
TargetID: d.TargetID,
|
||||||
MaxAttempts: d.Target.MaxAttempts,
|
|
||||||
AttemptNum: 1,
|
AttemptNum: 1,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -394,8 +398,7 @@ func (e *Engine) ExportRetryCh() chan Task {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NewTestEngine creates an Engine for unit tests without
|
// NewTestEngine creates an Engine for unit tests without
|
||||||
// database dependencies. Its archive writers have no data
|
// database dependencies.
|
||||||
// directory, so a delivery to a database target fails.
|
|
||||||
func NewTestEngine(
|
func NewTestEngine(
|
||||||
log *slog.Logger,
|
log *slog.Logger,
|
||||||
client *http.Client,
|
client *http.Client,
|
||||||
@@ -407,7 +410,6 @@ func NewTestEngine(
|
|||||||
retryCh: make(chan Task, retryChannelSize),
|
retryCh: make(chan Task, retryChannelSize),
|
||||||
workers: workers,
|
workers: workers,
|
||||||
mtr: metrics.New(prometheus.NewRegistry()),
|
mtr: metrics.New(prometheus.NewRegistry()),
|
||||||
archives: archive.NewWriters(archive.WritersParams{Logger: log}),
|
|
||||||
}
|
}
|
||||||
e.initTargets(client)
|
e.initTargets(client)
|
||||||
|
|
||||||
@@ -423,7 +425,6 @@ func NewTestEngineSmallRetry(
|
|||||||
log: log,
|
log: log,
|
||||||
retryCh: make(chan Task, 1),
|
retryCh: make(chan Task, 1),
|
||||||
mtr: metrics.New(prometheus.NewRegistry()),
|
mtr: metrics.New(prometheus.NewRegistry()),
|
||||||
archives: archive.NewWriters(archive.WritersParams{Logger: log}),
|
|
||||||
}
|
}
|
||||||
e.initTargets(nil)
|
e.initTargets(nil)
|
||||||
|
|
||||||
@@ -431,8 +432,7 @@ func NewTestEngineSmallRetry(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NewTestEngineWithDB creates an Engine with a real
|
// NewTestEngineWithDB creates an Engine with a real
|
||||||
// database and dbManager for integration tests, and archive
|
// database and dbManager for integration tests.
|
||||||
// writers of its own over the same two.
|
|
||||||
func NewTestEngineWithDB(
|
func NewTestEngineWithDB(
|
||||||
db *database.Database,
|
db *database.Database,
|
||||||
dbMgr *database.WebhookDBManager,
|
dbMgr *database.WebhookDBManager,
|
||||||
@@ -448,22 +448,12 @@ func NewTestEngineWithDB(
|
|||||||
retryCh: make(chan Task, retryChannelSize),
|
retryCh: make(chan Task, retryChannelSize),
|
||||||
workers: workers,
|
workers: workers,
|
||||||
mtr: metrics.New(prometheus.NewRegistry()),
|
mtr: metrics.New(prometheus.NewRegistry()),
|
||||||
archives: archive.NewWriters(archive.WritersParams{
|
|
||||||
Database: db,
|
|
||||||
DBManager: dbMgr,
|
|
||||||
Logger: log,
|
|
||||||
}),
|
|
||||||
}
|
}
|
||||||
e.initTargets(client)
|
e.initTargets(client)
|
||||||
|
|
||||||
return e
|
return e
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportArchives returns the engine's archive writers.
|
|
||||||
func (e *Engine) ExportArchives() *archive.Writers {
|
|
||||||
return e.archives
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportSetMetrics substitutes the engine's metric set, so a test can
|
// ExportSetMetrics substitutes the engine's metric set, so a test can
|
||||||
// assert on collectors registered on a registry it holds.
|
// assert on collectors registered on a registry it holds.
|
||||||
func (e *Engine) ExportSetMetrics(mtr *metrics.Set) {
|
func (e *Engine) ExportSetMetrics(mtr *metrics.Set) {
|
||||||
@@ -486,3 +476,273 @@ func NewTestCircuitBreaker(
|
|||||||
cooldown: cooldown,
|
cooldown: cooldown,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ExportArchivedEvent aliases the archive row type so black-box
|
||||||
|
// tests can construct and read archive rows.
|
||||||
|
type ExportArchivedEvent = archivedEvent
|
||||||
|
|
||||||
|
// ExportArchiveWriter wraps an archiveWriter so black-box tests
|
||||||
|
// can exercise the archive file mechanics.
|
||||||
|
type ExportArchiveWriter struct {
|
||||||
|
w *archiveWriter
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewExportArchiveWriter builds an archive writer for tests,
|
||||||
|
// optionally overriding the reopen debounce (a non-positive
|
||||||
|
// debounce keeps the production default).
|
||||||
|
func NewExportArchiveWriter(
|
||||||
|
path string, log *slog.Logger, debounce time.Duration,
|
||||||
|
) *ExportArchiveWriter {
|
||||||
|
w := newArchiveWriter(path, log)
|
||||||
|
if debounce > 0 {
|
||||||
|
w.debounce = debounce
|
||||||
|
}
|
||||||
|
|
||||||
|
return &ExportArchiveWriter{w: w}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write archives a row through the writer, into the file named
|
||||||
|
// without a period.
|
||||||
|
func (e *ExportArchiveWriter) Write(
|
||||||
|
row ExportArchivedEvent, expiry time.Duration,
|
||||||
|
) error {
|
||||||
|
return e.w.write(row, expiry, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// WritePeriod archives a row through the writer, into the file for
|
||||||
|
// period.
|
||||||
|
func (e *ExportArchiveWriter) WritePeriod(
|
||||||
|
row ExportArchivedEvent, expiry time.Duration, period string,
|
||||||
|
) error {
|
||||||
|
return e.w.write(row, expiry, period)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open opens the archive file, pruning when expiry is positive.
|
||||||
|
func (e *ExportArchiveWriter) Open(expiry time.Duration) error {
|
||||||
|
return e.w.open(e.w.path, expiry)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reopen closes and reopens the archive file.
|
||||||
|
func (e *ExportArchiveWriter) Reopen(
|
||||||
|
expiry time.Duration,
|
||||||
|
) error {
|
||||||
|
return e.w.reopen(e.w.path, expiry)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetNow replaces the clock the writer measures its reopen
|
||||||
|
// debounce on.
|
||||||
|
func (e *ExportArchiveWriter) SetNow(now func() time.Time) {
|
||||||
|
e.w.now = now
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reopens reports how many times the file has been opened.
|
||||||
|
func (e *ExportArchiveWriter) Reopens() int {
|
||||||
|
return e.w.reopens
|
||||||
|
}
|
||||||
|
|
||||||
|
// DB returns the writer's current open handle for row
|
||||||
|
// inspection in tests.
|
||||||
|
func (e *ExportArchiveWriter) DB() *gorm.DB {
|
||||||
|
return e.w.db
|
||||||
|
}
|
||||||
|
|
||||||
|
// Path returns the archive file the writer owns.
|
||||||
|
func (e *ExportArchiveWriter) Path() string {
|
||||||
|
return e.w.path
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenExisting opens the archive without permitting creation,
|
||||||
|
// the way the idle sweep does.
|
||||||
|
func (e *ExportArchiveWriter) OpenExisting(
|
||||||
|
expiry time.Duration,
|
||||||
|
) error {
|
||||||
|
return e.w.openMode(e.w.path, archiveModeExisting, expiry)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SweepExpired runs an idle sweep of the archive.
|
||||||
|
func (e *ExportArchiveWriter) SweepExpired(
|
||||||
|
expiry time.Duration,
|
||||||
|
) error {
|
||||||
|
return e.w.sweepExpired(expiry)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Evict marks the writer evicted and closes its handle, exactly
|
||||||
|
// as leaving the registry does.
|
||||||
|
func (e *ExportArchiveWriter) Evict() {
|
||||||
|
e.w.evict()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rename gives the archive file a new name in the same directory,
|
||||||
|
// as a rename of the webhook or target does.
|
||||||
|
func (e *ExportArchiveWriter) Rename(name string) error {
|
||||||
|
return e.w.rename(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// HandleOpen reports whether the writer currently holds an open
|
||||||
|
// archive handle.
|
||||||
|
func (e *ExportArchiveWriter) HandleOpen() bool {
|
||||||
|
e.w.mu.Lock()
|
||||||
|
defer e.w.mu.Unlock()
|
||||||
|
|
||||||
|
return e.w.db != nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Same reports whether both wrappers refer to the very same
|
||||||
|
// underlying archive writer, so a test can prove a registry entry
|
||||||
|
// is the writer it was handed rather than a replacement.
|
||||||
|
func (e *ExportArchiveWriter) Same(
|
||||||
|
other *ExportArchiveWriter,
|
||||||
|
) bool {
|
||||||
|
return other != nil && e.w == other.w
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportArchiveWriterFor returns the archive writer the registry
|
||||||
|
// currently caches for a database target, or nil when none is
|
||||||
|
// cached. It never creates one, so a test can hold a reference to
|
||||||
|
// the very writer an eviction is about to detach.
|
||||||
|
func (e *Engine) ExportArchiveWriterFor(
|
||||||
|
targetID string,
|
||||||
|
) *ExportArchiveWriter {
|
||||||
|
e.dbTarget.mu.Lock()
|
||||||
|
defer e.dbTarget.mu.Unlock()
|
||||||
|
|
||||||
|
w, ok := e.dbTarget.writers[targetID]
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return &ExportArchiveWriter{w: w}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportHasArchiveWriter reports whether the database target
|
||||||
|
// type currently caches an archive writer for a target.
|
||||||
|
func (e *Engine) ExportHasArchiveWriter(
|
||||||
|
targetID string,
|
||||||
|
) bool {
|
||||||
|
e.dbTarget.mu.Lock()
|
||||||
|
defer e.dbTarget.mu.Unlock()
|
||||||
|
|
||||||
|
_, ok := e.dbTarget.writers[targetID]
|
||||||
|
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportArchiveHandleOpen reports whether the cached archive
|
||||||
|
// writer for a target holds an open database handle. It
|
||||||
|
// returns false when no writer is cached.
|
||||||
|
func (e *Engine) ExportArchiveHandleOpen(
|
||||||
|
targetID string,
|
||||||
|
) bool {
|
||||||
|
e.dbTarget.mu.Lock()
|
||||||
|
w, ok := e.dbTarget.writers[targetID]
|
||||||
|
e.dbTarget.mu.Unlock()
|
||||||
|
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
|
||||||
|
return w.db != nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportEnsureArchiveWriter creates (if needed) and returns the
|
||||||
|
// archive file path of the cached writer for a target, so a
|
||||||
|
// test can prime the registry the way a delivery would.
|
||||||
|
func (e *Engine) ExportEnsureArchiveWriter(
|
||||||
|
targetID string,
|
||||||
|
) (string, error) {
|
||||||
|
w, err := e.dbTarget.writerFor(targetID)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return w.path, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportSweepWriterFor takes a target's registry writer exactly
|
||||||
|
// as the idle sweep does, reporting whether the sweep had to
|
||||||
|
// create the entry. It lets a test drive the registry through the
|
||||||
|
// sweep's own entry point instead of choreographing goroutines.
|
||||||
|
func (e *Engine) ExportSweepWriterFor(
|
||||||
|
targetID string,
|
||||||
|
) (*ExportArchiveWriter, bool, error) {
|
||||||
|
w, created, err := e.dbTarget.sweepWriterFor(targetID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return &ExportArchiveWriter{w: w}, created, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportReleaseSweepWriter releases a sweep-created registry entry
|
||||||
|
// exactly as a finished sweep does.
|
||||||
|
func (e *Engine) ExportReleaseSweepWriter(
|
||||||
|
targetID string, w *ExportArchiveWriter,
|
||||||
|
) {
|
||||||
|
e.dbTarget.releaseSweepWriter(targetID, w.w)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewTestArchiveSweeper builds an ArchiveSweeper backed by the
|
||||||
|
// given main database and engine, without the fx lifecycle.
|
||||||
|
// Intended for tests.
|
||||||
|
func NewTestArchiveSweeper(
|
||||||
|
db *database.Database,
|
||||||
|
eng *Engine,
|
||||||
|
log *slog.Logger,
|
||||||
|
) *ArchiveSweeper {
|
||||||
|
return &ArchiveSweeper{
|
||||||
|
db: db,
|
||||||
|
eng: eng,
|
||||||
|
log: log,
|
||||||
|
interval: time.Hour,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportSweep runs a single archive sweep synchronously for
|
||||||
|
// tests.
|
||||||
|
func (s *ArchiveSweeper) ExportSweep(ctx context.Context) {
|
||||||
|
s.sweep(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportStart starts the sweeper's background loop for tests.
|
||||||
|
func (s *ArchiveSweeper) ExportStart() {
|
||||||
|
s.start()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportRegisterHooks registers the sweeper's real fx lifecycle
|
||||||
|
// hooks on a lifecycle supplied by a test, so a test can drive
|
||||||
|
// the exact OnStart/OnStop functions the application runs and
|
||||||
|
// hand OnStart the kind of context fx actually supplies.
|
||||||
|
func (s *ArchiveSweeper) ExportRegisterHooks(lc fx.Lifecycle) {
|
||||||
|
s.registerHooks(lc)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportStop stops the sweeper's background loop for tests.
|
||||||
|
func (s *ArchiveSweeper) ExportStop(ctx context.Context) error {
|
||||||
|
return s.stop(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportWedgeLoop adds a goroutine to the sweeper's WaitGroup
|
||||||
|
// that never observes cancellation and returns only when release
|
||||||
|
// is closed. It stands in for a prune stuck on a locked archive.
|
||||||
|
func (s *ArchiveSweeper) ExportWedgeLoop(
|
||||||
|
release <-chan struct{},
|
||||||
|
) {
|
||||||
|
s.wg.Go(func() {
|
||||||
|
<-release
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportSetInterval overrides the sweep interval for tests.
|
||||||
|
func (s *ArchiveSweeper) ExportSetInterval(d time.Duration) {
|
||||||
|
s.interval = d
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExportParseArchiveExpiry exposes parseArchiveExpiry.
|
||||||
|
func ExportParseArchiveExpiry(
|
||||||
|
configJSON string,
|
||||||
|
) (time.Duration, error) {
|
||||||
|
return parseArchiveExpiry(configJSON)
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ package delivery_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -30,7 +32,7 @@ func fSweepSetup(
|
|||||||
|
|
||||||
iCreateTarget(t, s.MainDB, targetID,
|
iCreateTarget(t, s.MainDB, targetID,
|
||||||
s.WebhookID, name,
|
s.WebhookID, name,
|
||||||
database.TargetTypeLog, "", 1,
|
database.TargetTypeLog, "", 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
require.NoError(t, s.MainDB.Create(&database.Webhook{
|
require.NoError(t, s.MainDB.Create(&database.Webhook{
|
||||||
@@ -64,6 +66,33 @@ func fDrain(e *delivery.Engine) []delivery.Task {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestArchiveHandleIsWAL closes the last gap in the durability
|
||||||
|
// evidence: the main and per-webhook tiers each assert their journal
|
||||||
|
// mode on a live handle, and the archive tier gets its settings from
|
||||||
|
// the same code path but nothing checked the running file.
|
||||||
|
func TestArchiveHandleIsWAL(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
w := delivery.NewExportArchiveWriter(
|
||||||
|
filepath.Join(t.TempDir(), "archive-wal.db"),
|
||||||
|
archiveTestLogger(), 0,
|
||||||
|
)
|
||||||
|
|
||||||
|
require.NoError(t, w.Open(0))
|
||||||
|
|
||||||
|
var mode string
|
||||||
|
|
||||||
|
row := w.DB().Raw("pragma journal_mode").Row()
|
||||||
|
require.NoError(t, row.Scan(&mode))
|
||||||
|
assert.Equal(t, "wal", strings.ToLower(mode))
|
||||||
|
|
||||||
|
var busy string
|
||||||
|
|
||||||
|
row = w.DB().Raw("pragma busy_timeout").Row()
|
||||||
|
require.NoError(t, row.Scan(&busy))
|
||||||
|
assert.Equal(t, "10000", busy)
|
||||||
|
}
|
||||||
|
|
||||||
// TestSweepLeavesAQueuedDeliveryAlone is the case the age bound cannot
|
// TestSweepLeavesAQueuedDeliveryAlone is the case the age bound cannot
|
||||||
// see. The delivery is queued and untouched, so its row is arbitrarily
|
// see. The delivery is queued and untouched, so its row is arbitrarily
|
||||||
// old and still perfectly healthy; only ownership distinguishes it
|
// old and still perfectly healthy; only ownership distinguishes it
|
||||||
@@ -200,7 +229,7 @@ func TestOwnershipIsReleasedAfterDelivery(t *testing.T) {
|
|||||||
|
|
||||||
iCreateTarget(t, s.MainDB, targetID,
|
iCreateTarget(t, s.MainDB, targetID,
|
||||||
s.WebhookID, "released",
|
s.WebhookID, "released",
|
||||||
database.TargetTypeLog, "", 1,
|
database.TargetTypeLog, "", 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
event := iSeedEvent(
|
event := iSeedEvent(
|
||||||
|
|||||||
@@ -291,20 +291,20 @@ func TestDeliveryMetrics_CircuitBreakerGauge(t *testing.T) {
|
|||||||
body := event.Body
|
body := event.Body
|
||||||
cfg := iHTTPConfig(ts.URL)
|
cfg := iHTTPConfig(ts.URL)
|
||||||
|
|
||||||
// An attempt budget above the failure threshold, so the breaker
|
// A retry budget above the failure threshold, so the breaker
|
||||||
// rather than the budget is what stops the delivery.
|
// rather than the budget is what stops the delivery.
|
||||||
maxAttempts := delivery.ExportDefaultFailureThreshold + 5
|
maxRetries := delivery.ExportDefaultFailureThreshold + 5
|
||||||
|
|
||||||
// The retries below are only run if their target still exists;
|
// The retries below are only run if their target still exists;
|
||||||
// see https://git.eeqj.de/sneak/webhooker/issues/107.
|
// see https://git.eeqj.de/sneak/webhooker/issues/107.
|
||||||
iCreateTarget(
|
iCreateTarget(
|
||||||
t, s.MainDB, targetID, s.WebhookID, "metrics-trip",
|
t, s.MainDB, targetID, s.WebhookID, "metrics-trip",
|
||||||
database.TargetTypeHTTP, cfg, maxAttempts,
|
database.TargetTypeHTTP, cfg, maxRetries,
|
||||||
)
|
)
|
||||||
|
|
||||||
first := iTask(
|
first := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"metrics-trip", cfg, maxAttempts, 1, &body,
|
"metrics-trip", cfg, maxRetries, 1, &body,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(context.TODO(), &first)
|
s.Engine.ExportProcessNewTask(context.TODO(), &first)
|
||||||
@@ -316,7 +316,7 @@ func TestDeliveryMetrics_CircuitBreakerGauge(t *testing.T) {
|
|||||||
ExportDefaultFailureThreshold; attempt++ {
|
ExportDefaultFailureThreshold; attempt++ {
|
||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"metrics-trip", cfg, maxAttempts, attempt, &body,
|
"metrics-trip", cfg, maxRetries, attempt, &body,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportProcessRetryTask(
|
s.Engine.ExportProcessRetryTask(
|
||||||
@@ -364,16 +364,16 @@ func TestDeliveryMetrics_BreakerBlockedIsNotAnAttempt(
|
|||||||
|
|
||||||
body := event.Body
|
body := event.Body
|
||||||
cfg := iHTTPConfig(ts.URL)
|
cfg := iHTTPConfig(ts.URL)
|
||||||
maxAttempts := delivery.ExportDefaultFailureThreshold + 5
|
maxRetries := delivery.ExportDefaultFailureThreshold + 5
|
||||||
|
|
||||||
iCreateTarget(
|
iCreateTarget(
|
||||||
t, s.MainDB, targetID, s.WebhookID, "metrics-blocked",
|
t, s.MainDB, targetID, s.WebhookID, "metrics-blocked",
|
||||||
database.TargetTypeHTTP, cfg, maxAttempts,
|
database.TargetTypeHTTP, cfg, maxRetries,
|
||||||
)
|
)
|
||||||
|
|
||||||
first := iTask(
|
first := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"metrics-blocked", cfg, maxAttempts, 1, &body,
|
"metrics-blocked", cfg, maxRetries, 1, &body,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(context.TODO(), &first)
|
s.Engine.ExportProcessNewTask(context.TODO(), &first)
|
||||||
@@ -382,7 +382,7 @@ func TestDeliveryMetrics_BreakerBlockedIsNotAnAttempt(
|
|||||||
ExportDefaultFailureThreshold; attempt++ {
|
ExportDefaultFailureThreshold; attempt++ {
|
||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"metrics-blocked", cfg, maxAttempts, attempt, &body,
|
"metrics-blocked", cfg, maxRetries, attempt, &body,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportProcessRetryTask(context.TODO(), &task)
|
s.Engine.ExportProcessRetryTask(context.TODO(), &task)
|
||||||
@@ -405,7 +405,7 @@ func TestDeliveryMetrics_BreakerBlockedIsNotAnAttempt(
|
|||||||
|
|
||||||
blocked := iTask(
|
blocked := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"metrics-blocked", cfg, maxAttempts,
|
"metrics-blocked", cfg, maxRetries,
|
||||||
delivery.ExportDefaultFailureThreshold+1, &body,
|
delivery.ExportDefaultFailureThreshold+1, &body,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -66,7 +66,7 @@ func TestRecoverySkipsPendingWithSuccessfulResult(
|
|||||||
|
|
||||||
iCreateTarget(t, s.MainDB, targetID,
|
iCreateTarget(t, s.MainDB, targetID,
|
||||||
s.WebhookID, "already-delivered",
|
s.WebhookID, "already-delivered",
|
||||||
database.TargetTypeLog, "", 1,
|
database.TargetTypeLog, "", 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
event := iSeedEvent(
|
event := iSeedEvent(
|
||||||
@@ -129,7 +129,7 @@ func TestRecoveryContinuesTheAttemptNumbering(t *testing.T) {
|
|||||||
|
|
||||||
iCreateTarget(t, s.MainDB, targetID,
|
iCreateTarget(t, s.MainDB, targetID,
|
||||||
s.WebhookID, "numbering",
|
s.WebhookID, "numbering",
|
||||||
database.TargetTypeLog, "", 1,
|
database.TargetTypeLog, "", 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
event := iSeedEvent(
|
event := iSeedEvent(
|
||||||
@@ -357,14 +357,13 @@ func TestFailedResultWriteLeavesDeliveryRecoverable(
|
|||||||
Name: "unwritable",
|
Name: "unwritable",
|
||||||
Type: database.TargetTypeHTTP,
|
Type: database.TargetTypeHTTP,
|
||||||
Config: iHTTPConfig(ts.URL),
|
Config: iHTTPConfig(ts.URL),
|
||||||
MaxAttempts: 1,
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
full.ID = d.ID
|
full.ID = d.ID
|
||||||
|
|
||||||
s.Engine.ExportDeliverHTTP(
|
s.Engine.ExportDeliverHTTP(
|
||||||
context.Background(), s.WebhookDB, full,
|
context.Background(), s.WebhookDB, full,
|
||||||
&delivery.Task{DeliveryID: d.ID, MaxAttempts: 1, AttemptNum: 1},
|
&delivery.Task{DeliveryID: d.ID, AttemptNum: 1},
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
@@ -448,7 +447,7 @@ func TestFailedResultWriteWithRetriesLeavesDeliveryRecoverable(
|
|||||||
Name: "unwritable",
|
Name: "unwritable",
|
||||||
Type: database.TargetTypeHTTP,
|
Type: database.TargetTypeHTTP,
|
||||||
Config: iHTTPConfig(ts.URL),
|
Config: iHTTPConfig(ts.URL),
|
||||||
MaxAttempts: 3,
|
MaxRetries: 3,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
full.ID = d.ID
|
full.ID = d.ID
|
||||||
@@ -458,8 +457,9 @@ func TestFailedResultWriteWithRetriesLeavesDeliveryRecoverable(
|
|||||||
s.Engine.ExportDeliverHTTPWithScheduler(
|
s.Engine.ExportDeliverHTTPWithScheduler(
|
||||||
context.Background(), s.WebhookDB, full,
|
context.Background(), s.WebhookDB, full,
|
||||||
&delivery.Task{
|
&delivery.Task{
|
||||||
DeliveryID: d.ID, TargetID: targetID,
|
DeliveryID: d.ID,
|
||||||
MaxAttempts: 3, AttemptNum: 1,
|
TargetID: targetID,
|
||||||
|
AttemptNum: 1,
|
||||||
},
|
},
|
||||||
sched,
|
sched,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -96,7 +96,7 @@ func deliverWithProbeHeaders(
|
|||||||
body := event.Body
|
body := event.Body
|
||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, targetID,
|
d, event, s.WebhookID, targetID,
|
||||||
"redirect-target", string(cfg), 1, 1, &body,
|
"redirect-target", string(cfg), 0, 1, &body,
|
||||||
)
|
)
|
||||||
|
|
||||||
s.Engine.ExportProcessNewTask(context.TODO(), &task)
|
s.Engine.ExportProcessNewTask(context.TODO(), &task)
|
||||||
|
|||||||
@@ -32,8 +32,7 @@ type Scheduler interface {
|
|||||||
// Engine.observeAttempt, alongside the DeliveryResult it records for
|
// Engine.observeAttempt, alongside the DeliveryResult it records for
|
||||||
// it. Deliver is also entered for attempts that never happen — an
|
// it. Deliver is also entered for attempts that never happen — an
|
||||||
// open circuit breaker refuses one — so the count cannot be taken
|
// open circuit breaker refuses one — so the count cannot be taken
|
||||||
// from around this call. An attempt cut off by a stop is neither
|
// from around this call.
|
||||||
// counted nor recorded.
|
|
||||||
type Target interface {
|
type Target interface {
|
||||||
Deliver(
|
Deliver(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
@@ -103,13 +102,16 @@ func (e *Engine) initTargets(client *http.Client) {
|
|||||||
client: client,
|
client: client,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
dbT := &databaseTarget{eng: e}
|
||||||
|
|
||||||
e.httpTarget = httpT
|
e.httpTarget = httpT
|
||||||
e.slackTarget = slackT
|
e.slackTarget = slackT
|
||||||
|
e.dbTarget = dbT
|
||||||
|
|
||||||
e.targets = map[database.TargetType]Target{
|
e.targets = map[database.TargetType]Target{
|
||||||
database.TargetTypeHTTP: httpT,
|
database.TargetTypeHTTP: httpT,
|
||||||
database.TargetTypeSlack: slackT,
|
database.TargetTypeSlack: slackT,
|
||||||
database.TargetTypeDatabase: &databaseTarget{eng: e},
|
database.TargetTypeDatabase: dbT,
|
||||||
database.TargetTypeLog: &logTarget{eng: e},
|
database.TargetTypeLog: &logTarget{eng: e},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -106,7 +105,7 @@ func databaseConfigForm(
|
|||||||
return TargetConfigForm{}, nil
|
return TargetConfigForm{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var cfg archive.DatabaseTargetConfig
|
var cfg databaseTargetConfig
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(configJSON), &cfg)
|
err := json.Unmarshal([]byte(configJSON), &cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -115,18 +114,18 @@ func databaseConfigForm(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = archive.ValidateRotation(cfg.Rotation)
|
err = ValidateArchiveRotation(cfg.Rotation)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return TargetConfigForm{}, err
|
return TargetConfigForm{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
form := TargetConfigForm{Rotation: cfg.Rotation}
|
form := TargetConfigForm{Rotation: cfg.Rotation}
|
||||||
|
|
||||||
if cfg.Expiry == "" || cfg.Expiry == archive.ExpiryNever {
|
if cfg.Expiry == "" || cfg.Expiry == archiveExpiryNever {
|
||||||
return form, nil
|
return form, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
err = archive.ValidateExpiry(cfg.Expiry)
|
err = ValidateArchiveExpiry(cfg.Expiry)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return TargetConfigForm{}, err
|
return TargetConfigForm{}, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -121,8 +120,8 @@ func unavailableConfigFields() []ConfigField {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// slackConfigFields describes a Slack target: its masked
|
// slackConfigFields describes a Slack target: its masked
|
||||||
// webhook URL and its number of delivery attempts. Only the
|
// webhook URL and its retry count. Only the masked URL is
|
||||||
// masked URL is shown; the full URL is the credential.
|
// shown; the full URL is the credential.
|
||||||
func slackConfigFields(t *database.Target) []ConfigField {
|
func slackConfigFields(t *database.Target) []ConfigField {
|
||||||
cfg, err := parseSlackConfig(t.Config)
|
cfg, err := parseSlackConfig(t.Config)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -132,7 +131,7 @@ func slackConfigFields(t *database.Target) []ConfigField {
|
|||||||
return []ConfigField{{
|
return []ConfigField{{
|
||||||
Label: "Webhook URL",
|
Label: "Webhook URL",
|
||||||
Value: cfg.MaskedWebhookURL(),
|
Value: cfg.MaskedWebhookURL(),
|
||||||
}, maxAttemptsField(t)}
|
}, maxRetriesField(t)}
|
||||||
}
|
}
|
||||||
|
|
||||||
// httpConfigFields describes an HTTP target: its destination
|
// httpConfigFields describes an HTTP target: its destination
|
||||||
@@ -179,18 +178,23 @@ func httpConfigFields(t *database.Target) []ConfigField {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fields = append(fields, maxAttemptsField(t))
|
fields = append(fields, maxRetriesField(t))
|
||||||
|
|
||||||
return fields
|
return fields
|
||||||
}
|
}
|
||||||
|
|
||||||
// maxAttemptsField describes a target's number of delivery
|
// maxRetriesField describes a target's retry count, which lives
|
||||||
// attempts, which lives on the target row rather than in its
|
// on the target row rather than in its configuration blob. A
|
||||||
// configuration blob.
|
// stored 0 makes a single attempt, so it is shown as 1.
|
||||||
func maxAttemptsField(t *database.Target) ConfigField {
|
func maxRetriesField(t *database.Target) ConfigField {
|
||||||
|
attempts := strconv.Itoa(t.MaxRetries)
|
||||||
|
if t.MaxRetries == 0 {
|
||||||
|
attempts = "1 (fire-and-forget: no retries, no circuit breaker)"
|
||||||
|
}
|
||||||
|
|
||||||
return ConfigField{
|
return ConfigField{
|
||||||
Label: "Delivery attempts",
|
Label: "Delivery attempts",
|
||||||
Value: strconv.Itoa(t.MaxAttempts),
|
Value: attempts,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -201,17 +205,17 @@ func maxAttemptsField(t *database.Target) ConfigField {
|
|||||||
// not one of the four, is reported as unavailable rather than
|
// not one of the four, is reported as unavailable rather than
|
||||||
// echoed back.
|
// echoed back.
|
||||||
func databaseConfigFields(configJSON string) []ConfigField {
|
func databaseConfigFields(configJSON string) []ConfigField {
|
||||||
expiry, err := archive.ParseExpiry(configJSON)
|
expiry, err := parseArchiveExpiry(configJSON)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return unavailableConfigFields()
|
return unavailableConfigFields()
|
||||||
}
|
}
|
||||||
|
|
||||||
rotation, err := archive.ParseRotation(configJSON)
|
rotation, err := parseArchiveRotation(configJSON)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return unavailableConfigFields()
|
return unavailableConfigFields()
|
||||||
}
|
}
|
||||||
|
|
||||||
value := archive.ExpiryNever
|
value := archiveExpiryNever
|
||||||
if expiry > 0 {
|
if expiry > 0 {
|
||||||
value = plainDuration(expiry)
|
value = plainDuration(expiry)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,18 +32,7 @@ const (
|
|||||||
viewMaskedOrigin = viewExampleOrigin + "/..."
|
viewMaskedOrigin = viewExampleOrigin + "/..."
|
||||||
viewUnavailable = "(unavailable)"
|
viewUnavailable = "(unavailable)"
|
||||||
viewExpiryNever = "never"
|
viewExpiryNever = "never"
|
||||||
viewMaxAttempts = "Delivery attempts"
|
viewMaxRetries = "Delivery attempts"
|
||||||
)
|
|
||||||
|
|
||||||
// The archive rotations a database target's settings can name, and
|
|
||||||
// the config of one whose rotation is not one of the four.
|
|
||||||
const (
|
|
||||||
rotationNone = "none"
|
|
||||||
rotationMonthly = "monthly"
|
|
||||||
rotationDaily = "daily"
|
|
||||||
rotationHourly = "hourly"
|
|
||||||
|
|
||||||
weeklyConfig = `{"rotation":"weekly"}`
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMaskedWebhookURL(t *testing.T) {
|
func TestMaskedWebhookURL(t *testing.T) {
|
||||||
@@ -182,7 +171,6 @@ func slackTarget() database.Target {
|
|||||||
Active: true,
|
Active: true,
|
||||||
Config: `{"webhookUrl":"` +
|
Config: `{"webhookUrl":"` +
|
||||||
slackWebhookURL + `"}`,
|
slackWebhookURL + `"}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -202,20 +190,19 @@ func TestNewTargetViews_Slack(t *testing.T) {
|
|||||||
t,
|
t,
|
||||||
map[string]string{
|
map[string]string{
|
||||||
"Webhook URL": slackMaskedURL,
|
"Webhook URL": slackMaskedURL,
|
||||||
viewMaxAttempts: "1",
|
viewMaxRetries: "1 (fire-and-forget: no retries, no circuit breaker)",
|
||||||
},
|
},
|
||||||
fieldMap(view.Config),
|
fieldMap(view.Config),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestNewTargetViews_SlackAttempts proves a Slack target shows
|
// TestNewTargetViews_SlackRetries proves a Slack target shows
|
||||||
// its number of delivery attempts the same way an HTTP target
|
// its retry count the same way an HTTP target does.
|
||||||
// does.
|
func TestNewTargetViews_SlackRetries(t *testing.T) {
|
||||||
func TestNewTargetViews_SlackAttempts(t *testing.T) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
target := slackTarget()
|
target := slackTarget()
|
||||||
target.MaxAttempts = 2
|
target.MaxRetries = 2
|
||||||
|
|
||||||
view := viewFor(t, target)
|
view := viewFor(t, target)
|
||||||
|
|
||||||
@@ -223,7 +210,7 @@ func TestNewTargetViews_SlackAttempts(t *testing.T) {
|
|||||||
t,
|
t,
|
||||||
map[string]string{
|
map[string]string{
|
||||||
"Webhook URL": slackMaskedURL,
|
"Webhook URL": slackMaskedURL,
|
||||||
viewMaxAttempts: "2",
|
viewMaxRetries: "2",
|
||||||
},
|
},
|
||||||
fieldMap(view.Config),
|
fieldMap(view.Config),
|
||||||
)
|
)
|
||||||
@@ -238,7 +225,7 @@ func TestNewTargetViews_HTTP(t *testing.T) {
|
|||||||
`"timeout":30,` +
|
`"timeout":30,` +
|
||||||
`"headers":{"Authorization":"Bearer sekrit"},` +
|
`"headers":{"Authorization":"Bearer sekrit"},` +
|
||||||
`"forwardQuery":true}`,
|
`"forwardQuery":true}`,
|
||||||
MaxAttempts: 5,
|
MaxRetries: 5,
|
||||||
})
|
})
|
||||||
|
|
||||||
fields := fieldMap(view.Config)
|
fields := fieldMap(view.Config)
|
||||||
@@ -250,7 +237,7 @@ func TestNewTargetViews_HTTP(t *testing.T) {
|
|||||||
"Timeout": "30s",
|
"Timeout": "30s",
|
||||||
"Headers": "1 configured",
|
"Headers": "1 configured",
|
||||||
"Query string": "passed on to this target",
|
"Query string": "passed on to this target",
|
||||||
viewMaxAttempts: "5",
|
viewMaxRetries: "5",
|
||||||
},
|
},
|
||||||
fields,
|
fields,
|
||||||
)
|
)
|
||||||
@@ -261,22 +248,19 @@ func TestNewTargetViews_HTTP(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestNewTargetViews_HTTPSingleAttempt proves a single attempt is
|
func TestNewTargetViews_HTTPFireAndForget(t *testing.T) {
|
||||||
// shown as the plain number, like any other.
|
|
||||||
func TestNewTargetViews_HTTPSingleAttempt(t *testing.T) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
view := viewFor(t, database.Target{
|
view := viewFor(t, database.Target{
|
||||||
Type: database.TargetTypeHTTP,
|
Type: database.TargetTypeHTTP,
|
||||||
Config: `{"url":"` + viewExampleHook + `"}`,
|
Config: `{"url":"` + viewExampleHook + `"}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
})
|
})
|
||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
map[string]string{
|
map[string]string{
|
||||||
"Destination URL": viewMaskedOrigin,
|
"Destination URL": viewMaskedOrigin,
|
||||||
viewMaxAttempts: "1",
|
viewMaxRetries: "1 (fire-and-forget: no retries, no circuit breaker)",
|
||||||
},
|
},
|
||||||
fieldMap(view.Config),
|
fieldMap(view.Config),
|
||||||
)
|
)
|
||||||
@@ -292,7 +276,6 @@ func TestNewTargetViews_HTTPMasksDestinationURL(t *testing.T) {
|
|||||||
view := viewFor(t, database.Target{
|
view := viewFor(t, database.Target{
|
||||||
Type: database.TargetTypeHTTP,
|
Type: database.TargetTypeHTTP,
|
||||||
Config: `{"url":"` + slackWebhookURL + `"}`,
|
Config: `{"url":"` + slackWebhookURL + `"}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
})
|
})
|
||||||
|
|
||||||
fields := fieldMap(view.Config)
|
fields := fieldMap(view.Config)
|
||||||
@@ -417,21 +400,17 @@ func TestNewTargetViews_Unpresentable(t *testing.T) {
|
|||||||
"unparseable json": {
|
"unparseable json": {
|
||||||
Type: database.TargetTypeSlack,
|
Type: database.TargetTypeSlack,
|
||||||
Config: blob,
|
Config: blob,
|
||||||
MaxAttempts: 1,
|
|
||||||
},
|
},
|
||||||
"empty slack config": {
|
"empty slack config": {
|
||||||
Type: database.TargetTypeSlack,
|
Type: database.TargetTypeSlack,
|
||||||
MaxAttempts: 1,
|
|
||||||
},
|
},
|
||||||
"slack config without url": {
|
"slack config without url": {
|
||||||
Type: database.TargetTypeSlack,
|
Type: database.TargetTypeSlack,
|
||||||
Config: `{}`,
|
Config: `{}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
},
|
},
|
||||||
"unparseable http json": {
|
"unparseable http json": {
|
||||||
Type: database.TargetTypeHTTP,
|
Type: database.TargetTypeHTTP,
|
||||||
Config: `{"url":`,
|
Config: `{"url":`,
|
||||||
MaxAttempts: 1,
|
|
||||||
},
|
},
|
||||||
"unparseable archive json": {
|
"unparseable archive json": {
|
||||||
Type: database.TargetTypeDatabase,
|
Type: database.TargetTypeDatabase,
|
||||||
|
|||||||
@@ -2,23 +2,83 @@ package delivery
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// archiveNameMaxLen is how many characters of a webhook or target
|
||||||
|
// name an archive file name keeps.
|
||||||
|
const archiveNameMaxLen = 40
|
||||||
|
|
||||||
// databaseTarget is a no-retry target that archives the full
|
// databaseTarget is a no-retry target that archives the full
|
||||||
// inbound event into the target's own archive SQLite file, separate
|
// inbound event into the target's own archive SQLite file, separate
|
||||||
// from the per-webhook event database. The event is already
|
// from the per-webhook event database. The event is already
|
||||||
// persisted in the per-webhook event DB by the time delivery runs;
|
// persisted in the per-webhook event DB by the time delivery runs;
|
||||||
// the database target additionally writes a durable long-term copy
|
// the database target additionally writes a durable long-term copy
|
||||||
// through the engine's archive.Writers, and then records a single
|
// into the file ArchiveFileName names, with a period added when the
|
||||||
|
// target rotates (see archivePeriodPath), and then records a single
|
||||||
// attempt whose outcome reflects whether the archive write
|
// attempt whose outcome reflects whether the archive write
|
||||||
// succeeded. Package archive holds everything about the files
|
// succeeded. See archiveWriter for the close/reopen, auto-recreate,
|
||||||
// themselves.
|
// and expiry semantics.
|
||||||
type databaseTarget struct {
|
type databaseTarget struct {
|
||||||
eng *Engine
|
eng *Engine
|
||||||
|
|
||||||
|
// writers holds one archive writer per database target, keyed
|
||||||
|
// by target ID.
|
||||||
|
mu sync.Mutex
|
||||||
|
writers map[string]*archiveWriter
|
||||||
|
}
|
||||||
|
|
||||||
|
// ArchiveFileName returns the file name of a database target's
|
||||||
|
// archive: archive-WEBHOOKNAME-TARGETNAME-TARGETID.db, with both
|
||||||
|
// names passed through archiveNamePart. The target ID keeps the
|
||||||
|
// name unique when two targets' names come out the same.
|
||||||
|
func ArchiveFileName(webhookName, targetName, targetID string) string {
|
||||||
|
return "archive-" + archiveNamePart(webhookName) + "-" +
|
||||||
|
archiveNamePart(targetName) + "-" + targetID + ".db"
|
||||||
|
}
|
||||||
|
|
||||||
|
// archiveNamePart makes a webhook or target name safe to put in a
|
||||||
|
// file name. It is lowercased; ASCII letters and digits are kept,
|
||||||
|
// every other run of characters becomes a single "-", and no "-" is
|
||||||
|
// left at either end. It is cut to archiveNameMaxLen characters, and
|
||||||
|
// a name with nothing left is "unnamed".
|
||||||
|
func archiveNamePart(name string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
|
||||||
|
dash := false
|
||||||
|
|
||||||
|
for _, r := range strings.ToLower(name) {
|
||||||
|
if (r < 'a' || r > 'z') && (r < '0' || r > '9') {
|
||||||
|
dash = b.Len() > 0
|
||||||
|
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if dash {
|
||||||
|
b.WriteByte('-')
|
||||||
|
|
||||||
|
dash = false
|
||||||
|
}
|
||||||
|
|
||||||
|
b.WriteRune(r)
|
||||||
|
}
|
||||||
|
|
||||||
|
part := b.String()
|
||||||
|
if len(part) > archiveNameMaxLen {
|
||||||
|
part = strings.TrimRight(part[:archiveNameMaxLen], "-")
|
||||||
|
}
|
||||||
|
|
||||||
|
if part == "" {
|
||||||
|
return "unnamed"
|
||||||
|
}
|
||||||
|
|
||||||
|
return part
|
||||||
}
|
}
|
||||||
|
|
||||||
// Deliver implements Target. It archives the event, then
|
// Deliver implements Target. It archives the event, then
|
||||||
@@ -38,7 +98,7 @@ func (t *databaseTarget) Deliver(
|
|||||||
) {
|
) {
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
|
|
||||||
err := t.eng.archives.Write(d)
|
err := t.archive(d)
|
||||||
|
|
||||||
elapsed := time.Since(start)
|
elapsed := time.Since(start)
|
||||||
|
|
||||||
@@ -85,3 +145,308 @@ func (t *databaseTarget) Deliver(
|
|||||||
database.DeliveryStatusDelivered,
|
database.DeliveryStatusDelivered,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// archive writes the full event as a row into the target's
|
||||||
|
// archive database, honouring the optional per-target expiry and
|
||||||
|
// rotation parsed from the target config JSON. With rotation, the
|
||||||
|
// event goes to the file for the period of its receive time.
|
||||||
|
func (t *databaseTarget) archive(d *database.Delivery) error {
|
||||||
|
webhookID := d.Event.WebhookID
|
||||||
|
if webhookID == "" {
|
||||||
|
return errArchiveMissingWebhookID
|
||||||
|
}
|
||||||
|
|
||||||
|
expiry, err := parseArchiveExpiry(d.Target.Config)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
rotation, err := parseArchiveRotation(d.Target.Config)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// An event whose stored row was gone before its delivery ran has
|
||||||
|
// no receive time (see Engine.hydrateEvent), and goes to the
|
||||||
|
// file for now.
|
||||||
|
receivedAt := d.Event.CreatedAt
|
||||||
|
if receivedAt.IsZero() {
|
||||||
|
receivedAt = time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
w, err := t.writerFor(d.TargetID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
row := archivedEvent{
|
||||||
|
EventID: d.Event.ID,
|
||||||
|
WebhookID: webhookID,
|
||||||
|
EntrypointID: d.Event.EntrypointID,
|
||||||
|
Method: d.Event.Method,
|
||||||
|
RawQuery: d.Event.RawQuery,
|
||||||
|
Headers: d.Event.Headers,
|
||||||
|
Body: d.Event.Body,
|
||||||
|
ContentType: d.Event.ContentType,
|
||||||
|
}
|
||||||
|
|
||||||
|
return w.write(row, expiry, archivePeriod(rotation, receivedAt))
|
||||||
|
}
|
||||||
|
|
||||||
|
// writerFor returns the archive writer for a database target,
|
||||||
|
// creating and caching it on first use. Each target has one writer
|
||||||
|
// so its close/reopen debounce state is shared across concurrent
|
||||||
|
// deliveries, and so a rename and the idle sweep take the same lock
|
||||||
|
// as its writes.
|
||||||
|
func (t *databaseTarget) writerFor(
|
||||||
|
targetID string,
|
||||||
|
) (*archiveWriter, error) {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
|
||||||
|
w, ok := t.writers[targetID]
|
||||||
|
if !ok {
|
||||||
|
var err error
|
||||||
|
|
||||||
|
w, err = t.newWriter(targetID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if t.writers == nil {
|
||||||
|
t.writers = make(map[string]*archiveWriter)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.writers[targetID] = w
|
||||||
|
}
|
||||||
|
|
||||||
|
// A delivery claims the entry: even if the idle sweep created
|
||||||
|
// it moments ago, it now belongs to the registry proper and
|
||||||
|
// the sweep must leave it in place when it finishes.
|
||||||
|
w.sweepOwned = false
|
||||||
|
|
||||||
|
return w, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// sweepWriterFor returns the archive writer the idle sweep should
|
||||||
|
// prune a target's archive through, together with whether the sweep
|
||||||
|
// itself created the registry entry.
|
||||||
|
//
|
||||||
|
// The sweep must route its prune through the registered writer so
|
||||||
|
// the writer's mutex orders it against concurrent writes, but it
|
||||||
|
// must never leave a registry entry behind: a sweep that ran
|
||||||
|
// concurrently with the target's deletion would otherwise
|
||||||
|
// re-create an entry that nothing will ever evict again, which is
|
||||||
|
// exactly the leak eviction exists to prevent. An entry the sweep
|
||||||
|
// creates is therefore marked sweep-owned and handed back to
|
||||||
|
// releaseSweepWriter when the sweep is done.
|
||||||
|
func (t *databaseTarget) sweepWriterFor(
|
||||||
|
targetID string,
|
||||||
|
) (*archiveWriter, bool, error) {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
|
||||||
|
w, ok := t.writers[targetID]
|
||||||
|
if ok {
|
||||||
|
return w, false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
w, err := t.newWriter(targetID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if t.writers == nil {
|
||||||
|
t.writers = make(map[string]*archiveWriter)
|
||||||
|
}
|
||||||
|
|
||||||
|
w.sweepOwned = true
|
||||||
|
t.writers[targetID] = w
|
||||||
|
|
||||||
|
return w, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// releaseSweepWriter drops a registry entry that the idle sweep
|
||||||
|
// created, so a sweep leaves the registry exactly as it found it.
|
||||||
|
//
|
||||||
|
// The entry is removed only if it is still the very writer the
|
||||||
|
// sweep installed and no delivery has claimed it in the meantime
|
||||||
|
// (writerFor clears sweepOwned when it hands a writer to the
|
||||||
|
// write path). Both conditions are evaluated under the registry
|
||||||
|
// lock, so an eviction that raced the sweep — which removes the
|
||||||
|
// entry outright — simply finds nothing left to do here, and a
|
||||||
|
// delivery that adopted the writer keeps a registered, evictable
|
||||||
|
// one.
|
||||||
|
func (t *databaseTarget) releaseSweepWriter(
|
||||||
|
targetID string, w *archiveWriter,
|
||||||
|
) {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
|
||||||
|
cur, ok := t.writers[targetID]
|
||||||
|
if !ok || cur != w || !cur.sweepOwned {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
delete(t.writers, targetID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// newWriter builds the writer for a database target's archive. Its
|
||||||
|
// path is the one ArchivePath gives for the webhook and the target as
|
||||||
|
// the main database names them now; from then on only rename changes
|
||||||
|
// the name the writer uses. It does not touch the archive files.
|
||||||
|
func (t *databaseTarget) newWriter(
|
||||||
|
targetID string,
|
||||||
|
) (*archiveWriter, error) {
|
||||||
|
if t.eng.dbManager == nil {
|
||||||
|
return nil, errArchiveNoDataDir
|
||||||
|
}
|
||||||
|
|
||||||
|
var target database.Target
|
||||||
|
|
||||||
|
err := t.eng.database.DB().
|
||||||
|
Preload("Webhook").
|
||||||
|
First(&target, "id = ?", targetID).Error
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf(
|
||||||
|
"loading database target %s: %w", targetID, err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
w := newArchiveWriter(
|
||||||
|
ArchivePath(t.eng.dbManager, &target.Webhook, &target),
|
||||||
|
t.eng.log,
|
||||||
|
)
|
||||||
|
w.webhookID = target.WebhookID
|
||||||
|
|
||||||
|
return w, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// rename moves every one of a database target's archive files to the
|
||||||
|
// name for webhookName and targetName. It goes through the target's
|
||||||
|
// writer, so the move holds the lock that writes and the idle sweep
|
||||||
|
// take, and later writes use the new name.
|
||||||
|
//
|
||||||
|
// The writer is created if there is none, and it stays cached. The
|
||||||
|
// handlers rename before they save the new name, so until the save
|
||||||
|
// the main database still has the old one; a delivery in that window
|
||||||
|
// must find this writer rather than build one from the old name.
|
||||||
|
func (t *databaseTarget) rename(
|
||||||
|
targetID, webhookName, targetName string,
|
||||||
|
) error {
|
||||||
|
w, err := t.writerFor(targetID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return w.rename(ArchiveFileName(webhookName, targetName, targetID))
|
||||||
|
}
|
||||||
|
|
||||||
|
// evict drops a database target's archive writer from the registry
|
||||||
|
// and closes its handle, so a deleted target does not leave a
|
||||||
|
// writer (and an open archive handle within its debounce window)
|
||||||
|
// alive for the process lifetime.
|
||||||
|
//
|
||||||
|
// The map entry is removed under the registry lock, which is
|
||||||
|
// then released before the handle is closed under the writer's
|
||||||
|
// own lock: that ordering keeps the registry available to other
|
||||||
|
// targets while an in-flight write on this one drains, and
|
||||||
|
// closing under the writer's lock means eviction can never race
|
||||||
|
// a write.
|
||||||
|
//
|
||||||
|
// Eviction is idempotent and silent for a target with no writer,
|
||||||
|
// which is the common case: only a database target that has
|
||||||
|
// received an event or been renamed has one. It never deletes the
|
||||||
|
// archive file.
|
||||||
|
func (t *databaseTarget) evict(targetID string) {
|
||||||
|
t.mu.Lock()
|
||||||
|
|
||||||
|
w, ok := t.writers[targetID]
|
||||||
|
if ok {
|
||||||
|
delete(t.writers, targetID)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.mu.Unlock()
|
||||||
|
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
w.evict()
|
||||||
|
|
||||||
|
t.eng.log.Info(
|
||||||
|
"evicted archive writer",
|
||||||
|
"target_id", targetID,
|
||||||
|
"path", w.path,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// evictWebhook evicts, exactly as evict does, the writer of every
|
||||||
|
// database target of a webhook.
|
||||||
|
func (t *databaseTarget) evictWebhook(webhookID string) {
|
||||||
|
t.mu.Lock()
|
||||||
|
|
||||||
|
var gone []*archiveWriter
|
||||||
|
|
||||||
|
for targetID, w := range t.writers {
|
||||||
|
if w.webhookID == webhookID {
|
||||||
|
delete(t.writers, targetID)
|
||||||
|
|
||||||
|
gone = append(gone, w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
t.mu.Unlock()
|
||||||
|
|
||||||
|
for _, w := range gone {
|
||||||
|
w.evict()
|
||||||
|
|
||||||
|
t.eng.log.Info(
|
||||||
|
"evicted archive writer",
|
||||||
|
"webhook_id", webhookID,
|
||||||
|
"path", w.path,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// evictAll evicts every cached archive writer, exactly as evict
|
||||||
|
// does for one target. The engine calls it at shutdown, once its
|
||||||
|
// workers have returned. Closing the last handle on an archive
|
||||||
|
// moves the contents of its -wal into the .db and removes the
|
||||||
|
// -wal, so a clean stop leaves each archive as a single file.
|
||||||
|
func (t *databaseTarget) evictAll() {
|
||||||
|
t.mu.Lock()
|
||||||
|
|
||||||
|
writers := t.writers
|
||||||
|
t.writers = nil
|
||||||
|
|
||||||
|
t.mu.Unlock()
|
||||||
|
|
||||||
|
for _, w := range writers {
|
||||||
|
w.evict()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sweepArchive prunes one database target's archive of rows older
|
||||||
|
// than expiry, without requiring a write. A missing archive file is
|
||||||
|
// left missing (see sweepExpired), so a sweep never creates an
|
||||||
|
// archive for a target that has never received an event.
|
||||||
|
//
|
||||||
|
// It also never leaves a registry entry behind: an entry it had
|
||||||
|
// to create to reach the writer's mutex is released again once
|
||||||
|
// the prune is done, so a sweep racing a target deletion cannot
|
||||||
|
// resurrect the writer the eviction just dropped.
|
||||||
|
func (t *databaseTarget) sweepArchive(
|
||||||
|
targetID string, expiry time.Duration,
|
||||||
|
) error {
|
||||||
|
w, created, err := t.sweepWriterFor(targetID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if created {
|
||||||
|
defer t.releaseSweepWriter(targetID, w)
|
||||||
|
}
|
||||||
|
|
||||||
|
return w.sweepExpired(expiry)
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
package archive
|
package delivery
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
@@ -18,9 +18,9 @@ import (
|
|||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ExpiryNever is the expiry sentinel (and default) that
|
// archiveExpiryNever is the expiry sentinel (and default) that
|
||||||
// disables pruning so archived rows are kept forever.
|
// disables pruning so archived rows are kept forever.
|
||||||
const ExpiryNever = "never"
|
const archiveExpiryNever = "never"
|
||||||
|
|
||||||
// archiveReopenDebounce bounds how often an archive file is
|
// archiveReopenDebounce bounds how often an archive file is
|
||||||
// closed and reopened. After each write the handle is closed
|
// closed and reopened. After each write the handle is closed
|
||||||
@@ -71,17 +71,17 @@ var (
|
|||||||
"archive writer has been evicted",
|
"archive writer has been evicted",
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrNameTaken is returned when an archive cannot be
|
// ErrArchiveNameTaken is returned when an archive cannot be
|
||||||
// renamed because a file already has the new name. That file may
|
// renamed because a file already has the new name. That file may
|
||||||
// be an archive with rows of its own, so it is never replaced.
|
// be an archive with rows of its own, so it is never replaced.
|
||||||
ErrNameTaken = errors.New(
|
ErrArchiveNameTaken = errors.New(
|
||||||
"a file already has the archive's new name",
|
"a file already has the archive's new name",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
// DatabaseTargetConfig is the optional per-target JSON config
|
// databaseTargetConfig is the optional per-target JSON config
|
||||||
// for a database (archive) target.
|
// for a database (archive) target.
|
||||||
type DatabaseTargetConfig struct {
|
type databaseTargetConfig struct {
|
||||||
// Expiry is a Go duration (e.g. "720h") after which
|
// Expiry is a Go duration (e.g. "720h") after which
|
||||||
// archived rows are pruned, or "never" (the default) to
|
// archived rows are pruned, or "never" (the default) to
|
||||||
// keep them forever.
|
// keep them forever.
|
||||||
@@ -112,21 +112,21 @@ type archivedEvent struct {
|
|||||||
ArchivedAt time.Time `gorm:"index"`
|
ArchivedAt time.Time `gorm:"index"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseExpiry reads the optional expiry from a database
|
// parseArchiveExpiry reads the optional expiry from a database
|
||||||
// target's config JSON. An empty config, an empty expiry, or
|
// target's config JSON. An empty config, an empty expiry, or
|
||||||
// the literal "never" all mean keep forever, returned as a zero
|
// the literal "never" all mean keep forever, returned as a zero
|
||||||
// duration. Any other value must parse as a positive Go
|
// duration. Any other value must parse as a positive Go
|
||||||
// duration; a set-but-invalid value (unparseable, zero, or
|
// duration; a set-but-invalid value (unparseable, zero, or
|
||||||
// negative) is an error rather than a silent default, matching
|
// negative) is an error rather than a silent default, matching
|
||||||
// ValidateExpiry at target creation.
|
// ValidateArchiveExpiry at target creation.
|
||||||
func ParseExpiry(
|
func parseArchiveExpiry(
|
||||||
configJSON string,
|
configJSON string,
|
||||||
) (time.Duration, error) {
|
) (time.Duration, error) {
|
||||||
if configJSON == "" {
|
if configJSON == "" {
|
||||||
return 0, nil
|
return 0, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var cfg DatabaseTargetConfig
|
var cfg databaseTargetConfig
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(configJSON), &cfg)
|
err := json.Unmarshal([]byte(configJSON), &cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -135,7 +135,7 @@ func ParseExpiry(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if cfg.Expiry == "" || cfg.Expiry == ExpiryNever {
|
if cfg.Expiry == "" || cfg.Expiry == archiveExpiryNever {
|
||||||
return 0, nil
|
return 0, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -155,14 +155,14 @@ func ParseExpiry(
|
|||||||
return dur, nil
|
return dur, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidateExpiry checks a user-supplied archive expiry
|
// ValidateArchiveExpiry checks a user-supplied archive expiry
|
||||||
// for a database target at configuration time. Valid values are
|
// for a database target at configuration time. Valid values are
|
||||||
// empty, "never" (both meaning keep forever), or a positive Go
|
// empty, "never" (both meaning keep forever), or a positive Go
|
||||||
// duration such as "720h". Anything else is an error, so a bad
|
// duration such as "720h". Anything else is an error, so a bad
|
||||||
// expiry is rejected when the target is created rather than
|
// expiry is rejected when the target is created rather than
|
||||||
// failing every subsequent delivery.
|
// failing every subsequent delivery.
|
||||||
func ValidateExpiry(expiry string) error {
|
func ValidateArchiveExpiry(expiry string) error {
|
||||||
if expiry == "" || expiry == ExpiryNever {
|
if expiry == "" || expiry == archiveExpiryNever {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -171,7 +171,7 @@ func ValidateExpiry(expiry string) error {
|
|||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"expiry must be %q or a Go duration "+
|
"expiry must be %q or a Go duration "+
|
||||||
"such as \"720h\": %w",
|
"such as \"720h\": %w",
|
||||||
ExpiryNever, err,
|
archiveExpiryNever, err,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -194,7 +194,7 @@ func ValidateExpiry(expiry string) error {
|
|||||||
type archiveWriter struct {
|
type archiveWriter struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
|
|
||||||
// path is the target's archive file as Path names it. A
|
// path is the target's archive file as ArchivePath names it. A
|
||||||
// target that rotates writes to the files archivePeriodPath names
|
// target that rotates writes to the files archivePeriodPath names
|
||||||
// for path and a period instead.
|
// for path and a period instead.
|
||||||
path string
|
path string
|
||||||
@@ -232,7 +232,7 @@ type archiveWriter struct {
|
|||||||
// registry proper.
|
// registry proper.
|
||||||
//
|
//
|
||||||
// Unlike every other field here it is guarded by
|
// Unlike every other field here it is guarded by
|
||||||
// Writers.mu, not by this writer's mu: it describes the
|
// databaseTarget.mu, not by this writer's mu: it describes the
|
||||||
// registry entry rather than the file.
|
// registry entry rather than the file.
|
||||||
sweepOwned bool
|
sweepOwned bool
|
||||||
}
|
}
|
||||||
@@ -499,7 +499,7 @@ func (w *archiveWriter) sweepFile(
|
|||||||
// the next write creates its file under the new name.
|
// the next write creates its file under the new name.
|
||||||
//
|
//
|
||||||
// If a file already has one of the new names, nothing is moved and
|
// If a file already has one of the new names, nothing is moved and
|
||||||
// the error is ErrNameTaken. If one file fails to move, those
|
// the error is ErrArchiveNameTaken. If one file fails to move, those
|
||||||
// already moved are moved back before the error is returned, so the
|
// already moved are moved back before the error is returned, so the
|
||||||
// archive is never split across two names.
|
// archive is never split across two names.
|
||||||
func (w *archiveWriter) rename(name string) error {
|
func (w *archiveWriter) rename(name string) error {
|
||||||
@@ -537,7 +537,7 @@ func (w *archiveWriter) rename(name string) error {
|
|||||||
for _, taken := range to {
|
for _, taken := range to {
|
||||||
if fileExists(taken) {
|
if fileExists(taken) {
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"%w: %s", ErrNameTaken, filepath.Base(taken),
|
"%w: %s", ErrArchiveNameTaken, filepath.Base(taken),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -590,7 +590,7 @@ func (w *archiveWriter) evict() {
|
|||||||
// each row's archived time. It runs on every (re)open, so a
|
// each row's archived time. It runs on every (re)open, so a
|
||||||
// steadily written archive is swept by its own write traffic. An
|
// steadily written archive is swept by its own write traffic. An
|
||||||
// archive that goes idle receives no further reopens, which is
|
// archive that goes idle receives no further reopens, which is
|
||||||
// why Sweeper exists to drive sweepExpired on a timer.
|
// why ArchiveSweeper exists to drive sweepExpired on a timer.
|
||||||
// Failures are logged, not fatal: a prune error must not stop
|
// Failures are logged, not fatal: a prune error must not stop
|
||||||
// archiving.
|
// archiving.
|
||||||
func (w *archiveWriter) prune(expiry time.Duration) {
|
func (w *archiveWriter) prune(expiry time.Duration) {
|
||||||
@@ -617,9 +617,9 @@ func (w *archiveWriter) prune(expiry time.Duration) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// FileInfo is what the metadata of a database target's archive
|
// ArchiveFileInfo is what the metadata of a database target's archive
|
||||||
// files says about them.
|
// files says about them.
|
||||||
type FileInfo struct {
|
type ArchiveFileInfo struct {
|
||||||
// Files counts the files.
|
// Files counts the files.
|
||||||
Files int
|
Files int
|
||||||
|
|
||||||
@@ -631,18 +631,18 @@ type FileInfo struct {
|
|||||||
Written time.Time
|
Written time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stat reads the metadata of a database target's archive
|
// StatArchive reads the metadata of a database target's archive
|
||||||
// files, given the path Path gives it (see archiveFiles), and
|
// files, given the path ArchivePath gives it (see archiveFiles), and
|
||||||
// of their -wal, without opening them. With no files, which is so
|
// of their -wal, without opening them. With no files, which is so
|
||||||
// before the first write and after the operator moved them away, the
|
// before the first write and after the operator moved them away, the
|
||||||
// error wraps fs.ErrNotExist.
|
// error wraps fs.ErrNotExist.
|
||||||
func Stat(path string) (FileInfo, error) {
|
func StatArchive(path string) (ArchiveFileInfo, error) {
|
||||||
files, err := archiveFiles(path)
|
files, err := archiveFiles(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return FileInfo{}, err
|
return ArchiveFileInfo{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var info FileInfo
|
var info ArchiveFileInfo
|
||||||
|
|
||||||
for _, file := range files {
|
for _, file := range files {
|
||||||
db, err := os.Stat(file.path)
|
db, err := os.Stat(file.path)
|
||||||
@@ -651,7 +651,7 @@ func Stat(path string) (FileInfo, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return FileInfo{}, err
|
return ArchiveFileInfo{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
info.Files++
|
info.Files++
|
||||||
@@ -667,7 +667,7 @@ func Stat(path string) (FileInfo, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return FileInfo{}, err
|
return ArchiveFileInfo{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
info.Size += wal.Size()
|
info.Size += wal.Size()
|
||||||
@@ -678,7 +678,7 @@ func Stat(path string) (FileInfo, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if info.Files == 0 {
|
if info.Files == 0 {
|
||||||
return FileInfo{}, fmt.Errorf(
|
return ArchiveFileInfo{}, fmt.Errorf(
|
||||||
"no archive file for %s: %w", path, fs.ErrNotExist,
|
"no archive file for %s: %w", path, fs.ErrNotExist,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
+6
-6
@@ -1,4 +1,4 @@
|
|||||||
package archive_test
|
package delivery_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
@@ -14,7 +14,7 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
gormlogger "gorm.io/gorm/logger"
|
gormlogger "gorm.io/gorm/logger"
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
"sneak.berlin/go/webhooker/internal/middleware"
|
"sneak.berlin/go/webhooker/internal/middleware"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -57,7 +57,7 @@ func (g *gormDefaultBuf) String() string {
|
|||||||
// without exporting production code to carry it, and a logging
|
// without exporting production code to carry it, and a logging
|
||||||
// detector is not worth a production symbol. What it detects is the
|
// detector is not worth a production symbol. What it detects is the
|
||||||
// third gorm.Open in this service, at
|
// third gorm.Open in this service, at
|
||||||
// internal/archive/writer.go — the archive writer,
|
// internal/delivery/target_database_archive.go — the archive writer,
|
||||||
// whose type is unexported, so nothing outside this package can drive
|
// whose type is unexported, so nothing outside this package can drive
|
||||||
// it.
|
// it.
|
||||||
func captureArchiveGORMDefault(t *testing.T) *gormDefaultBuf {
|
func captureArchiveGORMDefault(t *testing.T) *gormDefaultBuf {
|
||||||
@@ -85,7 +85,7 @@ func captureArchiveGORMDefault(t *testing.T) *gormDefaultBuf {
|
|||||||
// writer's gorm.Open to the adapter.
|
// writer's gorm.Open to the adapter.
|
||||||
//
|
//
|
||||||
// Restore a bare &gorm.Config{} at
|
// Restore a bare &gorm.Config{} at
|
||||||
// internal/archive/writer.go and this fails: the
|
// internal/delivery/target_database_archive.go and this fails: the
|
||||||
// default logger prints the fully interpolated SELECT on every
|
// default logger prints the fully interpolated SELECT on every
|
||||||
// ErrRecordNotFound, so the client-chosen event id below arrives whole
|
// ErrRecordNotFound, so the client-chosen event id below arrives whole
|
||||||
// and unbounded on stdout, answering to no level the operator set.
|
// and unbounded on stdout, answering to no level the operator set.
|
||||||
@@ -100,7 +100,7 @@ func TestArchiveWriter_NeverUsesGORMsDefaultLogger(t *testing.T) {
|
|||||||
|
|
||||||
gormDefault := captureArchiveGORMDefault(t)
|
gormDefault := captureArchiveGORMDefault(t)
|
||||||
|
|
||||||
w := archive.NewExportArchiveWriter(
|
w := delivery.NewExportArchiveWriter(
|
||||||
filepath.Join(t.TempDir(), "archive.db"),
|
filepath.Join(t.TempDir(), "archive.db"),
|
||||||
slog.New(slog.NewTextHandler(
|
slog.New(slog.NewTextHandler(
|
||||||
&captured, &slog.HandlerOptions{Level: slog.LevelDebug},
|
&captured, &slog.HandlerOptions{Level: slog.LevelDebug},
|
||||||
@@ -118,7 +118,7 @@ func TestArchiveWriter_NeverUsesGORMsDefaultLogger(t *testing.T) {
|
|||||||
value := strings.Repeat("\x01", archiveGORMFillBytes) +
|
value := strings.Repeat("\x01", archiveGORMFillBytes) +
|
||||||
archiveGORMTailMarker
|
archiveGORMTailMarker
|
||||||
|
|
||||||
var row archive.ExportArchivedEvent
|
var row delivery.ExportArchivedEvent
|
||||||
|
|
||||||
err := w.DB().Where("event_id = ?", value).First(&row).Error
|
err := w.DB().Where("event_id = ?", value).First(&row).Error
|
||||||
require.ErrorIs(t, err, gorm.ErrRecordNotFound)
|
require.ErrorIs(t, err, gorm.ErrRecordNotFound)
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
package archive_test
|
package delivery_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -12,8 +13,8 @@ import (
|
|||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// deliverTo archives one event to a database target, leaving the
|
// deliverTo archives one event to a database target, leaving the
|
||||||
@@ -23,9 +24,12 @@ func deliverTo(
|
|||||||
) {
|
) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
event := newEvent(`{"archived":true}`)
|
webhookDB := testWebhookDB(t)
|
||||||
|
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
||||||
|
|
||||||
require.NoError(t, env.writers.Write(databaseTargetDelivery(event, tgt)))
|
env.eng.ExportDeliverDatabase(
|
||||||
|
webhookDB, seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestEvictWebhook_ClosesAndRemovesWriter proves that evicting
|
// TestEvictWebhook_ClosesAndRemovesWriter proves that evicting
|
||||||
@@ -45,20 +49,20 @@ func TestEvictWebhook_ClosesAndRemovesWriter(t *testing.T) {
|
|||||||
deliverTo(t, env, tgt)
|
deliverTo(t, env, tgt)
|
||||||
|
|
||||||
require.True(
|
require.True(
|
||||||
t, env.writers.ExportArchiveHandleOpen(tgt.ID),
|
t, env.eng.ExportArchiveHandleOpen(tgt.ID),
|
||||||
"the writer should hold an open handle after a write",
|
"the writer should hold an open handle after a write",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
env.writers.EvictWebhook(first.WebhookID)
|
env.eng.EvictWebhook(first.WebhookID)
|
||||||
|
|
||||||
for _, tgt := range []*database.Target{first, second} {
|
for _, tgt := range []*database.Target{first, second} {
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"eviction should remove the registry entry",
|
"eviction should remove the registry entry",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.writers.ExportArchiveHandleOpen(tgt.ID),
|
t, env.eng.ExportArchiveHandleOpen(tgt.ID),
|
||||||
"eviction should close the archive handle",
|
"eviction should close the archive handle",
|
||||||
)
|
)
|
||||||
assert.FileExists(
|
assert.FileExists(
|
||||||
@@ -68,7 +72,7 @@ func TestEvictWebhook_ClosesAndRemovesWriter(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
assert.True(
|
assert.True(
|
||||||
t, env.writers.ExportArchiveHandleOpen(other.ID),
|
t, env.eng.ExportArchiveHandleOpen(other.ID),
|
||||||
"another webhook's writer must be left alone",
|
"another webhook's writer must be left alone",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -86,15 +90,15 @@ func TestEvictTarget_LeavesOtherTargets(t *testing.T) {
|
|||||||
deliverTo(t, env, doomed)
|
deliverTo(t, env, doomed)
|
||||||
deliverTo(t, env, kept)
|
deliverTo(t, env, kept)
|
||||||
|
|
||||||
env.writers.EvictTarget(doomed.ID)
|
env.eng.EvictTarget(doomed.ID)
|
||||||
|
|
||||||
assert.False(t, env.writers.ExportHasArchiveWriter(doomed.ID))
|
assert.False(t, env.eng.ExportHasArchiveWriter(doomed.ID))
|
||||||
assert.FileExists(
|
assert.FileExists(
|
||||||
t, env.archivePath(doomed),
|
t, env.archivePath(doomed),
|
||||||
"eviction must not delete the archive file",
|
"eviction must not delete the archive file",
|
||||||
)
|
)
|
||||||
assert.True(
|
assert.True(
|
||||||
t, env.writers.ExportArchiveHandleOpen(kept.ID),
|
t, env.eng.ExportArchiveHandleOpen(kept.ID),
|
||||||
"the other target's writer must be left alone",
|
"the other target's writer must be left alone",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -108,21 +112,21 @@ func TestEvictWebhook_UnknownWebhookIsNoOp(t *testing.T) {
|
|||||||
env := setupArchiveTest(t)
|
env := setupArchiveTest(t)
|
||||||
|
|
||||||
assert.NotPanics(t, func() {
|
assert.NotPanics(t, func() {
|
||||||
env.writers.EvictWebhook("no-such-webhook")
|
env.eng.EvictWebhook("no-such-webhook")
|
||||||
env.writers.EvictWebhook("no-such-webhook")
|
env.eng.EvictWebhook("no-such-webhook")
|
||||||
env.writers.EvictTarget("no-such-target")
|
env.eng.EvictTarget("no-such-target")
|
||||||
env.writers.EvictTarget("no-such-target")
|
env.eng.EvictTarget("no-such-target")
|
||||||
})
|
})
|
||||||
|
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.writers.ExportHasArchiveWriter("no-such-target"),
|
t, env.eng.ExportHasArchiveWriter("no-such-target"),
|
||||||
"eviction must not create a writer",
|
"eviction must not create a writer",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// evictTestRow builds an archive row for the eviction tests.
|
// evictTestRow builds an archive row for the eviction tests.
|
||||||
func evictTestRow(eventID string) archive.ExportArchivedEvent {
|
func evictTestRow(eventID string) delivery.ExportArchivedEvent {
|
||||||
return archive.ExportArchivedEvent{
|
return delivery.ExportArchivedEvent{
|
||||||
EventID: eventID,
|
EventID: eventID,
|
||||||
WebhookID: "wh-evict",
|
WebhookID: "wh-evict",
|
||||||
Method: http.MethodPost,
|
Method: http.MethodPost,
|
||||||
@@ -143,7 +147,7 @@ func TestEvictedWriter_WriteDoesNotReopenFile(t *testing.T) {
|
|||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive-evicted.db")
|
path := filepath.Join(t.TempDir(), "archive-evicted.db")
|
||||||
|
|
||||||
w := archive.NewExportArchiveWriter(
|
w := delivery.NewExportArchiveWriter(
|
||||||
path, archiveTestLogger(), 0,
|
path, archiveTestLogger(), 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -160,7 +164,7 @@ func TestEvictedWriter_WriteDoesNotReopenFile(t *testing.T) {
|
|||||||
err := w.Write(evictTestRow("ev-2"), 0)
|
err := w.Write(evictTestRow("ev-2"), 0)
|
||||||
|
|
||||||
require.ErrorIs(
|
require.ErrorIs(
|
||||||
t, err, archive.ErrExportArchiveWriterEvicted,
|
t, err, delivery.ErrExportArchiveWriterEvicted,
|
||||||
"an evicted writer must refuse writes",
|
"an evicted writer must refuse writes",
|
||||||
)
|
)
|
||||||
assert.NoFileExists(
|
assert.NoFileExists(
|
||||||
@@ -183,7 +187,7 @@ func TestEvictedWriter_SweepDoesNotReopenFile(t *testing.T) {
|
|||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive-evicted.db")
|
path := filepath.Join(t.TempDir(), "archive-evicted.db")
|
||||||
|
|
||||||
w := archive.NewExportArchiveWriter(
|
w := delivery.NewExportArchiveWriter(
|
||||||
path, archiveTestLogger(), 0,
|
path, archiveTestLogger(), 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -195,7 +199,7 @@ func TestEvictedWriter_SweepDoesNotReopenFile(t *testing.T) {
|
|||||||
err := w.SweepExpired(time.Hour)
|
err := w.SweepExpired(time.Hour)
|
||||||
|
|
||||||
require.ErrorIs(
|
require.ErrorIs(
|
||||||
t, err, archive.ErrExportArchiveWriterEvicted,
|
t, err, delivery.ErrExportArchiveWriterEvicted,
|
||||||
"an evicted writer must refuse an idle sweep",
|
"an evicted writer must refuse an idle sweep",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
@@ -226,7 +230,7 @@ const racingWriteGoroutines = 4
|
|||||||
// against a hang, not a timing assumption: the first write after
|
// against a hang, not a timing assumption: the first write after
|
||||||
// the eviction is refused.
|
// the eviction is refused.
|
||||||
func startRacingWrites(
|
func startRacingWrites(
|
||||||
w *archive.ExportArchiveWriter,
|
w *delivery.ExportArchiveWriter,
|
||||||
) *racingWrites {
|
) *racingWrites {
|
||||||
r := &racingWrites{
|
r := &racingWrites{
|
||||||
started: make(chan struct{}, racingWriteGoroutines),
|
started: make(chan struct{}, racingWriteGoroutines),
|
||||||
@@ -272,7 +276,7 @@ func (r *racingWrites) record(err error) {
|
|||||||
r.mu.Lock()
|
r.mu.Lock()
|
||||||
defer r.mu.Unlock()
|
defer r.mu.Unlock()
|
||||||
|
|
||||||
if errors.Is(err, archive.ErrExportArchiveWriterEvicted) {
|
if errors.Is(err, delivery.ErrExportArchiveWriterEvicted) {
|
||||||
r.sawEvicted = true
|
r.sawEvicted = true
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -316,7 +320,7 @@ func TestEvictWebhook_RacingWriteDoesNotReopenHandle(
|
|||||||
// eviction is about to detach.
|
// eviction is about to detach.
|
||||||
deliverTo(t, env, tgt)
|
deliverTo(t, env, tgt)
|
||||||
|
|
||||||
w := env.writers.ExportArchiveWriterFor(tgt.ID)
|
w := env.eng.ExportArchiveWriterFor(tgt.ID)
|
||||||
require.NotNil(t, w)
|
require.NotNil(t, w)
|
||||||
require.True(t, w.HandleOpen())
|
require.True(t, w.HandleOpen())
|
||||||
|
|
||||||
@@ -326,7 +330,7 @@ func TestEvictWebhook_RacingWriteDoesNotReopenHandle(
|
|||||||
// eviction has to contend for the writer's mutex.
|
// eviction has to contend for the writer's mutex.
|
||||||
race.awaitFirstWrite()
|
race.awaitFirstWrite()
|
||||||
|
|
||||||
env.writers.EvictWebhook(tgt.WebhookID)
|
env.eng.EvictWebhook(tgt.WebhookID)
|
||||||
|
|
||||||
sawEvicted, otherErr := race.wait()
|
sawEvicted, otherErr := race.wait()
|
||||||
|
|
||||||
@@ -341,7 +345,7 @@ func TestEvictWebhook_RacingWriteDoesNotReopenHandle(
|
|||||||
"been evicted",
|
"been evicted",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"the registry entry must stay gone",
|
"the registry entry must stay gone",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -358,25 +362,25 @@ func TestEvictWebhook_LaterDeliveryRecreatesWriter(t *testing.T) {
|
|||||||
tgt := env.seedDatabaseTarget(t, "")
|
tgt := env.seedDatabaseTarget(t, "")
|
||||||
|
|
||||||
deliverTo(t, env, tgt)
|
deliverTo(t, env, tgt)
|
||||||
require.True(t, env.writers.ExportHasArchiveWriter(tgt.ID))
|
require.True(t, env.eng.ExportHasArchiveWriter(tgt.ID))
|
||||||
|
|
||||||
env.writers.EvictWebhook(tgt.WebhookID)
|
env.eng.EvictWebhook(tgt.WebhookID)
|
||||||
|
|
||||||
// A fresh delivery for the same target gets a brand new
|
// A fresh delivery for the same target gets a brand new
|
||||||
// writer from the registry, so archiving keeps working.
|
// writer from the registry, so archiving keeps working.
|
||||||
deliverTo(t, env, tgt)
|
deliverTo(t, env, tgt)
|
||||||
|
|
||||||
assert.True(
|
assert.True(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"a later delivery should recreate the writer",
|
"a later delivery should recreate the writer",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestEvictAll_WriteAfterIsRefused proves that EvictAll, which the
|
// TestEngineStop_WriteAfterStopIsRefused proves the engine's stop
|
||||||
// delivery engine calls when it stops, closes each archive writer the
|
// closes each archive writer the way deleting its webhook does: a
|
||||||
// way deleting its webhook does: a write that reaches a writer after
|
// write that reaches a writer after the stop is refused, reopens
|
||||||
// it is refused, reopens nothing and adds no row.
|
// nothing and adds no row.
|
||||||
func TestEvictAll_WriteAfterIsRefused(t *testing.T) {
|
func TestEngineStop_WriteAfterStopIsRefused(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupArchiveTest(t)
|
||||||
@@ -384,16 +388,16 @@ func TestEvictAll_WriteAfterIsRefused(t *testing.T) {
|
|||||||
|
|
||||||
deliverTo(t, env, tgt)
|
deliverTo(t, env, tgt)
|
||||||
|
|
||||||
w := env.writers.ExportArchiveWriterFor(tgt.ID)
|
w := env.eng.ExportArchiveWriterFor(tgt.ID)
|
||||||
require.NotNil(t, w)
|
require.NotNil(t, w)
|
||||||
require.True(t, w.HandleOpen())
|
require.True(t, w.HandleOpen())
|
||||||
|
|
||||||
env.writers.EvictAll()
|
require.NoError(t, env.eng.ExportStop(context.Background()))
|
||||||
|
|
||||||
err := w.Write(evictTestRow("ev-after-stop"), 0)
|
err := w.Write(evictTestRow("ev-after-stop"), 0)
|
||||||
|
|
||||||
require.ErrorIs(
|
require.ErrorIs(
|
||||||
t, err, archive.ErrExportArchiveWriterEvicted,
|
t, err, delivery.ErrExportArchiveWriterEvicted,
|
||||||
"a write after the stop must be refused",
|
"a write after the stop must be refused",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
@@ -401,7 +405,7 @@ func TestEvictAll_WriteAfterIsRefused(t *testing.T) {
|
|||||||
"a refused write must not reopen the archive",
|
"a refused write must not reopen the archive",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.writers.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||||
"the stop should empty the registry",
|
"the stop should empty the registry",
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package archive
|
package delivery
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"compress/gzip"
|
"compress/gzip"
|
||||||
@@ -28,25 +28,25 @@ import (
|
|||||||
const archiveTableQuery = "SELECT count(*) FROM sqlite_master " +
|
const archiveTableQuery = "SELECT count(*) FROM sqlite_master " +
|
||||||
"WHERE type = 'table' AND name = 'archived_events'"
|
"WHERE type = 'table' AND name = 'archived_events'"
|
||||||
|
|
||||||
// Path returns where a database target's archive file is: in
|
// ArchivePath returns where a database target's archive file is: in
|
||||||
// the data directory, beside the webhook's event database, under the
|
// the data directory, beside the webhook's event database, under the
|
||||||
// name FileName gives it.
|
// name ArchiveFileName gives it.
|
||||||
func Path(
|
func ArchivePath(
|
||||||
dbMgr *database.WebhookDBManager,
|
dbMgr *database.WebhookDBManager,
|
||||||
webhook *database.Webhook,
|
webhook *database.Webhook,
|
||||||
target *database.Target,
|
target *database.Target,
|
||||||
) string {
|
) string {
|
||||||
return filepath.Join(
|
return filepath.Join(
|
||||||
filepath.Dir(dbMgr.DBPath(webhook.ID)),
|
filepath.Dir(dbMgr.DBPath(webhook.ID)),
|
||||||
FileName(webhook.Name, target.Name, target.ID),
|
ArchiveFileName(webhook.Name, target.Name, target.ID),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportFileName returns the name a database target's archive
|
// ArchiveExportFileName returns the name a database target's archive
|
||||||
// downloads under:
|
// downloads under:
|
||||||
// archive-WEBHOOKNAME-TARGETNAME-YYYYMMDDTHHMMSSZ.json.gz, the names
|
// archive-WEBHOOKNAME-TARGETNAME-YYYYMMDDTHHMMSSZ.json.gz, the names
|
||||||
// made safe as in FileName and the time in UTC.
|
// made safe as in ArchiveFileName and the time in UTC.
|
||||||
func ExportFileName(
|
func ArchiveExportFileName(
|
||||||
webhookName, targetName string, at time.Time,
|
webhookName, targetName string, at time.Time,
|
||||||
) string {
|
) string {
|
||||||
return "archive-" + archiveNamePart(webhookName) + "-" +
|
return "archive-" + archiveNamePart(webhookName) + "-" +
|
||||||
@@ -54,7 +54,7 @@ func ExportFileName(
|
|||||||
at.UTC().Format("20060102T150405Z") + ".json.gz"
|
at.UTC().Format("20060102T150405Z") + ".json.gz"
|
||||||
}
|
}
|
||||||
|
|
||||||
// Export is a database target's archive listed for download. It
|
// ArchiveExport is a database target's archive listed for download. It
|
||||||
// opens one of the target's files at a time, only when its rows are
|
// opens one of the target's files at a time, only when its rows are
|
||||||
// about to be written out, and closes it before it opens the next, so
|
// about to be written out, and closes it before it opens the next, so
|
||||||
// an export holds at most one file open however many the target has.
|
// an export holds at most one file open however many the target has.
|
||||||
@@ -66,7 +66,7 @@ func ExportFileName(
|
|||||||
// is open, and the export does not see them. SQLite cannot checkpoint
|
// is open, and the export does not see them. SQLite cannot checkpoint
|
||||||
// a -wal past an open snapshot, so the open file's -wal grows until the
|
// a -wal past an open snapshot, so the open file's -wal grows until the
|
||||||
// export has written that file out.
|
// export has written that file out.
|
||||||
type Export struct {
|
type ArchiveExport struct {
|
||||||
// periods are the periods of the target's files when the export
|
// periods are the periods of the target's files when the export
|
||||||
// was listed, "" for the file without one, in the order
|
// was listed, "" for the file without one, in the order
|
||||||
// archiveFiles lists them.
|
// archiveFiles lists them.
|
||||||
@@ -77,7 +77,7 @@ type Export struct {
|
|||||||
// between.
|
// between.
|
||||||
lock sync.Locker
|
lock sync.Locker
|
||||||
|
|
||||||
// currentPath returns the path Path gives the target under
|
// currentPath returns the path ArchivePath gives the target under
|
||||||
// the names stored for it now, which a rename may have changed since
|
// the names stored for it now, which a rename may have changed since
|
||||||
// the export was listed.
|
// the export was listed.
|
||||||
currentPath func() (string, error)
|
currentPath func() (string, error)
|
||||||
@@ -103,12 +103,11 @@ type exportedName struct {
|
|||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewExport lists a database target's archive files for export,
|
// NewArchiveExport lists a database target's archive files for export,
|
||||||
// given the path Path gives it (see archiveFiles). It opens none
|
// given the path ArchivePath gives it (see archiveFiles). It opens none
|
||||||
// of them. Its caller holds lock, which every rename of the target's
|
// of them. Its caller holds lock, which every rename of the target's
|
||||||
// files runs under (Writers.RenameLock returns it), from reading the names
|
// files runs under, from reading the names path is made of until it
|
||||||
// path is made of until it returns, so the files it lists are the ones
|
// returns, so the files it lists are the ones those names give.
|
||||||
// those names give.
|
|
||||||
//
|
//
|
||||||
// WriteGzipJSON, called without lock held, finds each file again by its
|
// WriteGzipJSON, called without lock held, finds each file again by its
|
||||||
// period under the path currentPath gives, holding lock while it does
|
// period under the path currentPath gives, holding lock while it does
|
||||||
@@ -116,18 +115,18 @@ type exportedName struct {
|
|||||||
// file. A file that is gone by then, emptied by the sweep or moved
|
// file. A file that is gone by then, emptied by the sweep or moved
|
||||||
// away, is skipped. The export never creates a file: with no files, it
|
// away, is skipped. The export never creates a file: with no files, it
|
||||||
// has no rows.
|
// has no rows.
|
||||||
func NewExport(
|
func NewArchiveExport(
|
||||||
path string,
|
path string,
|
||||||
lock sync.Locker,
|
lock sync.Locker,
|
||||||
currentPath func() (string, error),
|
currentPath func() (string, error),
|
||||||
log *slog.Logger,
|
log *slog.Logger,
|
||||||
) (*Export, error) {
|
) (*ArchiveExport, error) {
|
||||||
files, err := archiveFiles(path)
|
files, err := archiveFiles(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
x := &Export{lock: lock, currentPath: currentPath, log: log}
|
x := &ArchiveExport{lock: lock, currentPath: currentPath, log: log}
|
||||||
|
|
||||||
for _, file := range files {
|
for _, file := range files {
|
||||||
x.periods = append(x.periods, file.period)
|
x.periods = append(x.periods, file.period)
|
||||||
@@ -200,7 +199,7 @@ func openExportFile(
|
|||||||
// closed once its rows are written, before the next is opened. When it
|
// closed once its rows are written, before the next is opened. When it
|
||||||
// returns, no file is open. After an error the gzip stream is left
|
// returns, no file is open. After an error the gzip stream is left
|
||||||
// unfinished, so what was written does not decompress as a whole file.
|
// unfinished, so what was written does not decompress as a whole file.
|
||||||
func (x *Export) WriteGzipJSON(
|
func (x *ArchiveExport) WriteGzipJSON(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
w io.Writer,
|
w io.Writer,
|
||||||
webhook *database.Webhook,
|
webhook *database.Webhook,
|
||||||
@@ -229,7 +228,7 @@ func (x *Export) WriteGzipJSON(
|
|||||||
// openFile finds the target's archive file for period under the path
|
// openFile finds the target's archive file for period under the path
|
||||||
// currentPath gives now, and opens it for the export, holding x.lock
|
// currentPath gives now, and opens it for the export, holding x.lock
|
||||||
// for both. For a file that is gone, the error wraps fs.ErrNotExist.
|
// for both. For a file that is gone, the error wraps fs.ErrNotExist.
|
||||||
func (x *Export) openFile(
|
func (x *ArchiveExport) openFile(
|
||||||
ctx context.Context, period string,
|
ctx context.Context, period string,
|
||||||
) (*exportFile, error) {
|
) (*exportFile, error) {
|
||||||
x.lock.Lock()
|
x.lock.Lock()
|
||||||
@@ -259,7 +258,7 @@ func (f *exportFile) close() error {
|
|||||||
|
|
||||||
// writeJSON writes head with archived_events added as its last key,
|
// writeJSON writes head with archived_events added as its last key,
|
||||||
// the rows going into it one at a time.
|
// the rows going into it one at a time.
|
||||||
func (x *Export) writeJSON(
|
func (x *ArchiveExport) writeJSON(
|
||||||
ctx context.Context, w io.Writer, head []byte,
|
ctx context.Context, w io.Writer, head []byte,
|
||||||
) error {
|
) error {
|
||||||
// head goes out without its closing brace, so that
|
// head goes out without its closing brace, so that
|
||||||
@@ -287,7 +286,7 @@ func (x *Export) writeJSON(
|
|||||||
// writeRows writes the archived rows of each file to w, one per line,
|
// writeRows writes the archived rows of each file to w, one per line,
|
||||||
// separated by commas, opening each file in turn and closing it once
|
// separated by commas, opening each file in turn and closing it once
|
||||||
// its rows are written.
|
// its rows are written.
|
||||||
func (x *Export) writeRows(ctx context.Context, w io.Writer) error {
|
func (x *ArchiveExport) writeRows(ctx context.Context, w io.Writer) error {
|
||||||
sep := "\n"
|
sep := "\n"
|
||||||
|
|
||||||
for _, period := range x.periods {
|
for _, period := range x.periods {
|
||||||
+27
-27
@@ -1,4 +1,4 @@
|
|||||||
package archive_test
|
package delivery_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
@@ -19,8 +19,8 @@ import (
|
|||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The webhook and the target the export tests' archives belong to.
|
// The webhook and the target the export tests' archives belong to.
|
||||||
@@ -37,7 +37,7 @@ const binaryBody = "\xff\xfe\x00\x01binary\x80"
|
|||||||
// writeExportTo writes export to w as the archive of the export tests'
|
// writeExportTo writes export to w as the archive of the export tests'
|
||||||
// webhook and target, exported at 2026-10-02T12:03:04Z.
|
// webhook and target, exported at 2026-10-02T12:03:04Z.
|
||||||
func writeExportTo(
|
func writeExportTo(
|
||||||
t *testing.T, export *archive.Export, w io.Writer,
|
t *testing.T, export *delivery.ArchiveExport, w io.Writer,
|
||||||
) error {
|
) error {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -57,7 +57,7 @@ func writeExportTo(
|
|||||||
|
|
||||||
// listExport lists the archive at path for export, as the archive of a
|
// listExport lists the archive at path for export, as the archive of a
|
||||||
// target whose names do not change.
|
// target whose names do not change.
|
||||||
func listExport(t *testing.T, path string) *archive.Export {
|
func listExport(t *testing.T, path string) *delivery.ArchiveExport {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
return newExport(t, path, &sync.Mutex{}, func() (string, error) {
|
return newExport(t, path, &sync.Mutex{}, func() (string, error) {
|
||||||
@@ -73,10 +73,10 @@ func newExport(
|
|||||||
path string,
|
path string,
|
||||||
lock sync.Locker,
|
lock sync.Locker,
|
||||||
currentPath func() (string, error),
|
currentPath func() (string, error),
|
||||||
) *archive.Export {
|
) *delivery.ArchiveExport {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
export, err := archive.NewExport(
|
export, err := delivery.NewArchiveExport(
|
||||||
path, lock, currentPath, archiveTestLogger(),
|
path, lock, currentPath, archiveTestLogger(),
|
||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
@@ -96,7 +96,7 @@ func exportArchive(t *testing.T, path string) map[string]any {
|
|||||||
// decompressed and parsed. Reading to the end makes the gzip reader
|
// decompressed and parsed. Reading to the end makes the gzip reader
|
||||||
// check that the stream was finished.
|
// check that the stream was finished.
|
||||||
func writeExport(
|
func writeExport(
|
||||||
t *testing.T, export *archive.Export,
|
t *testing.T, export *delivery.ArchiveExport,
|
||||||
) map[string]any {
|
) map[string]any {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -157,11 +157,11 @@ func TestArchiveExport_MatchesStoredRows(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive.db")
|
path := filepath.Join(t.TempDir(), "archive.db")
|
||||||
w := archive.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
||||||
bodies := []string{`{"order":1}`, "plain text", "", binaryBody}
|
bodies := []string{`{"order":1}`, "plain text", "", binaryBody}
|
||||||
|
|
||||||
for i, body := range bodies {
|
for i, body := range bodies {
|
||||||
require.NoError(t, w.Write(archive.ExportArchivedEvent{
|
require.NoError(t, w.Write(delivery.ExportArchivedEvent{
|
||||||
EventID: fmt.Sprintf("ev-%d", i),
|
EventID: fmt.Sprintf("ev-%d", i),
|
||||||
WebhookID: exportWebhookID,
|
WebhookID: exportWebhookID,
|
||||||
EntrypointID: "ep-1",
|
EntrypointID: "ep-1",
|
||||||
@@ -173,7 +173,7 @@ func TestArchiveExport_MatchesStoredRows(t *testing.T) {
|
|||||||
}, 0))
|
}, 0))
|
||||||
}
|
}
|
||||||
|
|
||||||
var stored []archive.ExportArchivedEvent
|
var stored []delivery.ExportArchivedEvent
|
||||||
|
|
||||||
require.NoError(t, openArchiveDBForRead(t, path).
|
require.NoError(t, openArchiveDBForRead(t, path).
|
||||||
Order("id").Find(&stored).Error)
|
Order("id").Find(&stored).Error)
|
||||||
@@ -201,7 +201,7 @@ func TestArchiveExport_MatchesStoredRows(t *testing.T) {
|
|||||||
// assertExportedRow checks that ev, from an export, holds every column
|
// assertExportedRow checks that ev, from an export, holds every column
|
||||||
// of the stored row.
|
// of the stored row.
|
||||||
func assertExportedRow(
|
func assertExportedRow(
|
||||||
t *testing.T, row archive.ExportArchivedEvent, ev map[string]any,
|
t *testing.T, row delivery.ExportArchivedEvent, ev map[string]any,
|
||||||
) {
|
) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -248,7 +248,7 @@ func TestArchiveExport_Empty(t *testing.T) {
|
|||||||
|
|
||||||
require.NoError(t, os.WriteFile(noTable, nil, 0o600))
|
require.NoError(t, os.WriteFile(noTable, nil, 0o600))
|
||||||
require.NoError(t,
|
require.NoError(t,
|
||||||
archive.NewExportArchiveWriter(noRows, archiveTestLogger(), 0).
|
delivery.NewExportArchiveWriter(noRows, archiveTestLogger(), 0).
|
||||||
Open(0),
|
Open(0),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -284,18 +284,18 @@ func TestArchiveExport_ReadsOneSnapshot(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive.db")
|
path := filepath.Join(t.TempDir(), "archive.db")
|
||||||
w := archive.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
||||||
|
|
||||||
require.NoError(t, w.Write(archive.ExportArchivedEvent{EventID: "listed"}, 0))
|
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "listed"}, 0))
|
||||||
|
|
||||||
opened := &unlockHook{fn: func() {
|
opened := &unlockHook{fn: func() {
|
||||||
require.NoError(t, w.Write(archive.ExportArchivedEvent{EventID: "during"}, 0))
|
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "during"}, 0))
|
||||||
}}
|
}}
|
||||||
export := newExport(t, path, opened, func() (string, error) {
|
export := newExport(t, path, opened, func() (string, error) {
|
||||||
return path, nil
|
return path, nil
|
||||||
})
|
})
|
||||||
|
|
||||||
require.NoError(t, w.Write(archive.ExportArchivedEvent{EventID: "before-open"}, 0))
|
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "before-open"}, 0))
|
||||||
|
|
||||||
assert.Equal(t,
|
assert.Equal(t,
|
||||||
[]string{"listed", "before-open"},
|
[]string{"listed", "before-open"},
|
||||||
@@ -305,7 +305,7 @@ func TestArchiveExport_ReadsOneSnapshot(t *testing.T) {
|
|||||||
var stored int64
|
var stored int64
|
||||||
|
|
||||||
require.NoError(t, openArchiveDBForRead(t, path).
|
require.NoError(t, openArchiveDBForRead(t, path).
|
||||||
Model(&archive.ExportArchivedEvent{}).Count(&stored).Error)
|
Model(&delivery.ExportArchivedEvent{}).Count(&stored).Error)
|
||||||
assert.Equal(t, int64(3), stored)
|
assert.Equal(t, int64(3), stored)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -318,11 +318,11 @@ func TestArchiveExport_FindsFilesAfterRename(t *testing.T) {
|
|||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
path := filepath.Join(dir, "archive-old.db")
|
path := filepath.Join(dir, "archive-old.db")
|
||||||
w := archive.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
||||||
|
|
||||||
for _, period := range []string{"", dayPeriod, hourPeriod} {
|
for _, period := range []string{"", dayPeriod, hourPeriod} {
|
||||||
require.NoError(t, w.WritePeriod(
|
require.NoError(t, w.WritePeriod(
|
||||||
archive.ExportArchivedEvent{EventID: "in-" + period}, 0, period,
|
delivery.ExportArchivedEvent{EventID: "in-" + period}, 0, period,
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -353,19 +353,19 @@ func TestArchiveExport_EveryFileOldestFirst(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
||||||
w := archive.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
||||||
|
|
||||||
// Written in an order that is not the export's.
|
// Written in an order that is not the export's.
|
||||||
for _, period := range []string{nextDayPeriod, "", hourPeriod, "2026-03"} {
|
for _, period := range []string{nextDayPeriod, "", hourPeriod, "2026-03"} {
|
||||||
require.NoError(t, w.WritePeriod(
|
require.NoError(t, w.WritePeriod(
|
||||||
archive.ExportArchivedEvent{EventID: "in-" + period}, 0, period,
|
delivery.ExportArchivedEvent{EventID: "in-" + period}, 0, period,
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
export := listExport(t, path)
|
export := listExport(t, path)
|
||||||
|
|
||||||
require.NoError(t, w.WritePeriod(
|
require.NoError(t, w.WritePeriod(
|
||||||
archive.ExportArchivedEvent{EventID: "later"}, 0, "2026-03-06",
|
delivery.ExportArchivedEvent{EventID: "later"}, 0, "2026-03-06",
|
||||||
))
|
))
|
||||||
|
|
||||||
events := exportedEvents(t, writeExport(t, export))
|
events := exportedEvents(t, writeExport(t, export))
|
||||||
@@ -434,13 +434,13 @@ func TestArchiveExport_OneFileOpenAtATime(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
path := filepath.Join(dir, "archive-wh.db")
|
path := filepath.Join(dir, "archive-wh.db")
|
||||||
w := archive.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
||||||
random := make([]byte, 36<<10)
|
random := make([]byte, 36<<10)
|
||||||
|
|
||||||
for hour := range 24 {
|
for hour := range 24 {
|
||||||
_, _ = rand.Read(random)
|
_, _ = rand.Read(random)
|
||||||
|
|
||||||
require.NoError(t, w.WritePeriod(archive.ExportArchivedEvent{
|
require.NoError(t, w.WritePeriod(delivery.ExportArchivedEvent{
|
||||||
Body: base64.StdEncoding.EncodeToString(random),
|
Body: base64.StdEncoding.EncodeToString(random),
|
||||||
}, 0, fmt.Sprintf("2026-10-01-%02d", hour)))
|
}, 0, fmt.Sprintf("2026-10-01-%02d", hour)))
|
||||||
}
|
}
|
||||||
@@ -489,7 +489,7 @@ func exportHeapGrowth(t *testing.T, rows, bodySize int) uint64 {
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive.db")
|
path := filepath.Join(t.TempDir(), "archive.db")
|
||||||
w := archive.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
||||||
|
|
||||||
// Base64 makes four characters of every three bytes.
|
// Base64 makes four characters of every three bytes.
|
||||||
random := make([]byte, bodySize/4*3)
|
random := make([]byte, bodySize/4*3)
|
||||||
@@ -497,7 +497,7 @@ func exportHeapGrowth(t *testing.T, rows, bodySize int) uint64 {
|
|||||||
for range rows {
|
for range rows {
|
||||||
_, _ = rand.Read(random)
|
_, _ = rand.Read(random)
|
||||||
|
|
||||||
require.NoError(t, w.Write(archive.ExportArchivedEvent{
|
require.NoError(t, w.Write(delivery.ExportArchivedEvent{
|
||||||
Body: base64.StdEncoding.EncodeToString(random),
|
Body: base64.StdEncoding.EncodeToString(random),
|
||||||
}, 0))
|
}, 0))
|
||||||
}
|
}
|
||||||
@@ -565,7 +565,7 @@ func TestArchiveExportFileName(t *testing.T) {
|
|||||||
|
|
||||||
assert.Equal(t,
|
assert.Equal(t,
|
||||||
"archive-orders-eu-long-term-archive-20261002T120304Z.json.gz",
|
"archive-orders-eu-long-term-archive-20261002T120304Z.json.gz",
|
||||||
archive.ExportFileName(
|
delivery.ArchiveExportFileName(
|
||||||
exportWebhookName, exportTargetName,
|
exportWebhookName, exportTargetName,
|
||||||
time.Date(2026, 10, 2, 14, 3, 4, 0, cest),
|
time.Date(2026, 10, 2, 14, 3, 4, 0, cest),
|
||||||
),
|
),
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
package archive
|
package delivery
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
@@ -41,10 +41,10 @@ type archiveFile struct {
|
|||||||
period string
|
period string
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidateRotation checks a user-supplied archive rotation for
|
// ValidateArchiveRotation checks a user-supplied archive rotation for
|
||||||
// a database target: empty or none (both meaning one file), monthly,
|
// a database target: empty or none (both meaning one file), monthly,
|
||||||
// daily or hourly.
|
// daily or hourly.
|
||||||
func ValidateRotation(rotation string) error {
|
func ValidateArchiveRotation(rotation string) error {
|
||||||
switch rotation {
|
switch rotation {
|
||||||
case "", archiveRotationNone, archiveRotationMonthly,
|
case "", archiveRotationNone, archiveRotationMonthly,
|
||||||
archiveRotationDaily, archiveRotationHourly:
|
archiveRotationDaily, archiveRotationHourly:
|
||||||
@@ -54,21 +54,21 @@ func ValidateRotation(rotation string) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseRotation reads the rotation from a database target's
|
// parseArchiveRotation reads the rotation from a database target's
|
||||||
// config JSON. An empty config or an empty rotation is none.
|
// config JSON. An empty config or an empty rotation is none.
|
||||||
func ParseRotation(configJSON string) (string, error) {
|
func parseArchiveRotation(configJSON string) (string, error) {
|
||||||
if configJSON == "" {
|
if configJSON == "" {
|
||||||
return archiveRotationNone, nil
|
return archiveRotationNone, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var cfg DatabaseTargetConfig
|
var cfg databaseTargetConfig
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(configJSON), &cfg)
|
err := json.Unmarshal([]byte(configJSON), &cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("parsing database target config: %w", err)
|
return "", fmt.Errorf("parsing database target config: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = ValidateRotation(cfg.Rotation)
|
err = ValidateArchiveRotation(cfg.Rotation)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@@ -97,7 +97,7 @@ func archivePeriod(rotation string, receivedAt time.Time) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// archivePeriodPath returns the path of a database target's archive
|
// archivePeriodPath returns the path of a database target's archive
|
||||||
// file for a period: path, as Path gives it, with "-" and the
|
// file for a period: path, as ArchivePath gives it, with "-" and the
|
||||||
// period put before its ".db". The period "" gives path itself.
|
// period put before its ".db". The period "" gives path itself.
|
||||||
func archivePeriodPath(path, period string) string {
|
func archivePeriodPath(path, period string) string {
|
||||||
if period == "" {
|
if period == "" {
|
||||||
@@ -107,28 +107,28 @@ func archivePeriodPath(path, period string) string {
|
|||||||
return strings.TrimSuffix(path, ".db") + "-" + period + ".db"
|
return strings.TrimSuffix(path, ".db") + "-" + period + ".db"
|
||||||
}
|
}
|
||||||
|
|
||||||
// PathAt returns the archive file a database target writes an
|
// ArchivePathAt returns the archive file a database target writes an
|
||||||
// event received at receivedAt to: Path's file, with the
|
// event received at receivedAt to: ArchivePath's file, with the
|
||||||
// period in its name when the target rotates.
|
// period in its name when the target rotates.
|
||||||
func PathAt(
|
func ArchivePathAt(
|
||||||
dbMgr *database.WebhookDBManager,
|
dbMgr *database.WebhookDBManager,
|
||||||
webhook *database.Webhook,
|
webhook *database.Webhook,
|
||||||
target *database.Target,
|
target *database.Target,
|
||||||
receivedAt time.Time,
|
receivedAt time.Time,
|
||||||
) (string, error) {
|
) (string, error) {
|
||||||
rotation, err := ParseRotation(target.Config)
|
rotation, err := parseArchiveRotation(target.Config)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
return archivePeriodPath(
|
return archivePeriodPath(
|
||||||
Path(dbMgr, webhook, target),
|
ArchivePath(dbMgr, webhook, target),
|
||||||
archivePeriod(rotation, receivedAt),
|
archivePeriod(rotation, receivedAt),
|
||||||
), nil
|
), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// archiveFiles lists the database target's archive files that exist,
|
// archiveFiles lists the database target's archive files that exist,
|
||||||
// given the path Path gives it: the file at path, then each
|
// given the path ArchivePath gives it: the file at path, then each
|
||||||
// file archivePeriodPath names for path and a period, oldest period
|
// file archivePeriodPath names for path and a period, oldest period
|
||||||
// first. Which rotation wrote a file does not matter, so the files of
|
// first. Which rotation wrote a file does not matter, so the files of
|
||||||
// an earlier rotation setting are listed too.
|
// an earlier rotation setting are listed too.
|
||||||
+26
-24
@@ -1,4 +1,4 @@
|
|||||||
package archive_test
|
package delivery_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"os"
|
"os"
|
||||||
@@ -9,9 +9,9 @@ import (
|
|||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
||||||
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The archive rotations, and the configs of a daily target and of one
|
// The archive rotations, and the configs of a daily target and of one
|
||||||
@@ -40,30 +40,32 @@ func periodPath(path, period string) string {
|
|||||||
return strings.TrimSuffix(path, ".db") + "-" + period + ".db"
|
return strings.TrimSuffix(path, ".db") + "-" + period + ".db"
|
||||||
}
|
}
|
||||||
|
|
||||||
// deliverReceivedAt archives, as a delivery to a database target
|
// deliverReceivedAt delivers to a database target an event whose
|
||||||
// does, an event whose receive time is receivedAt, and returns the
|
// receive time is receivedAt, and returns the event's id. The receive
|
||||||
// event's id. The receive time is what decides a rotated archive's
|
// time is what decides a rotated archive's file, so setting it is how
|
||||||
// file, so setting it is how these tests move the clock across a
|
// these tests move the clock across a period boundary.
|
||||||
// period boundary.
|
|
||||||
func (env *archiveEnv) deliverReceivedAt(
|
func (env *archiveEnv) deliverReceivedAt(
|
||||||
t *testing.T, tgt *database.Target, receivedAt time.Time,
|
t *testing.T, tgt *database.Target, receivedAt time.Time,
|
||||||
) string {
|
) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
event := newEvent(`{"n":1}`)
|
webhookDB := testWebhookDB(t)
|
||||||
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
event.CreatedAt = receivedAt
|
event.CreatedAt = receivedAt
|
||||||
|
|
||||||
require.NoError(t, env.writers.Write(databaseTargetDelivery(event, tgt)))
|
env.eng.ExportDeliverDatabase(
|
||||||
|
webhookDB, seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
|
||||||
|
)
|
||||||
|
|
||||||
return event.ID
|
return event.ID
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestWrite_RotatesAtEachPeriodBoundary writes, for each
|
// TestDeliverDatabase_RotatesAtEachPeriodBoundary delivers, for each
|
||||||
// rotation, an event received in the last second of a period and one
|
// rotation, an event received in the last second of a period and one
|
||||||
// received in the first second of the next, and checks each lands in
|
// received in the first second of the next, and checks each lands in
|
||||||
// the file named for its own period, in UTC. Rotation none keeps both
|
// the file named for its own period, in UTC. Rotation none keeps both
|
||||||
// in the one file.
|
// in the one file.
|
||||||
func TestWrite_RotatesAtEachPeriodBoundary(t *testing.T) {
|
func TestDeliverDatabase_RotatesAtEachPeriodBoundary(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
berlin := time.FixedZone("CEST", 2*60*60)
|
berlin := time.FixedZone("CEST", 2*60*60)
|
||||||
@@ -137,10 +139,10 @@ func TestWrite_RotatesAtEachPeriodBoundary(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestWrite_EventWithoutReceiveTime proves an event whose
|
// TestDeliverDatabase_EventWithoutReceiveTime proves an event whose
|
||||||
// receive time is not known goes to the file for the time it is
|
// receive time is not known goes to the file for the time it is
|
||||||
// archived, rather than to one for the year 1.
|
// archived, rather than to one for the year 1.
|
||||||
func TestWrite_EventWithoutReceiveTime(t *testing.T) {
|
func TestDeliverDatabase_EventWithoutReceiveTime(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupArchiveTest(t)
|
||||||
@@ -162,10 +164,10 @@ func TestWrite_EventWithoutReceiveTime(t *testing.T) {
|
|||||||
assert.NoFileExists(t, periodPath(path, "0001-01"))
|
assert.NoFileExists(t, periodPath(path, "0001-01"))
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestWrite_RotationChangeKeepsOldFiles changes a target's
|
// TestDeliverDatabase_RotationChangeKeepsOldFiles changes a target's
|
||||||
// rotation from none to daily between two events, and checks the
|
// rotation from none to daily between two events, and checks the
|
||||||
// second goes to the daily file while the first stays where it was.
|
// second goes to the daily file while the first stays where it was.
|
||||||
func TestWrite_RotationChangeKeepsOldFiles(t *testing.T) {
|
func TestDeliverDatabase_RotationChangeKeepsOldFiles(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupArchiveTest(t)
|
||||||
@@ -235,7 +237,7 @@ func TestRename_MovesEveryFile(t *testing.T) {
|
|||||||
first := env.deliverReceivedAt(t, tgt, day)
|
first := env.deliverReceivedAt(t, tgt, day)
|
||||||
second := env.deliverReceivedAt(t, tgt, day.Add(24*time.Hour))
|
second := env.deliverReceivedAt(t, tgt, day.Add(24*time.Hour))
|
||||||
|
|
||||||
require.NoError(t, env.writers.Rename(tgt.ID, "Orders", "Long Term"))
|
require.NoError(t, env.eng.Rename(tgt.ID, "Orders", "Long Term"))
|
||||||
|
|
||||||
newPath := filepath.Join(
|
newPath := filepath.Join(
|
||||||
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
||||||
@@ -281,8 +283,8 @@ func TestRename_NeverReplacesARotatedFile(t *testing.T) {
|
|||||||
require.NoError(t, os.WriteFile(planted, []byte("planted"), 0o600))
|
require.NoError(t, os.WriteFile(planted, []byte("planted"), 0o600))
|
||||||
|
|
||||||
require.ErrorIs(
|
require.ErrorIs(
|
||||||
t, env.writers.Rename(tgt.ID, "Orders", "Long Term"),
|
t, env.eng.Rename(tgt.ID, "Orders", "Long Term"),
|
||||||
archive.ErrNameTaken,
|
delivery.ErrArchiveNameTaken,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.FileExists(t, periodPath(oldPath, dayPeriod))
|
assert.FileExists(t, periodPath(oldPath, dayPeriod))
|
||||||
@@ -290,7 +292,7 @@ func TestRename_NeverReplacesARotatedFile(t *testing.T) {
|
|||||||
assert.NoFileExists(t, periodPath(newPath, dayPeriod))
|
assert.NoFileExists(t, periodPath(newPath, dayPeriod))
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestStatArchive_EveryFile proves Stat counts and adds up every
|
// TestStatArchive_EveryFile proves StatArchive counts and adds up every
|
||||||
// one of a target's files, takes the latest write of any of them, and
|
// one of a target's files, takes the latest write of any of them, and
|
||||||
// leaves out files whose names only look like the target's.
|
// leaves out files whose names only look like the target's.
|
||||||
func TestStatArchive_EveryFile(t *testing.T) {
|
func TestStatArchive_EveryFile(t *testing.T) {
|
||||||
@@ -322,7 +324,7 @@ func TestStatArchive_EveryFile(t *testing.T) {
|
|||||||
os.WriteFile(filepath.Join(dir, other), []byte("x"), 0o600))
|
os.WriteFile(filepath.Join(dir, other), []byte("x"), 0o600))
|
||||||
}
|
}
|
||||||
|
|
||||||
got, err := archive.Stat(path)
|
got, err := delivery.StatArchive(path)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, len(files), got.Files)
|
assert.Equal(t, len(files), got.Files)
|
||||||
assert.Equal(t, size, got.Size)
|
assert.Equal(t, size, got.Size)
|
||||||
@@ -355,7 +357,7 @@ func TestArchivePathAt(t *testing.T) {
|
|||||||
Config: config,
|
Config: config,
|
||||||
}
|
}
|
||||||
|
|
||||||
got, err := archive.PathAt(dbMgr, webhook, target, at)
|
got, err := delivery.ArchivePathAt(dbMgr, webhook, target, at)
|
||||||
require.NoError(t, err, config)
|
require.NoError(t, err, config)
|
||||||
assert.Equal(t,
|
assert.Equal(t,
|
||||||
filepath.Join(
|
filepath.Join(
|
||||||
@@ -365,7 +367,7 @@ func TestArchivePathAt(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := archive.PathAt(dbMgr, webhook, &database.Target{
|
_, err := delivery.ArchivePathAt(dbMgr, webhook, &database.Target{
|
||||||
Config: weeklyConfig,
|
Config: weeklyConfig,
|
||||||
}, at)
|
}, at)
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
@@ -379,10 +381,10 @@ func TestValidateArchiveRotation(t *testing.T) {
|
|||||||
for _, ok := range []string{
|
for _, ok := range []string{
|
||||||
"", rotationNone, rotationMonthly, rotationDaily, rotationHourly,
|
"", rotationNone, rotationMonthly, rotationDaily, rotationHourly,
|
||||||
} {
|
} {
|
||||||
require.NoError(t, archive.ValidateRotation(ok), ok)
|
require.NoError(t, delivery.ValidateArchiveRotation(ok), ok)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, bad := range []string{"weekly", "Daily", "hourly "} {
|
for _, bad := range []string{"weekly", "Daily", "hourly "} {
|
||||||
require.Error(t, archive.ValidateRotation(bad), bad)
|
require.Error(t, delivery.ValidateArchiveRotation(bad), bad)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3,65 +3,24 @@ package delivery_test
|
|||||||
import (
|
import (
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"gorm.io/driver/sqlite"
|
"gorm.io/driver/sqlite"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"gorm.io/gorm/clause"
|
|
||||||
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/database/databasetest"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
"sneak.berlin/go/webhooker/internal/gormlog"
|
||||||
)
|
)
|
||||||
|
|
||||||
// archiveTestWebhookName is the name of every webhook
|
|
||||||
// seedDatabaseTarget creates. It is not safe in a file name as it
|
|
||||||
// stands, so every archive file name made from it is made safe first.
|
|
||||||
const archiveTestWebhookName = "Sweep Test!"
|
|
||||||
|
|
||||||
// archiveEnv bundles the pieces a database target test drives: a main
|
|
||||||
// configuration database holding webhooks and targets, a delivery
|
|
||||||
// engine whose archive writers write into the data directory, and that
|
|
||||||
// directory.
|
|
||||||
type archiveEnv struct {
|
|
||||||
eng *delivery.Engine
|
|
||||||
mainDB *database.Database
|
|
||||||
dataDir string
|
|
||||||
}
|
|
||||||
|
|
||||||
func setupArchiveTest(t *testing.T) *archiveEnv {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
dataDir := t.TempDir()
|
|
||||||
|
|
||||||
mainDB, err := database.Open(dataDir, slog.New(slog.DiscardHandler))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
t.Cleanup(func() { _ = mainDB.Close() })
|
|
||||||
|
|
||||||
return &archiveEnv{
|
|
||||||
eng: delivery.NewTestEngineWithDB(
|
|
||||||
mainDB,
|
|
||||||
databasetest.NewWebhookDBManager(t, dataDir),
|
|
||||||
archiveTestLogger(),
|
|
||||||
&http.Client{Timeout: 5 * time.Second},
|
|
||||||
1,
|
|
||||||
),
|
|
||||||
mainDB: mainDB,
|
|
||||||
dataDir: dataDir,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func archiveTestLogger() *slog.Logger {
|
func archiveTestLogger() *slog.Logger {
|
||||||
return slog.New(slog.NewTextHandler(
|
return slog.New(slog.NewTextHandler(
|
||||||
os.Stderr,
|
os.Stderr,
|
||||||
@@ -69,85 +28,8 @@ func archiveTestLogger() *slog.Logger {
|
|||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
// archivePath returns where the engine keeps a database target's
|
|
||||||
// archive file, for the names seedDatabaseTarget gave it.
|
|
||||||
func (env *archiveEnv) archivePath(tgt *database.Target) string {
|
|
||||||
return filepath.Join(
|
|
||||||
env.dataDir,
|
|
||||||
archive.FileName(archiveTestWebhookName, tgt.Name, tgt.ID),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// seedDatabaseTarget creates a webhook with one database target
|
|
||||||
// carrying the given target config JSON, and returns the target.
|
|
||||||
func (env *archiveEnv) seedDatabaseTarget(
|
|
||||||
t *testing.T, configJSON string,
|
|
||||||
) *database.Target {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
wh := &database.Webhook{
|
|
||||||
UserID: uuid.New().String(),
|
|
||||||
Name: archiveTestWebhookName,
|
|
||||||
}
|
|
||||||
require.NoError(
|
|
||||||
t,
|
|
||||||
env.mainDB.DB().
|
|
||||||
Omit(clause.Associations).
|
|
||||||
Create(wh).Error,
|
|
||||||
)
|
|
||||||
|
|
||||||
tgt := &database.Target{
|
|
||||||
WebhookID: wh.ID,
|
|
||||||
Name: "Archive",
|
|
||||||
Type: database.TargetTypeDatabase,
|
|
||||||
Active: true,
|
|
||||||
Config: configJSON,
|
|
||||||
MaxAttempts: 1,
|
|
||||||
}
|
|
||||||
require.NoError(
|
|
||||||
t,
|
|
||||||
env.mainDB.DB().
|
|
||||||
Omit(clause.Associations).
|
|
||||||
Create(tgt).Error,
|
|
||||||
)
|
|
||||||
|
|
||||||
return tgt
|
|
||||||
}
|
|
||||||
|
|
||||||
// seedDatabaseTargetDelivery seeds a pending delivery of an event
|
|
||||||
// to a database target and returns the in-memory delivery the
|
|
||||||
// target handler is invoked with.
|
|
||||||
func seedDatabaseTargetDelivery(
|
|
||||||
t *testing.T,
|
|
||||||
webhookDB *gorm.DB,
|
|
||||||
event database.Event,
|
|
||||||
tgt *database.Target,
|
|
||||||
) *database.Delivery {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
dlv := seedDelivery(
|
|
||||||
t, webhookDB, event.ID, tgt.ID,
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
|
|
||||||
d := &database.Delivery{
|
|
||||||
EventID: event.ID,
|
|
||||||
TargetID: tgt.ID,
|
|
||||||
Status: database.DeliveryStatusPending,
|
|
||||||
Event: event,
|
|
||||||
Target: database.Target{
|
|
||||||
Name: tgt.Name,
|
|
||||||
Type: database.TargetTypeDatabase,
|
|
||||||
Config: tgt.Config,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
d.ID = dlv.ID
|
|
||||||
|
|
||||||
return d
|
|
||||||
}
|
|
||||||
|
|
||||||
// openArchiveDBForRead opens an archive file read-only so a
|
// openArchiveDBForRead opens an archive file read-only so a
|
||||||
// test can inspect the rows the database target archived.
|
// test can inspect the rows the writer persisted.
|
||||||
func openArchiveDBForRead(
|
func openArchiveDBForRead(
|
||||||
t *testing.T, path string,
|
t *testing.T, path string,
|
||||||
) *gorm.DB {
|
) *gorm.DB {
|
||||||
@@ -170,45 +52,26 @@ func openArchiveDBForRead(
|
|||||||
return gdb
|
return gdb
|
||||||
}
|
}
|
||||||
|
|
||||||
// countArchivedRows counts the rows in an archive file without
|
// archiveFileSuffixes returns the archive file itself and the
|
||||||
// asserting anything, so it is safe to poll from an
|
// SQLite sidecars that accompany an open database. A test that
|
||||||
// assert.Eventually condition (which runs off the test
|
// asserts no archive was created has to check all of them.
|
||||||
// goroutine, where testify assertions must not be used).
|
func archiveFileSuffixes() []string {
|
||||||
func countArchivedRows(path string) (int64, error) {
|
return []string{"", "-wal", "-shm"}
|
||||||
sqlDB, err := sql.Open(
|
|
||||||
"sqlite", fmt.Sprintf("file:%s?mode=ro", path),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = sqlDB.Close() }()
|
|
||||||
|
|
||||||
gdb, err := gorm.Open(
|
|
||||||
sqlite.Dialector{Conn: sqlDB},
|
|
||||||
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
var count int64
|
|
||||||
|
|
||||||
err = gdb.Table("archived_events").Count(&count).Error
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return count, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// archivedRow is the columns of an archive row these tests read.
|
// removeArchiveFiles simulates an operator moving the archive
|
||||||
type archivedRow struct {
|
// away by deleting the SQLite file and its sidecar files.
|
||||||
EventID string
|
func removeArchiveFiles(t *testing.T, path string) {
|
||||||
WebhookID string
|
t.Helper()
|
||||||
Method string
|
|
||||||
RawQuery string
|
for _, suffix := range []string{
|
||||||
Body string
|
"", "-wal", "-shm", "-journal",
|
||||||
|
} {
|
||||||
|
err := os.Remove(path + suffix)
|
||||||
|
if err != nil && !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("removing %s%s: %v", path, suffix, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestDeliverDatabase_ArchivesEvent verifies that delivering to
|
// TestDeliverDatabase_ArchivesEvent verifies that delivering to
|
||||||
@@ -244,9 +107,9 @@ func TestDeliverDatabase_ArchivesEvent(t *testing.T) {
|
|||||||
|
|
||||||
rdb := openArchiveDBForRead(t, archivePath)
|
rdb := openArchiveDBForRead(t, archivePath)
|
||||||
|
|
||||||
var rows []archivedRow
|
var rows []delivery.ExportArchivedEvent
|
||||||
|
|
||||||
require.NoError(t, rdb.Table("archived_events").Find(&rows).Error)
|
require.NoError(t, rdb.Find(&rows).Error)
|
||||||
require.Len(t, rows, 1)
|
require.Len(t, rows, 1)
|
||||||
assert.Equal(t, event.ID, rows[0].EventID)
|
assert.Equal(t, event.ID, rows[0].EventID)
|
||||||
assert.Equal(t, event.WebhookID, rows[0].WebhookID)
|
assert.Equal(t, event.WebhookID, rows[0].WebhookID)
|
||||||
@@ -255,6 +118,262 @@ func TestDeliverDatabase_ArchivesEvent(t *testing.T) {
|
|||||||
assert.JSONEq(t, `{"archived":true}`, rows[0].Body)
|
assert.JSONEq(t, `{"archived":true}`, rows[0].Body)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestArchiveWriter_WritesRow(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
||||||
|
w := delivery.NewExportArchiveWriter(
|
||||||
|
path, archiveTestLogger(), 0,
|
||||||
|
)
|
||||||
|
|
||||||
|
row := delivery.ExportArchivedEvent{
|
||||||
|
EventID: "ev-1",
|
||||||
|
WebhookID: "wh-1",
|
||||||
|
EntrypointID: "ep-1",
|
||||||
|
Method: "POST",
|
||||||
|
Headers: `{"X":"Y"}`,
|
||||||
|
Body: `{"hello":"world"}`,
|
||||||
|
ContentType: "application/json",
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, w.Write(row, 0))
|
||||||
|
assert.FileExists(t, path)
|
||||||
|
|
||||||
|
var got []delivery.ExportArchivedEvent
|
||||||
|
|
||||||
|
require.NoError(t, w.DB().Find(&got).Error)
|
||||||
|
require.Len(t, got, 1)
|
||||||
|
assert.Equal(t, "ev-1", got[0].EventID)
|
||||||
|
assert.Equal(t, "wh-1", got[0].WebhookID)
|
||||||
|
assert.Equal(t, "ep-1", got[0].EntrypointID)
|
||||||
|
assert.Equal(t, row.Method, got[0].Method)
|
||||||
|
assert.Equal(t, row.ContentType, got[0].ContentType)
|
||||||
|
assert.JSONEq(t, `{"hello":"world"}`, got[0].Body)
|
||||||
|
assert.False(t, got[0].ArchivedAt.IsZero())
|
||||||
|
|
||||||
|
var archivedAt string
|
||||||
|
|
||||||
|
require.NoError(t, w.DB().Raw(
|
||||||
|
"SELECT CAST(archived_at AS TEXT) FROM archived_events",
|
||||||
|
).Row().Scan(&archivedAt))
|
||||||
|
assertStoredTime(t, archivedAt)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveWriter_RecreatesAfterRemoval(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
||||||
|
w := delivery.NewExportArchiveWriter(
|
||||||
|
path, archiveTestLogger(), 0,
|
||||||
|
)
|
||||||
|
|
||||||
|
require.NoError(t, w.Write(
|
||||||
|
delivery.ExportArchivedEvent{EventID: "a"}, 0,
|
||||||
|
))
|
||||||
|
assert.FileExists(t, path)
|
||||||
|
|
||||||
|
// The operator moves the archive away while the handle is
|
||||||
|
// still open.
|
||||||
|
removeArchiveFiles(t, path)
|
||||||
|
require.NoFileExists(t, path)
|
||||||
|
|
||||||
|
// The next write recreates the file with a fresh schema and
|
||||||
|
// only the new row.
|
||||||
|
require.NoError(t, w.Write(
|
||||||
|
delivery.ExportArchivedEvent{EventID: "b"}, 0,
|
||||||
|
))
|
||||||
|
assert.FileExists(t, path)
|
||||||
|
|
||||||
|
var got []delivery.ExportArchivedEvent
|
||||||
|
|
||||||
|
require.NoError(t, w.DB().Find(&got).Error)
|
||||||
|
require.Len(t, got, 1)
|
||||||
|
assert.Equal(t, "b", got[0].EventID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestStatArchive proves StatArchive finds no file before the first
|
||||||
|
// write; after a write still held in the -wal, counts the -wal in the
|
||||||
|
// size and takes its later time as the last write; and finds no file
|
||||||
|
// again once the file has been moved away.
|
||||||
|
func TestStatArchive(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
||||||
|
|
||||||
|
_, err := delivery.StatArchive(path)
|
||||||
|
require.ErrorIs(t, err, fs.ErrNotExist)
|
||||||
|
|
||||||
|
// With the clock stopped, the reopen debounce never passes, so
|
||||||
|
// the handle stays open after the write.
|
||||||
|
stopped := time.Now()
|
||||||
|
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
||||||
|
w.SetNow(func() time.Time { return stopped })
|
||||||
|
|
||||||
|
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "a"}, 0))
|
||||||
|
|
||||||
|
written := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||||
|
earlier := written.Add(-time.Hour)
|
||||||
|
require.NoError(t, os.Chtimes(path, earlier, earlier))
|
||||||
|
require.NoError(t, os.Chtimes(path+"-wal", written, written))
|
||||||
|
|
||||||
|
file, err := os.Stat(path)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
wal, err := os.Stat(path + "-wal")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Positive(t, wal.Size())
|
||||||
|
|
||||||
|
got, err := delivery.StatArchive(path)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, 1, got.Files)
|
||||||
|
assert.Equal(t, file.Size()+wal.Size(), got.Size)
|
||||||
|
assert.True(t, written.Equal(got.Written), got.Written)
|
||||||
|
|
||||||
|
removeArchiveFiles(t, path)
|
||||||
|
|
||||||
|
_, err = delivery.StatArchive(path)
|
||||||
|
require.ErrorIs(t, err, fs.ErrNotExist)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveWriter_ReopenDebounce(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const debounce = 2 * time.Second
|
||||||
|
|
||||||
|
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
||||||
|
w := delivery.NewExportArchiveWriter(
|
||||||
|
path, archiveTestLogger(), debounce,
|
||||||
|
)
|
||||||
|
|
||||||
|
// The writer measures its reopen debounce on this clock, which
|
||||||
|
// only the test moves, so how long the host takes between
|
||||||
|
// writes cannot change the result.
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
w.SetNow(func() time.Time { return now })
|
||||||
|
|
||||||
|
require.NoError(t, w.Write(
|
||||||
|
delivery.ExportArchivedEvent{EventID: "a"}, 0,
|
||||||
|
))
|
||||||
|
require.NoError(t, w.Write(
|
||||||
|
delivery.ExportArchivedEvent{EventID: "b"}, 0,
|
||||||
|
))
|
||||||
|
|
||||||
|
// Two writes inside the debounce window trigger only the
|
||||||
|
// initial open — no extra close/reopen.
|
||||||
|
assert.Equal(t, 1, w.Reopens())
|
||||||
|
|
||||||
|
now = now.Add(debounce)
|
||||||
|
|
||||||
|
require.NoError(t, w.Write(
|
||||||
|
delivery.ExportArchivedEvent{EventID: "c"}, 0,
|
||||||
|
))
|
||||||
|
|
||||||
|
// A write after the window elapses closes and reopens once.
|
||||||
|
assert.Equal(t, 2, w.Reopens())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestArchiveWriter_ExpiryPrune(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
||||||
|
w := delivery.NewExportArchiveWriter(
|
||||||
|
path, archiveTestLogger(), 0,
|
||||||
|
)
|
||||||
|
|
||||||
|
require.NoError(t, w.Open(0))
|
||||||
|
|
||||||
|
old := delivery.ExportArchivedEvent{
|
||||||
|
EventID: "old",
|
||||||
|
ArchivedAt: time.Now().Add(-2 * time.Hour),
|
||||||
|
}
|
||||||
|
fresh := delivery.ExportArchivedEvent{
|
||||||
|
EventID: "fresh",
|
||||||
|
ArchivedAt: time.Now(),
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, w.DB().Create(&old).Error)
|
||||||
|
require.NoError(t, w.DB().Create(&fresh).Error)
|
||||||
|
|
||||||
|
// Reopening with a one-hour expiry prunes the old row.
|
||||||
|
require.NoError(t, w.Reopen(time.Hour))
|
||||||
|
|
||||||
|
var got []delivery.ExportArchivedEvent
|
||||||
|
|
||||||
|
require.NoError(t, w.DB().Find(&got).Error)
|
||||||
|
require.Len(t, got, 1)
|
||||||
|
assert.Equal(t, "fresh", got[0].EventID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseArchiveExpiry(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
in string
|
||||||
|
want time.Duration
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{"empty config", "", 0, false},
|
||||||
|
{"explicit never", `{"expiry":"never"}`, 0, false},
|
||||||
|
{"empty expiry", `{"expiry":""}`, 0, false},
|
||||||
|
{"duration", `{"expiry":"1h"}`, time.Hour, false},
|
||||||
|
{"unparseable", `{"expiry":"nonsense"}`, 0, true},
|
||||||
|
{"zero duration", `{"expiry":"0s"}`, 0, true},
|
||||||
|
{"negative duration", `{"expiry":"-5h"}`, 0, true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
got, err := delivery.ExportParseArchiveExpiry(tc.in)
|
||||||
|
if tc.wantErr {
|
||||||
|
require.Error(t, err)
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, tc.want, got)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// seedDatabaseTargetDelivery seeds a pending delivery of an event
|
||||||
|
// to a database target and returns the in-memory delivery the
|
||||||
|
// target handler is invoked with.
|
||||||
|
func seedDatabaseTargetDelivery(
|
||||||
|
t *testing.T,
|
||||||
|
webhookDB *gorm.DB,
|
||||||
|
event database.Event,
|
||||||
|
tgt *database.Target,
|
||||||
|
) *database.Delivery {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
dlv := seedDelivery(
|
||||||
|
t, webhookDB, event.ID, tgt.ID,
|
||||||
|
database.DeliveryStatusPending,
|
||||||
|
)
|
||||||
|
|
||||||
|
d := &database.Delivery{
|
||||||
|
EventID: event.ID,
|
||||||
|
TargetID: tgt.ID,
|
||||||
|
Status: database.DeliveryStatusPending,
|
||||||
|
Event: event,
|
||||||
|
Target: database.Target{
|
||||||
|
Name: tgt.Name,
|
||||||
|
Type: database.TargetTypeDatabase,
|
||||||
|
Config: tgt.Config,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
d.ID = dlv.ID
|
||||||
|
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
// TestDeliverDatabase_ArchiveFailureFailsDelivery verifies that
|
// TestDeliverDatabase_ArchiveFailureFailsDelivery verifies that
|
||||||
// an archive error (here: an unparseable expiry in the target
|
// an archive error (here: an unparseable expiry in the target
|
||||||
// config) fails the delivery loudly: the attempt is recorded as
|
// config) fails the delivery loudly: the attempt is recorded as
|
||||||
@@ -304,3 +423,348 @@ func TestDeliverDatabase_ArchiveFailureFailsDelivery(
|
|||||||
"no archive file should exist for a failed config",
|
"no archive file should exist for a failed config",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidateArchiveExpiry(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
valid := []string{"", "never", "1h", "720h", "30m"}
|
||||||
|
for _, in := range valid {
|
||||||
|
require.NoError(t,
|
||||||
|
delivery.ValidateArchiveExpiry(in),
|
||||||
|
"expiry %q should be accepted", in,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
invalid := []string{"nonsense", "7d", "-5h", "0s", "0"}
|
||||||
|
for _, in := range invalid {
|
||||||
|
require.Error(t,
|
||||||
|
delivery.ValidateArchiveExpiry(in),
|
||||||
|
"expiry %q should be rejected", in,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestArchiveFileName pins the archive file name and the rules
|
||||||
|
// that make a webhook or target name safe to put in it.
|
||||||
|
func TestArchiveFileName(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
const id = "3f2a1c9e-8d4b-4c1a-9e2f-0a1b2c3d4e5f"
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
webhook string
|
||||||
|
target string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
"plain names", "orders", "archive",
|
||||||
|
"archive-orders-archive-" + id + ".db",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"lowercased", "Orders", "Main Archive",
|
||||||
|
"archive-orders-main-archive-" + id + ".db",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"a run of other characters is one dash",
|
||||||
|
`a /\..b`, "c__--d",
|
||||||
|
"archive-a-b-c-d-" + id + ".db",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"no dash at either end", " --orders!! ", "(archive)",
|
||||||
|
"archive-orders-archive-" + id + ".db",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path separators", "../../etc/passwd", "a/b",
|
||||||
|
"archive-etc-passwd-a-b-" + id + ".db",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"letters outside ASCII are dropped",
|
||||||
|
"Bestellungen Größe", "café",
|
||||||
|
"archive-bestellungen-gr-e-caf-" + id + ".db",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"nothing left is unnamed", "", "!!!",
|
||||||
|
"archive-unnamed-unnamed-" + id + ".db",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cut to 40 characters", strings.Repeat("a", 50), "x",
|
||||||
|
"archive-" + strings.Repeat("a", 40) + "-x-" + id + ".db",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"no dash left by the cut",
|
||||||
|
strings.Repeat("a", 39) + " b", "x",
|
||||||
|
"archive-" + strings.Repeat("a", 39) + "-x-" + id + ".db",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
assert.Equal(
|
||||||
|
t, tc.want,
|
||||||
|
delivery.ArchiveFileName(tc.webhook, tc.target, id),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestDeliverDatabase_EachTargetHasItsOwnArchive proves two
|
||||||
|
// database targets of one webhook archive into separate files.
|
||||||
|
func TestDeliverDatabase_EachTargetHasItsOwnArchive(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupArchiveTest(t)
|
||||||
|
first := env.seedDatabaseTarget(t, "")
|
||||||
|
second := env.addDatabaseTarget(t, first.WebhookID, "")
|
||||||
|
|
||||||
|
webhookDB := testWebhookDB(t)
|
||||||
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
|
|
||||||
|
for _, tgt := range []*database.Target{first, second} {
|
||||||
|
env.eng.ExportDeliverDatabase(
|
||||||
|
webhookDB,
|
||||||
|
seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NotEqual(
|
||||||
|
t, env.archivePath(first), env.archivePath(second),
|
||||||
|
)
|
||||||
|
assert.Equal(
|
||||||
|
t, []string{event.ID},
|
||||||
|
archivedEventIDs(t, env.archivePath(first)),
|
||||||
|
)
|
||||||
|
assert.Equal(
|
||||||
|
t, []string{event.ID},
|
||||||
|
archivedEventIDs(t, env.archivePath(second)),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRename_MovesTheFile proves a rename moves the archive, rows
|
||||||
|
// and all, and that later writes go to the new name.
|
||||||
|
func TestRename_MovesTheFile(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupArchiveTest(t)
|
||||||
|
tgt := env.seedDatabaseTarget(t, "")
|
||||||
|
oldPath := env.archivePath(tgt)
|
||||||
|
|
||||||
|
webhookDB := testWebhookDB(t)
|
||||||
|
first := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
|
env.eng.ExportDeliverDatabase(
|
||||||
|
webhookDB, seedDatabaseTargetDelivery(t, webhookDB, first, tgt),
|
||||||
|
)
|
||||||
|
require.FileExists(t, oldPath)
|
||||||
|
|
||||||
|
require.NoError(
|
||||||
|
t, env.eng.Rename(tgt.ID, "Orders", "Long Term"),
|
||||||
|
)
|
||||||
|
|
||||||
|
newPath := filepath.Join(
|
||||||
|
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert.NoFileExists(t, oldPath)
|
||||||
|
assert.Equal(t, []string{first.ID}, archivedEventIDs(t, newPath))
|
||||||
|
|
||||||
|
second := seedEvent(t, webhookDB, `{"n":2}`)
|
||||||
|
env.eng.ExportDeliverDatabase(
|
||||||
|
webhookDB,
|
||||||
|
seedDatabaseTargetDelivery(t, webhookDB, second, tgt),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert.ElementsMatch(
|
||||||
|
t, []string{first.ID, second.ID},
|
||||||
|
archivedEventIDs(t, newPath),
|
||||||
|
)
|
||||||
|
assert.NoFileExists(
|
||||||
|
t, oldPath, "a write after the rename must use the new name",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRename_NeverReplacesAFile plants a file at the new name, once
|
||||||
|
// the .db alone, once a lone -wal and once a lone -shm, and proves
|
||||||
|
// each time that the rename is refused, the planted file survives,
|
||||||
|
// and the archive keeps its name and its rows.
|
||||||
|
func TestRename_NeverReplacesAFile(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, suffix := range archiveFileSuffixes() {
|
||||||
|
t.Run("planted .db"+suffix, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupArchiveTest(t)
|
||||||
|
tgt := env.seedDatabaseTarget(t, "")
|
||||||
|
oldPath := env.archivePath(tgt)
|
||||||
|
|
||||||
|
webhookDB := testWebhookDB(t)
|
||||||
|
first := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
|
env.eng.ExportDeliverDatabase(
|
||||||
|
webhookDB,
|
||||||
|
seedDatabaseTargetDelivery(t, webhookDB, first, tgt),
|
||||||
|
)
|
||||||
|
|
||||||
|
newPath := filepath.Join(
|
||||||
|
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
||||||
|
)
|
||||||
|
plantedPath := newPath + suffix
|
||||||
|
require.NoError(
|
||||||
|
t, os.WriteFile(plantedPath, []byte("planted"), 0o600),
|
||||||
|
)
|
||||||
|
|
||||||
|
require.ErrorIs(
|
||||||
|
t, env.eng.Rename(tgt.ID, "Orders", "Long Term"),
|
||||||
|
delivery.ErrArchiveNameTaken,
|
||||||
|
)
|
||||||
|
|
||||||
|
//nolint:gosec // reads the file the test planted under t.TempDir()
|
||||||
|
planted, err := os.ReadFile(plantedPath)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, "planted", string(planted))
|
||||||
|
|
||||||
|
second := seedEvent(t, webhookDB, `{"n":2}`)
|
||||||
|
env.eng.ExportDeliverDatabase(
|
||||||
|
webhookDB,
|
||||||
|
seedDatabaseTargetDelivery(t, webhookDB, second, tgt),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert.ElementsMatch(
|
||||||
|
t, []string{first.ID, second.ID},
|
||||||
|
archivedEventIDs(t, oldPath),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRename_BeforeTheNameIsSaved covers the order the handlers
|
||||||
|
// use: they rename before they save the new name, so a delivery in
|
||||||
|
// between must write under the new name although the main database
|
||||||
|
// still has the old one. It also shows that renaming an archive that
|
||||||
|
// does not exist yet is not an error.
|
||||||
|
func TestRename_BeforeTheNameIsSaved(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupArchiveTest(t)
|
||||||
|
tgt := env.seedDatabaseTarget(t, "")
|
||||||
|
|
||||||
|
require.NoError(
|
||||||
|
t, env.eng.Rename(tgt.ID, "Orders", "Archive"),
|
||||||
|
)
|
||||||
|
|
||||||
|
webhookDB := testWebhookDB(t)
|
||||||
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
|
env.eng.ExportDeliverDatabase(
|
||||||
|
webhookDB, seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert.FileExists(
|
||||||
|
t,
|
||||||
|
filepath.Join(
|
||||||
|
env.dataDir, "archive-orders-archive-"+tgt.ID+".db",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert.NoFileExists(t, env.archivePath(tgt))
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestArchiveWriter_RenameMovesSidecars proves a rename carries
|
||||||
|
// the -wal and -shm a crash can leave beside an archive no handle
|
||||||
|
// has opened since. SQLite finds them by name, so a -wal left
|
||||||
|
// behind would lose the transactions it holds.
|
||||||
|
func TestArchiveWriter_RenameMovesSidecars(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
oldPath := filepath.Join(dir, "archive-old.db")
|
||||||
|
newPath := filepath.Join(dir, "archive-new.db")
|
||||||
|
|
||||||
|
for _, suffix := range archiveFileSuffixes() {
|
||||||
|
require.NoError(
|
||||||
|
t, os.WriteFile(oldPath+suffix, []byte(suffix), 0o600),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
w := delivery.NewExportArchiveWriter(
|
||||||
|
oldPath, archiveTestLogger(), 0,
|
||||||
|
)
|
||||||
|
|
||||||
|
require.NoError(t, w.Rename("archive-new.db"))
|
||||||
|
|
||||||
|
for _, suffix := range archiveFileSuffixes() {
|
||||||
|
assert.NoFileExists(t, oldPath+suffix)
|
||||||
|
assert.FileExists(t, newPath+suffix)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Equal(t, newPath, w.Path())
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestArchiveWriter_RenameMovesBackOnFailure makes the -wal fail to
|
||||||
|
// move after the .db has moved, and proves the .db is moved back, so
|
||||||
|
// the archive is never split across two names. The new name is 255
|
||||||
|
// bytes, the longest a file name may be, so the .db can take it but
|
||||||
|
// the -wal, four bytes longer, cannot.
|
||||||
|
func TestArchiveWriter_RenameMovesBackOnFailure(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
oldPath := filepath.Join(dir, "archive-old.db")
|
||||||
|
newName := strings.Repeat("a", 252) + ".db"
|
||||||
|
|
||||||
|
for _, suffix := range archiveFileSuffixes() {
|
||||||
|
require.NoError(
|
||||||
|
t, os.WriteFile(oldPath+suffix, []byte(suffix), 0o600),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
w := delivery.NewExportArchiveWriter(
|
||||||
|
oldPath, archiveTestLogger(), 0,
|
||||||
|
)
|
||||||
|
|
||||||
|
require.Error(t, w.Rename(newName))
|
||||||
|
|
||||||
|
for _, suffix := range archiveFileSuffixes() {
|
||||||
|
assert.FileExists(t, oldPath+suffix)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.NoFileExists(t, filepath.Join(dir, newName))
|
||||||
|
assert.Equal(t, oldPath, w.Path())
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestArchiveWriter_RenameMovesBackEveryFile renames a target with a
|
||||||
|
// file without a period and a file for a month, and proves that when
|
||||||
|
// the month's file fails to move, the file already moved is moved back:
|
||||||
|
// both files are under the old name with their rows, and nothing is
|
||||||
|
// under the new name. The new name is 251 bytes, so the file without a
|
||||||
|
// period, with its -wal and -shm, can take it, but the month's file,
|
||||||
|
// eight bytes longer, cannot.
|
||||||
|
func TestArchiveWriter_RenameMovesBackEveryFile(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
oldPath := filepath.Join(dir, "archive-old.db")
|
||||||
|
monthPath := filepath.Join(dir, "archive-old-2026-03.db")
|
||||||
|
w := delivery.NewExportArchiveWriter(oldPath, archiveTestLogger(), 0)
|
||||||
|
|
||||||
|
require.NoError(t, w.WritePeriod(
|
||||||
|
delivery.ExportArchivedEvent{EventID: "in-none"}, 0, "",
|
||||||
|
))
|
||||||
|
require.NoError(t, w.WritePeriod(
|
||||||
|
delivery.ExportArchivedEvent{EventID: "in-month"}, 0, "2026-03",
|
||||||
|
))
|
||||||
|
|
||||||
|
require.Error(t, w.Rename(strings.Repeat("a", 248)+".db"))
|
||||||
|
|
||||||
|
assert.Equal(t, []string{"in-none"}, archivedEventIDs(t, oldPath))
|
||||||
|
assert.Equal(t, []string{"in-month"}, archivedEventIDs(t, monthPath))
|
||||||
|
|
||||||
|
entries, err := os.ReadDir(dir)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
for _, entry := range entries {
|
||||||
|
assert.True(t, strings.HasPrefix(entry.Name(), "archive-old"),
|
||||||
|
"%s is not under the old name", entry.Name())
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Equal(t, oldPath, w.Path())
|
||||||
|
}
|
||||||
|
|||||||
@@ -189,7 +189,6 @@ func TestNewTargetConfigForm(t *testing.T) {
|
|||||||
Config: `{"url":"https://example.com/h",` +
|
Config: `{"url":"https://example.com/h",` +
|
||||||
`"headers":{"Authorization":"Bearer abc"},` +
|
`"headers":{"Authorization":"Bearer abc"},` +
|
||||||
`"timeout":9}`,
|
`"timeout":9}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, "https://example.com/h", form.URL)
|
assert.Equal(t, "https://example.com/h", form.URL)
|
||||||
@@ -199,7 +198,6 @@ func TestNewTargetConfigForm(t *testing.T) {
|
|||||||
form, err = delivery.NewTargetConfigForm(&database.Target{
|
form, err = delivery.NewTargetConfigForm(&database.Target{
|
||||||
Type: database.TargetTypeSlack,
|
Type: database.TargetTypeSlack,
|
||||||
Config: `{"webhookUrl":"https://hooks.example/s"}`,
|
Config: `{"webhookUrl":"https://hooks.example/s"}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, "https://hooks.example/s", form.URL)
|
assert.Equal(t, "https://hooks.example/s", form.URL)
|
||||||
@@ -244,9 +242,9 @@ func TestNewTargetConfigForm_UnreadableConfigErrors(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
cases := []*database.Target{
|
cases := []*database.Target{
|
||||||
{Type: database.TargetTypeHTTP, Config: "not json", MaxAttempts: 1},
|
{Type: database.TargetTypeHTTP, Config: "not json"},
|
||||||
{Type: database.TargetTypeHTTP, Config: `{}`, MaxAttempts: 1},
|
{Type: database.TargetTypeHTTP, Config: `{}`},
|
||||||
{Type: database.TargetTypeSlack, Config: "", MaxAttempts: 1},
|
{Type: database.TargetTypeSlack, Config: ""},
|
||||||
{
|
{
|
||||||
Type: database.TargetTypeDatabase,
|
Type: database.TargetTypeDatabase,
|
||||||
Config: `{"expiry":"soon"}`,
|
Config: `{"expiry":"soon"}`,
|
||||||
|
|||||||
@@ -53,72 +53,34 @@ type httpCore struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// deliver runs one delivery attempt through the retry core.
|
// deliver runs one delivery attempt through the retry core.
|
||||||
// A maxAttempts of 1 is a single attempt, recorded without
|
// A maxRetries of 0 is fire-and-forget: a single attempt is
|
||||||
// consulting a circuit breaker. A higher maxAttempts gates the
|
// recorded and no circuit breaker is consulted. A positive
|
||||||
// attempt on the circuit breaker and schedules a backed-off retry
|
// maxRetries gates the attempt on the circuit breaker and
|
||||||
// on failure, until maxAttempts attempts have been made.
|
// schedules a backed-off retry on failure.
|
||||||
//
|
|
||||||
// ctx is the context attempt sends its request with. Stopping the
|
|
||||||
// engine cancels it; see interruptedByStop.
|
|
||||||
func (c *httpCore) deliver(
|
func (c *httpCore) deliver(
|
||||||
ctx context.Context,
|
|
||||||
webhookDB *gorm.DB,
|
webhookDB *gorm.DB,
|
||||||
d *database.Delivery,
|
d *database.Delivery,
|
||||||
task *Task,
|
task *Task,
|
||||||
sched Scheduler,
|
sched Scheduler,
|
||||||
maxAttempts int,
|
maxRetries int,
|
||||||
attempt func() attemptResult,
|
attempt func() attemptResult,
|
||||||
) {
|
) {
|
||||||
if maxAttempts == 1 {
|
if maxRetries == 0 {
|
||||||
c.fireAndForget(ctx, webhookDB, d, attempt())
|
c.fireAndForget(webhookDB, d, attempt())
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
c.withRetry(
|
c.withRetry(
|
||||||
ctx, webhookDB, d, task, sched, maxAttempts, attempt,
|
webhookDB, d, task, sched, maxRetries, attempt,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// interruptedByStop reports whether an attempt failed because the
|
|
||||||
// engine is stopping, which cancels the context its request is made
|
|
||||||
// with. Such an attempt is not recorded: no DeliveryResult, no circuit
|
|
||||||
// breaker outcome, no status change, no count in the attempt metrics.
|
|
||||||
// The delivery stays pending or retrying, and startup recovery sends
|
|
||||||
// it again on the next start.
|
|
||||||
//
|
|
||||||
// A target's own timeout leaves that context alone, so its attempt is
|
|
||||||
// still recorded as the failure it is.
|
|
||||||
func (c *httpCore) interruptedByStop(
|
|
||||||
ctx context.Context,
|
|
||||||
d *database.Delivery,
|
|
||||||
res attemptResult,
|
|
||||||
) bool {
|
|
||||||
if res.success || ctx.Err() == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
c.eng.log.Warn(
|
|
||||||
"delivery attempt interrupted by shutdown; not recorded, "+
|
|
||||||
"it is sent again on the next start",
|
|
||||||
"delivery_id", d.ID,
|
|
||||||
"event_id", d.EventID,
|
|
||||||
"target_id", d.TargetID,
|
|
||||||
)
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *httpCore) fireAndForget(
|
func (c *httpCore) fireAndForget(
|
||||||
ctx context.Context,
|
|
||||||
webhookDB *gorm.DB,
|
webhookDB *gorm.DB,
|
||||||
d *database.Delivery,
|
d *database.Delivery,
|
||||||
res attemptResult,
|
res attemptResult,
|
||||||
) {
|
) {
|
||||||
if c.interruptedByStop(ctx, d, res) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
c.eng.observeAttempt(d.Target.Type, res.elapsed())
|
c.eng.observeAttempt(d.Target.Type, res.elapsed())
|
||||||
|
|
||||||
err := c.eng.recordResult(
|
err := c.eng.recordResult(
|
||||||
@@ -148,12 +110,11 @@ func (c *httpCore) fireAndForget(
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *httpCore) withRetry(
|
func (c *httpCore) withRetry(
|
||||||
ctx context.Context,
|
|
||||||
webhookDB *gorm.DB,
|
webhookDB *gorm.DB,
|
||||||
d *database.Delivery,
|
d *database.Delivery,
|
||||||
task *Task,
|
task *Task,
|
||||||
sched Scheduler,
|
sched Scheduler,
|
||||||
maxAttempts int,
|
maxRetries int,
|
||||||
attempt func() attemptResult,
|
attempt func() attemptResult,
|
||||||
) {
|
) {
|
||||||
cb := c.getCircuitBreaker(task.TargetID)
|
cb := c.getCircuitBreaker(task.TargetID)
|
||||||
@@ -170,10 +131,6 @@ func (c *httpCore) withRetry(
|
|||||||
|
|
||||||
res := attempt()
|
res := attempt()
|
||||||
|
|
||||||
if c.interruptedByStop(ctx, d, res) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
c.eng.observeAttempt(d.Target.Type, res.elapsed())
|
c.eng.observeAttempt(d.Target.Type, res.elapsed())
|
||||||
|
|
||||||
err := c.eng.recordResult(
|
err := c.eng.recordResult(
|
||||||
@@ -205,7 +162,7 @@ func (c *httpCore) withRetry(
|
|||||||
cb.RecordFailure()
|
cb.RecordFailure()
|
||||||
|
|
||||||
c.handleRetry(
|
c.handleRetry(
|
||||||
webhookDB, d, task, sched, maxAttempts, attemptNum,
|
webhookDB, d, task, sched, maxRetries, attemptNum,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -266,10 +223,10 @@ func (c *httpCore) handleRetry(
|
|||||||
d *database.Delivery,
|
d *database.Delivery,
|
||||||
task *Task,
|
task *Task,
|
||||||
sched Scheduler,
|
sched Scheduler,
|
||||||
maxAttempts int,
|
maxRetries int,
|
||||||
attemptNum int,
|
attemptNum int,
|
||||||
) {
|
) {
|
||||||
if attemptNum >= maxAttempts {
|
if attemptNum >= maxRetries {
|
||||||
c.eng.settleStatus(
|
c.eng.settleStatus(
|
||||||
webhookDB, d, d.Target.Type,
|
webhookDB, d, d.Target.Type,
|
||||||
database.DeliveryStatusFailed,
|
database.DeliveryStatusFailed,
|
||||||
@@ -438,8 +395,8 @@ func (t *httpTarget) Deliver(
|
|||||||
}
|
}
|
||||||
|
|
||||||
t.deliver(
|
t.deliver(
|
||||||
ctx, webhookDB, d, task, sched,
|
webhookDB, d, task, sched,
|
||||||
d.Target.MaxAttempts, attempt,
|
d.Target.MaxRetries, attempt,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -47,7 +47,7 @@ func deliverWithQuery(t *testing.T, cfg string) database.DeliveryResult {
|
|||||||
database.DeliveryStatusPending,
|
database.DeliveryStatusPending,
|
||||||
)
|
)
|
||||||
task := iTask(
|
task := iTask(
|
||||||
d, event, s.WebhookID, d.TargetID, "query", cfg, 1, 1, &event.Body,
|
d, event, s.WebhookID, d.TargetID, "query", cfg, 0, 1, &event.Body,
|
||||||
)
|
)
|
||||||
task.RawQuery = eventQuery
|
task.RawQuery = eventQuery
|
||||||
|
|
||||||
@@ -146,7 +146,6 @@ func TestDeliverHTTP_ForwardedQueryKeepsTheTargetURLMasked(t *testing.T) {
|
|||||||
target := &database.Target{
|
target := &database.Target{
|
||||||
Type: database.TargetTypeHTTP,
|
Type: database.TargetTypeHTTP,
|
||||||
Config: httpTargetConfig(t, ts.URL+"/in?token="+secret, true),
|
Config: httpTargetConfig(t, ts.URL+"/in?token="+secret, true),
|
||||||
MaxAttempts: 1,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
deliverWithQuery(t, target.Config)
|
deliverWithQuery(t, target.Config)
|
||||||
|
|||||||
@@ -24,7 +24,6 @@ func TestRedactor_RemovesSlackWebhookURL(t *testing.T) {
|
|||||||
r := delivery.NewRedactor(&database.Target{
|
r := delivery.NewRedactor(&database.Target{
|
||||||
Type: database.TargetTypeSlack,
|
Type: database.TargetTypeSlack,
|
||||||
Config: `{"webhookUrl":"` + redactWebhookURL + `"}`,
|
Config: `{"webhookUrl":"` + redactWebhookURL + `"}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
})
|
})
|
||||||
|
|
||||||
got := r.Redact("no_service for " + redactWebhookURL)
|
got := r.Redact("no_service for " + redactWebhookURL)
|
||||||
@@ -55,7 +54,6 @@ func TestRedactor_RemovesSecretSeveredByACut(t *testing.T) {
|
|||||||
r := delivery.NewRedactor(&database.Target{
|
r := delivery.NewRedactor(&database.Target{
|
||||||
Type: database.TargetTypeSlack,
|
Type: database.TargetTypeSlack,
|
||||||
Config: `{"webhookUrl":"` + redactWebhookURL + `"}`,
|
Config: `{"webhookUrl":"` + redactWebhookURL + `"}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
})
|
})
|
||||||
|
|
||||||
// Every cut position inside the credential, not just a
|
// Every cut position inside the credential, not just a
|
||||||
@@ -98,7 +96,6 @@ func TestRedactor_RedactsCredentialShapedHeaderValues(
|
|||||||
`"X-Credential":"HHHHHHHHHHHH",` +
|
`"X-Credential":"HHHHHHHHHHHH",` +
|
||||||
`"Accept":"application/json",` +
|
`"Accept":"application/json",` +
|
||||||
`"User-Agent":"webhooker/1.0"}}`,
|
`"User-Agent":"webhooker/1.0"}}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
})
|
})
|
||||||
|
|
||||||
for _, secret := range []string{
|
for _, secret := range []string{
|
||||||
@@ -139,7 +136,6 @@ func TestRedactor_IgnoresVeryShortHeaderValues(t *testing.T) {
|
|||||||
Type: database.TargetTypeHTTP,
|
Type: database.TargetTypeHTTP,
|
||||||
Config: `{"url":"https://example.com/in",` +
|
Config: `{"url":"https://example.com/in",` +
|
||||||
`"headers":{"X-Api-Key":"ab"}}`,
|
`"headers":{"X-Api-Key":"ab"}}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
})
|
})
|
||||||
|
|
||||||
const response = "rabbit"
|
const response = "rabbit"
|
||||||
@@ -156,7 +152,6 @@ func TestRedactor_RemovesBarePath(t *testing.T) {
|
|||||||
r := delivery.NewRedactor(&database.Target{
|
r := delivery.NewRedactor(&database.Target{
|
||||||
Type: database.TargetTypeSlack,
|
Type: database.TargetTypeSlack,
|
||||||
Config: `{"webhookUrl":"` + redactWebhookURL + `"}`,
|
Config: `{"webhookUrl":"` + redactWebhookURL + `"}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
})
|
})
|
||||||
|
|
||||||
got := r.Redact("POST " + redactSecretPath + " 404")
|
got := r.Redact("POST " + redactSecretPath + " 404")
|
||||||
@@ -190,7 +185,6 @@ func TestRedactor_RemovesHTTPURLQueryAndUserinfo(t *testing.T) {
|
|||||||
r := delivery.NewRedactor(&database.Target{
|
r := delivery.NewRedactor(&database.Target{
|
||||||
Type: database.TargetTypeHTTP,
|
Type: database.TargetTypeHTTP,
|
||||||
Config: `{"url":"` + raw + `"}`,
|
Config: `{"url":"` + raw + `"}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
})
|
})
|
||||||
|
|
||||||
for _, echoed := range []string{
|
for _, echoed := range []string{
|
||||||
@@ -240,7 +234,6 @@ func TestRedactor_RemovesEchoedQueryOfURLWithoutPath(t *testing.T) {
|
|||||||
r := delivery.NewRedactor(&database.Target{
|
r := delivery.NewRedactor(&database.Target{
|
||||||
Type: database.TargetTypeHTTP,
|
Type: database.TargetTypeHTTP,
|
||||||
Config: `{"url":"` + dest + `"}`,
|
Config: `{"url":"` + dest + `"}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
})
|
})
|
||||||
|
|
||||||
for echoed, want := range echoes {
|
for echoed, want := range echoes {
|
||||||
@@ -262,7 +255,6 @@ func TestRedactor_LeavesUnrelatedTextAlone(t *testing.T) {
|
|||||||
r := delivery.NewRedactor(&database.Target{
|
r := delivery.NewRedactor(&database.Target{
|
||||||
Type: database.TargetTypeSlack,
|
Type: database.TargetTypeSlack,
|
||||||
Config: `{"webhookUrl":"` + redactWebhookURL + `"}`,
|
Config: `{"webhookUrl":"` + redactWebhookURL + `"}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
})
|
})
|
||||||
|
|
||||||
assert.Equal(t, response, r.Redact(response))
|
assert.Equal(t, response, r.Redact(response))
|
||||||
@@ -289,7 +281,6 @@ func TestRedactor_EmptyUserinfoDoesNotShredTheBody(t *testing.T) {
|
|||||||
r := delivery.NewRedactor(&database.Target{
|
r := delivery.NewRedactor(&database.Target{
|
||||||
Type: database.TargetTypeHTTP,
|
Type: database.TargetTypeHTTP,
|
||||||
Config: `{"url":"` + dest + `"}`,
|
Config: `{"url":"` + dest + `"}`,
|
||||||
MaxAttempts: 1,
|
|
||||||
})
|
})
|
||||||
|
|
||||||
const body = "ok=false error=channel_not_found"
|
const body = "ok=false error=channel_not_found"
|
||||||
@@ -323,8 +314,8 @@ func TestRedactor_ZeroValueAndConfiglessTargets(t *testing.T) {
|
|||||||
for _, tgt := range []database.Target{
|
for _, tgt := range []database.Target{
|
||||||
{Type: database.TargetTypeLog},
|
{Type: database.TargetTypeLog},
|
||||||
{Type: database.TargetTypeDatabase},
|
{Type: database.TargetTypeDatabase},
|
||||||
{Type: database.TargetTypeSlack, Config: "not json", MaxAttempts: 1},
|
{Type: database.TargetTypeSlack, Config: "not json"},
|
||||||
{Type: database.TargetTypeHTTP, Config: "", MaxAttempts: 1},
|
{Type: database.TargetTypeHTTP, Config: ""},
|
||||||
} {
|
} {
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, text,
|
t, text,
|
||||||
|
|||||||
@@ -30,8 +30,9 @@ type SlackTargetConfig struct {
|
|||||||
// slackTarget delivers events to Slack incoming webhooks. It
|
// slackTarget delivers events to Slack incoming webhooks. It
|
||||||
// formats the event into a Slack message and posts it as
|
// formats the event into a Slack message and posts it as
|
||||||
// JSON. It shares the retry core with the HTTP target: a
|
// JSON. It shares the retry core with the HTTP target: a
|
||||||
// MaxAttempts of 1 is a single attempt, while a higher one
|
// MaxRetries of 0 stays single-attempt fire-and-forget
|
||||||
// adds backoff and circuit breaking.
|
// (preserving existing Slack targets), while a positive
|
||||||
|
// MaxRetries adds backoff and circuit breaking.
|
||||||
type slackTarget struct {
|
type slackTarget struct {
|
||||||
*httpCore
|
*httpCore
|
||||||
|
|
||||||
@@ -81,8 +82,8 @@ func (t *slackTarget) Deliver(
|
|||||||
}
|
}
|
||||||
|
|
||||||
t.deliver(
|
t.deliver(
|
||||||
ctx, webhookDB, d, task, sched,
|
webhookDB, d, task, sched,
|
||||||
d.Target.MaxAttempts, attempt,
|
d.Target.MaxRetries, attempt,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -743,7 +743,7 @@ func TestSendRecoveredDeliveries_TargetMissingFromMap(
|
|||||||
|
|
||||||
iCreateTarget(
|
iCreateTarget(
|
||||||
t, s.MainDB, targetID, s.WebhookID, "found-on-lookup",
|
t, s.MainDB, targetID, s.WebhookID, "found-on-lookup",
|
||||||
database.TargetTypeLog, "", 1,
|
database.TargetTypeLog, "", 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
event := iSeedEvent(
|
event := iSeedEvent(
|
||||||
@@ -790,7 +790,7 @@ func TestRecoverPending_TargetUnreadable_LeavesDeliveryAlone(
|
|||||||
|
|
||||||
iCreateTarget(
|
iCreateTarget(
|
||||||
t, s.MainDB, targetID, s.WebhookID, "healthy",
|
t, s.MainDB, targetID, s.WebhookID, "healthy",
|
||||||
database.TargetTypeLog, "", 1,
|
database.TargetTypeLog, "", 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
event := iSeedEvent(
|
event := iSeedEvent(
|
||||||
|
|||||||
@@ -146,7 +146,7 @@ func openDB(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// assertBounded holds every line the adapter wrote to the stated
|
// assertBounded holds every line the adapter wrote to the stated
|
||||||
// ceiling and proves no line carried a whole client value.
|
// ceiling and proves each was cut rather than merely short.
|
||||||
func assertBounded(t *testing.T, out string) {
|
func assertBounded(t *testing.T, out string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
|||||||
@@ -12,8 +12,8 @@ import (
|
|||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// rotationNone is the archive rotation that keeps one file.
|
// rotationNone is the archive rotation that keeps one file.
|
||||||
@@ -181,7 +181,7 @@ func TestArchiveFileView_Rotated(t *testing.T) {
|
|||||||
`{"rotation":"daily"}`,
|
`{"rotation":"daily"}`,
|
||||||
)
|
)
|
||||||
|
|
||||||
path := archive.Path(env.dbMgr, &webhook, target)
|
path := delivery.ArchivePath(env.dbMgr, &webhook, target)
|
||||||
stem := strings.TrimSuffix(path, ".db")
|
stem := strings.TrimSuffix(path, ".db")
|
||||||
written := time.Date(2026, 10, 2, 9, 0, 0, 0, time.UTC)
|
written := time.Date(2026, 10, 2, 9, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
|
|||||||
@@ -308,7 +308,7 @@ func createReplayDelivery(
|
|||||||
TargetName: target.Name,
|
TargetName: target.Name,
|
||||||
TargetType: target.Type,
|
TargetType: target.Type,
|
||||||
TargetConfig: target.Config,
|
TargetConfig: target.Config,
|
||||||
MaxAttempts: target.MaxAttempts,
|
MaxRetries: target.MaxRetries,
|
||||||
Method: event.Method,
|
Method: event.Method,
|
||||||
RawQuery: event.RawQuery,
|
RawQuery: event.RawQuery,
|
||||||
Headers: event.Headers,
|
Headers: event.Headers,
|
||||||
|
|||||||
@@ -158,7 +158,8 @@ func TestHandleSourceLogs_RendersFailedAttempt(t *testing.T) {
|
|||||||
// TestHandleSourceLogs_EscapesResponseBody proves the
|
// TestHandleSourceLogs_EscapesResponseBody proves the
|
||||||
// response body is treated as the untrusted remote content it
|
// response body is treated as the untrusted remote content it
|
||||||
// is. The remote chooses these bytes and the page is rendered
|
// is. The remote chooses these bytes and the page is rendered
|
||||||
// inside the operator's authenticated origin.
|
// inside the operator's authenticated origin, where the
|
||||||
|
// application's own CSP allows inline script from 'self'.
|
||||||
func TestHandleSourceLogs_EscapesResponseBody(t *testing.T) {
|
func TestHandleSourceLogs_EscapesResponseBody(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -35,10 +35,11 @@ const eventBodyQuery = "SELECT cast(body as blob) " +
|
|||||||
// response is deliberately not renderable. Content-Disposition
|
// response is deliberately not renderable. Content-Disposition
|
||||||
// makes the browser download rather than display it, and the
|
// makes the browser download rather than display it, and the
|
||||||
// octet-stream type plus nosniff stop it being interpreted as
|
// octet-stream type plus nosniff stop it being interpreted as
|
||||||
// HTML or script. Without those a stored payload would be shown
|
// HTML or script. Without those a stored payload would execute
|
||||||
// as a page of the operator's own origin. The application's CSP
|
// as the logged-in operator. The application's CSP does not
|
||||||
// is a second defence behind them: it allows no inline script,
|
// help here: script-src allows 'unsafe-inline' from 'self', so
|
||||||
// so such a page could not run script of its own.
|
// a document served from this origin could run its own inline
|
||||||
|
// script.
|
||||||
func (h *Handlers) HandleEventBodyDownload() http.HandlerFunc {
|
func (h *Handlers) HandleEventBodyDownload() http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
webhook, ok := h.ownedWebhook(w, r)
|
webhook, ok := h.ownedWebhook(w, r)
|
||||||
|
|||||||
@@ -84,7 +84,7 @@ func TestHandleEventDetail_ShowsEventWholeWithDeliveries(t *testing.T) {
|
|||||||
assert.Contains(t, page, strconv.Itoa(len(body))+" bytes")
|
assert.Contains(t, page, strconv.Itoa(len(body))+" bytes")
|
||||||
assert.Contains(t, page, "{\n "pad": "xxx")
|
assert.Contains(t, page, "{\n "pad": "xxx")
|
||||||
assert.Contains(t, page, ""tail": ""+sentinel+""\n}")
|
assert.Contains(t, page, ""tail": ""+sentinel+""\n}")
|
||||||
assert.Contains(t, page, `break-all max-h-[32rem] overflow-y-auto"`)
|
assert.Contains(t, page, `style="max-height: 32rem; overflow-y: auto"`)
|
||||||
assert.NotContains(t, page, "Showing the first")
|
assert.NotContains(t, page, "Showing the first")
|
||||||
assert.Contains(t, page, target.Name)
|
assert.Contains(t, page, target.Name)
|
||||||
assert.Contains(t, page, ">failed</span>")
|
assert.Contains(t, page, ">failed</span>")
|
||||||
|
|||||||
@@ -496,7 +496,7 @@ const deliveryIDChunkSize = 500
|
|||||||
//
|
//
|
||||||
// Each response body is cut by SQLite rather than in Go, for
|
// Each response body is cut by SQLite rather than in Go, for
|
||||||
// the reason deliveryResultColumns gives. How many attempts a
|
// the reason deliveryResultColumns gives. How many attempts a
|
||||||
// delivery has is bounded by the target's MaxAttempts, which the
|
// delivery has is the target's MaxRetries, which the
|
||||||
// authenticated operator sets; how many of them reach the page
|
// authenticated operator sets; how many of them reach the page
|
||||||
// is bounded again by maxRenderedAttempts.
|
// is bounded again by maxRenderedAttempts.
|
||||||
func (h *Handlers) loadDeliveryResults(
|
func (h *Handlers) loadDeliveryResults(
|
||||||
|
|||||||
@@ -37,10 +37,10 @@ const MaxRenderedResponseBytesForTest = maxRenderedResponseBytes
|
|||||||
// per-delivery attempt ceiling to the handlers_test package.
|
// per-delivery attempt ceiling to the handlers_test package.
|
||||||
const MaxRenderedAttemptsForTest = maxRenderedAttempts
|
const MaxRenderedAttemptsForTest = maxRenderedAttempts
|
||||||
|
|
||||||
// MaxTargetAttemptsForTest exposes the target max_attempts ceiling to
|
// MaxTargetRetriesForTest exposes the target max_retries ceiling to
|
||||||
// the handlers_test package, so the tests assert against the constant
|
// the handlers_test package, so the tests assert against the constant
|
||||||
// the handlers enforce rather than a number copied beside it.
|
// the handlers enforce rather than a number copied beside it.
|
||||||
const MaxTargetAttemptsForTest = maxTargetAttempts
|
const MaxTargetRetriesForTest = maxTargetRetries
|
||||||
|
|
||||||
// EventDBLeftMsgForTest and SidecarLeftMsgForTest expose the two
|
// EventDBLeftMsgForTest and SidecarLeftMsgForTest expose the two
|
||||||
// messages the webhook delete handler logs when a file of the event
|
// messages the webhook delete handler logs when a file of the event
|
||||||
|
|||||||
@@ -374,7 +374,7 @@ func assertFloodBounded(t *testing.T, label, out string) {
|
|||||||
// this service's three gorm.Open sites: the main database behind the
|
// this service's three gorm.Open sites: the main database behind the
|
||||||
// two unauthenticated lookups, and the per-webhook database the
|
// two unauthenticated lookups, and the per-webhook database the
|
||||||
// WebhookDBManager opens. The third, the archive writer, is pinned in
|
// WebhookDBManager opens. The third, the archive writer, is pinned in
|
||||||
// internal/archive, where its type lives.
|
// internal/delivery, where its type lives.
|
||||||
//
|
//
|
||||||
// What each assertion is worth, since two of the three would pass
|
// What each assertion is worth, since two of the three would pass
|
||||||
// against a service that had never been fixed if the capture were set
|
// against a service that had never been fixed if the capture were set
|
||||||
|
|||||||
@@ -10,11 +10,11 @@ import (
|
|||||||
"html/template"
|
"html/template"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
@@ -65,7 +65,7 @@ type HandlersParams struct {
|
|||||||
Session *session.Session
|
Session *session.Session
|
||||||
Middleware *middleware.Middleware
|
Middleware *middleware.Middleware
|
||||||
Notifier delivery.Notifier
|
Notifier delivery.Notifier
|
||||||
Archives archive.Archives
|
Archives delivery.Archives
|
||||||
CircuitBreakers delivery.CircuitBreakers
|
CircuitBreakers delivery.CircuitBreakers
|
||||||
SSRFGuard *delivery.Guard
|
SSRFGuard *delivery.Guard
|
||||||
Metrics *metrics.Set
|
Metrics *metrics.Set
|
||||||
@@ -83,7 +83,7 @@ type Handlers struct {
|
|||||||
session *session.Session
|
session *session.Session
|
||||||
mw *middleware.Middleware
|
mw *middleware.Middleware
|
||||||
notifier delivery.Notifier
|
notifier delivery.Notifier
|
||||||
archives archive.Archives
|
archives delivery.Archives
|
||||||
breakers delivery.CircuitBreakers
|
breakers delivery.CircuitBreakers
|
||||||
mtr *metrics.Set
|
mtr *metrics.Set
|
||||||
templates map[string]*template.Template
|
templates map[string]*template.Template
|
||||||
@@ -93,6 +93,17 @@ type Handlers struct {
|
|||||||
// is one delivery will actually attempt.
|
// is one delivery will actually attempt.
|
||||||
ssrf *delivery.Guard
|
ssrf *delivery.Guard
|
||||||
|
|
||||||
|
// renameMu makes the webhook edit, the target edit and target
|
||||||
|
// creation run one at a time, each held from loading the stored
|
||||||
|
// names through the archive rename, the save and any move back.
|
||||||
|
// Interleaved, one could rename an archive between another's
|
||||||
|
// rename and save, leaving the file named for one edit and the
|
||||||
|
// stored names from the other. An archive download holds it while
|
||||||
|
// it reads the stored names and lists the files they give, and
|
||||||
|
// again for each file while it finds the file under the names
|
||||||
|
// stored then and opens it.
|
||||||
|
renameMu sync.Mutex
|
||||||
|
|
||||||
// dummyVerifications counts the equivalent-cost verifications
|
// dummyVerifications counts the equivalent-cost verifications
|
||||||
// charged for usernames that do not exist. It exists so a test
|
// charged for usernames that do not exist. It exists so a test
|
||||||
// can prove that path runs without measuring wall-clock time.
|
// can prove that path runs without measuring wall-clock time.
|
||||||
|
|||||||
@@ -16,7 +16,6 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"go.uber.org/fx/fxtest"
|
"go.uber.org/fx/fxtest"
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
@@ -56,14 +55,12 @@ func (n *recordingNotifier) Tasks() []delivery.Task {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// recordingArchives is an archive.Archives that records what it
|
// recordingArchives is a delivery.Archives that records what it
|
||||||
// was asked to do, so a test can prove that a deletion or rename
|
// was asked to do, so a test can prove that a deletion or rename
|
||||||
// path reached the archive writers. After FailRenames, every
|
// path reached the delivery engine. After FailRenames, every
|
||||||
// rename of that target fails with the given error. After
|
// rename of that target fails with the given error. After
|
||||||
// BlockNextRename, the next rename is recorded and then waits.
|
// BlockNextRename, the next rename is recorded and then waits.
|
||||||
// RenameLock returns a plain mutex, as in archive.Writers.
|
|
||||||
type recordingArchives struct {
|
type recordingArchives struct {
|
||||||
renameMu sync.Mutex
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
evicted []string
|
evicted []string
|
||||||
evictedTargets []string
|
evictedTargets []string
|
||||||
@@ -76,10 +73,10 @@ type recordingArchives struct {
|
|||||||
// errInjectedRename is the failure a test hands FailRenames.
|
// errInjectedRename is the failure a test hands FailRenames.
|
||||||
var errInjectedRename = errors.New("injected rename failure")
|
var errInjectedRename = errors.New("injected rename failure")
|
||||||
|
|
||||||
// errNameTaken is what the archive writers return when a file
|
// errNameTaken is what the delivery engine returns when a file
|
||||||
// already has an archive's new name, here archive-taken.db.
|
// already has an archive's new name, here archive-taken.db.
|
||||||
var errNameTaken = fmt.Errorf(
|
var errNameTaken = fmt.Errorf(
|
||||||
"%w: archive-taken.db", archive.ErrNameTaken,
|
"%w: archive-taken.db", delivery.ErrArchiveNameTaken,
|
||||||
)
|
)
|
||||||
|
|
||||||
// archiveRename is one recorded Rename call.
|
// archiveRename is one recorded Rename call.
|
||||||
@@ -89,10 +86,6 @@ type archiveRename struct {
|
|||||||
TargetName string
|
TargetName string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *recordingArchives) RenameLock() *sync.Mutex {
|
|
||||||
return &r.renameMu
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *recordingArchives) EvictWebhook(webhookID string) {
|
func (r *recordingArchives) EvictWebhook(webhookID string) {
|
||||||
r.mu.Lock()
|
r.mu.Lock()
|
||||||
defer r.mu.Unlock()
|
defer r.mu.Unlock()
|
||||||
@@ -272,7 +265,7 @@ func newTestAppWithConfig(
|
|||||||
func() *recordingArchives {
|
func() *recordingArchives {
|
||||||
return &recordingArchives{}
|
return &recordingArchives{}
|
||||||
},
|
},
|
||||||
func(r *recordingArchives) archive.Archives {
|
func(r *recordingArchives) delivery.Archives {
|
||||||
return r
|
return r
|
||||||
},
|
},
|
||||||
func() *testCircuitBreakers {
|
func() *testCircuitBreakers {
|
||||||
|
|||||||
@@ -1,15 +1,19 @@
|
|||||||
package handlers_test
|
package handlers_test
|
||||||
|
|
||||||
// The handler-side half of the log-field audit. The slog calls in
|
// The handler-side half of the log-field audit. Two slog calls in
|
||||||
// this package that reach a value an UNAUTHENTICATED client picks
|
// this package reach a value an UNAUTHENTICATED client picks outright
|
||||||
// outright, and of a length it picks outright, are the failed-login
|
// and of a length it picks outright:
|
||||||
// DEBUG lines, whose username is a form field.
|
|
||||||
//
|
//
|
||||||
// They are at DEBUG, which is off in production by default. That is
|
// - the unknown-entrypoint DEBUG line on /h/{uuid}, whose
|
||||||
|
// path segment matched no stored entrypoint and so is bounded by
|
||||||
|
// nothing;
|
||||||
|
// - the failed-login DEBUG lines, whose username is a form field.
|
||||||
|
//
|
||||||
|
// Both are at DEBUG, which is off in production by default. That is
|
||||||
// not a bound: an operator turning DEBUG on to diagnose a flood must
|
// not a bound: an operator turning DEBUG on to diagnose a flood must
|
||||||
// not thereby hand the flood an unbounded write. They spend the same
|
// not thereby hand the flood an unbounded write. Both spend the same
|
||||||
// internal/logfield budget as the access log, and are held here to
|
// internal/logfield budget as the access log, and both are held here
|
||||||
// middleware.MaxAccessLogLineBytes.
|
// to middleware.MaxAccessLogLineBytes.
|
||||||
//
|
//
|
||||||
// The two login lines past the username lookup — "invalid password"
|
// The two login lines past the username lookup — "invalid password"
|
||||||
// and "user logged in" — carry the same cap without needing it, since
|
// and "user logged in" — carry the same cap without needing it, since
|
||||||
@@ -191,12 +195,13 @@ func receiverRouter(h *handlers.Handlers) *chi.Mux {
|
|||||||
return router
|
return router
|
||||||
}
|
}
|
||||||
|
|
||||||
// postReceiver sends one POST with an empty body at /h/<segment>
|
// postReceiver sends one POST at /h/<segment>.
|
||||||
// through router and returns the response status.
|
|
||||||
//
|
//
|
||||||
// RawPath is cleared after parsing so chi routes on the decoded path
|
// RawPath is cleared after parsing so chi routes on the decoded path
|
||||||
// and the handler sees the raw bytes rather than their percent-escaped
|
// and the handler sees the raw bytes rather than their percent-escaped
|
||||||
// spelling.
|
// spelling. That is the harder case for the budget: the escaped
|
||||||
|
// spelling is plain ASCII, which costs one byte per byte, while the
|
||||||
|
// decoded bytes are what the log handler has to escape.
|
||||||
func postReceiver(
|
func postReceiver(
|
||||||
t *testing.T, router *chi.Mux, segment string,
|
t *testing.T, router *chi.Mux, segment string,
|
||||||
) int {
|
) int {
|
||||||
@@ -255,6 +260,43 @@ func postLoginWithPassword(
|
|||||||
return w.Code
|
return w.Code
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestUnknownEntrypoint_LogLineDoesNotTrackPathSize drives 8 KB of
|
||||||
|
// client-chosen path at the unauthenticated receiver's
|
||||||
|
// unknown-entrypoint DEBUG line and holds it to the same ceiling the
|
||||||
|
// access log states.
|
||||||
|
func TestUnknownEntrypoint_LogLineDoesNotTrackPathSize(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for handlerName, newHandler := range logHandlers() {
|
||||||
|
for fillName, fill := range escapeFills() {
|
||||||
|
t.Run(handlerName+"/"+fillName, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
h, buf := capturingHandlers(t, newHandler)
|
||||||
|
router := receiverRouter(h)
|
||||||
|
|
||||||
|
for i := range floodRequests {
|
||||||
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
http.StatusNotFound,
|
||||||
|
postReceiver(
|
||||||
|
t, router,
|
||||||
|
oversizedFill(fill)+
|
||||||
|
strings.Repeat("y", i),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
lines := logLines(t, buf)
|
||||||
|
require.Len(t, lines, floodRequests)
|
||||||
|
assertNoClientText(t, buf)
|
||||||
|
|
||||||
|
assertBoundedFlood(t, buf.Len())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestFailedLogin_LogLineDoesNotTrackUsernameSize drives 8 KB of
|
// TestFailedLogin_LogLineDoesNotTrackUsernameSize drives 8 KB of
|
||||||
// client-chosen username at the unauthenticated login endpoint's
|
// client-chosen username at the unauthenticated login endpoint's
|
||||||
// DEBUG line and holds it to the same ceiling.
|
// DEBUG line and holds it to the same ceiling.
|
||||||
|
|||||||
@@ -16,16 +16,21 @@ import (
|
|||||||
//
|
//
|
||||||
// An empty value yields fallback, which lets the create path apply the
|
// An empty value yields fallback, which lets the create path apply the
|
||||||
// default and the edit path leave the stored value unchanged. A value
|
// default and the edit path leave the stored value unchanged. A value
|
||||||
// of 0 means retain events forever and is returned as it is. Anything
|
// of 0 is returned as 0 and is rewritten to the retain-forever
|
||||||
// unparseable or negative is refused rather than silently given a
|
// sentinel by database.Webhook's BeforeSave hook. Anything unparseable
|
||||||
// default.
|
// or negative is refused rather than silently given a default.
|
||||||
//
|
//
|
||||||
// The upper bound is not cosmetic. The reaper computes its cutoff as a
|
// The upper bound is not cosmetic. The reaper computes its cutoff as a
|
||||||
// time.Duration, an int64 nanosecond count, so a day count above
|
// time.Duration, an int64 nanosecond count, so a day count above
|
||||||
// database.MaxFiniteRetentionDays overflows, puts the cutoff in the
|
// database.MaxFiniteRetentionDays overflows, puts the cutoff in the
|
||||||
// future, and deletes every event the webhook has. A value above that
|
// future, and deletes every event the webhook has. A finite value
|
||||||
// ceiling is therefore refused, and the message names the ceiling
|
// above that ceiling is therefore refused, and the message names the
|
||||||
// rather than implying the input was not a number.
|
// ceiling rather than implying the input was not a number.
|
||||||
|
//
|
||||||
|
// A value at or above the retain-forever sentinel is not out of range:
|
||||||
|
// it is what the edit form pre-fills for a retain-forever webhook, so
|
||||||
|
// submitting the form back unchanged has to keep meaning "forever"
|
||||||
|
// rather than being rejected.
|
||||||
func parseRetentionDays(raw string, fallback int) (int, string) {
|
func parseRetentionDays(raw string, fallback int) (int, string) {
|
||||||
raw = strings.TrimSpace(raw)
|
raw = strings.TrimSpace(raw)
|
||||||
if raw == "" {
|
if raw == "" {
|
||||||
@@ -38,6 +43,10 @@ func parseRetentionDays(raw string, fallback int) (int, string) {
|
|||||||
"retain events forever."
|
"retain events forever."
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if v >= database.RetentionForeverDays {
|
||||||
|
return database.RetentionForeverDays, ""
|
||||||
|
}
|
||||||
|
|
||||||
if v > database.MaxFiniteRetentionDays {
|
if v > database.MaxFiniteRetentionDays {
|
||||||
return 0, "Retention must be at most " +
|
return 0, "Retention must be at most " +
|
||||||
strconv.Itoa(database.MaxFiniteRetentionDays) +
|
strconv.Itoa(database.MaxFiniteRetentionDays) +
|
||||||
|
|||||||
@@ -16,8 +16,8 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"gorm.io/gorm/clause"
|
"gorm.io/gorm/clause"
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
"sneak.berlin/go/webhooker/internal/handlers"
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
"sneak.berlin/go/webhooker/internal/session"
|
||||||
)
|
)
|
||||||
@@ -68,7 +68,6 @@ func seedTarget(
|
|||||||
Name: "t-" + string(targetType),
|
Name: "t-" + string(targetType),
|
||||||
Type: targetType,
|
Type: targetType,
|
||||||
Active: true,
|
Active: true,
|
||||||
MaxAttempts: 1,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
require.NoError(
|
require.NoError(
|
||||||
@@ -180,7 +179,7 @@ func failSaveOnTable(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// archivePathFor returns the archive database path the
|
// archivePathFor returns the archive database path the
|
||||||
// archive writers would use for a database target: beside the
|
// delivery engine would use for a database target: beside the
|
||||||
// webhook's event database in the data directory.
|
// webhook's event database in the data directory.
|
||||||
func archivePathFor(
|
func archivePathFor(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
@@ -192,7 +191,7 @@ func archivePathFor(
|
|||||||
|
|
||||||
return filepath.Join(
|
return filepath.Join(
|
||||||
filepath.Dir(mgr.DBPath(wh.ID)),
|
filepath.Dir(mgr.DBPath(wh.ID)),
|
||||||
archive.FileName(wh.Name, tgt.Name, tgt.ID),
|
delivery.ArchiveFileName(wh.Name, tgt.Name, tgt.ID),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -228,9 +227,9 @@ func postRequest(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleSourceDelete_EvictsArchiveWriter proves that
|
// TestHandleSourceDelete_EvictsArchiveWriter proves that
|
||||||
// deleting a webhook releases the webhook's archive writers,
|
// deleting a webhook reaches the delivery engine and releases
|
||||||
// exercised through the real deletion handler rather than by
|
// the webhook's archive writers, exercised through the real
|
||||||
// calling the archive writers directly.
|
// deletion handler rather than by calling the engine directly.
|
||||||
func TestHandleSourceDelete_EvictsArchiveWriter(t *testing.T) {
|
func TestHandleSourceDelete_EvictsArchiveWriter(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -290,7 +289,7 @@ func TestHandleSourceDelete_KeepsArchiveFile(t *testing.T) {
|
|||||||
wh := seedWebhook(t, db)
|
wh := seedWebhook(t, db)
|
||||||
tgt := seedTarget(t, db, wh.ID, database.TargetTypeDatabase)
|
tgt := seedTarget(t, db, wh.ID, database.TargetTypeDatabase)
|
||||||
|
|
||||||
// Place an archive file where the archive writers would.
|
// Place an archive file where the delivery engine would.
|
||||||
archivePath := archivePathFor(t, mgr, wh, tgt)
|
archivePath := archivePathFor(t, mgr, wh, tgt)
|
||||||
require.NoError(
|
require.NoError(
|
||||||
t,
|
t,
|
||||||
|
|||||||
@@ -42,7 +42,6 @@ func seedConfiguredTarget(
|
|||||||
Type: targetType,
|
Type: targetType,
|
||||||
Active: true,
|
Active: true,
|
||||||
Config: config,
|
Config: config,
|
||||||
MaxAttempts: 1,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
require.NoError(
|
require.NoError(
|
||||||
@@ -274,7 +273,7 @@ func TestHandleSourceDetail_FitsWideAndNarrowWindows(t *testing.T) {
|
|||||||
|
|
||||||
assert.Contains(
|
assert.Contains(
|
||||||
t, body,
|
t, body,
|
||||||
`<div class="max-w-[108rem] mx-auto px-6 py-8">`,
|
`<div class="mx-auto px-6 py-8" style="max-width: 108rem"`,
|
||||||
)
|
)
|
||||||
assert.Contains(
|
assert.Contains(
|
||||||
t, body,
|
t, body,
|
||||||
@@ -287,8 +286,8 @@ func TestHandleSourceDetail_FitsWideAndNarrowWindows(t *testing.T) {
|
|||||||
// target and says what deleting it loses, that the webhook's gives its
|
// target and says what deleting it loses, that the webhook's gives its
|
||||||
// number of stored events (5 received, 2 removed by retention, so 3,
|
// number of stored events (5 received, 2 removed by retention, so 3,
|
||||||
// the statistics pane's "Within retention" figure), and that an
|
// the statistics pane's "Within retention" figure), and that an
|
||||||
// entrypoint with no description is named by its URL. Each prompt is
|
// entrypoint with no description is named by its URL. The template
|
||||||
// its form's data-confirm attribute, where is a line break.
|
// writes the slashes after http: as \/, which the browser reads as /.
|
||||||
func TestHandleSourceDetail_DeletePromptsNameWhatIsLost(t *testing.T) {
|
func TestHandleSourceDetail_DeletePromptsNameWhatIsLost(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -326,28 +325,29 @@ func TestHandleSourceDetail_DeletePromptsNameWhatIsLost(t *testing.T) {
|
|||||||
body := renderSourceDetailPage(t, h, sess, wh.ID)
|
body := renderSourceDetailPage(t, h, sess, wh.ID)
|
||||||
|
|
||||||
assert.Contains(t, body,
|
assert.Contains(t, body,
|
||||||
`data-confirm="Delete webhook "delete-me"? `+
|
`Delete webhook "delete-me"?\n\n`+
|
||||||
`This deletes its stored events (3) and their deliveries. `+
|
`This deletes its stored events (3) and their deliveries. `+
|
||||||
`Any archive files it wrote are kept."`)
|
`Any archive files it wrote are kept.`)
|
||||||
assert.Contains(t, body,
|
assert.Contains(t, body,
|
||||||
`data-confirm="Delete entrypoint "Stripe"? `+
|
`Delete entrypoint "Stripe"?\n\n`+
|
||||||
`Senders using its URL get an error from now on, `+
|
`Senders using its URL get an error from now on, `+
|
||||||
`and the URL cannot be restored."`)
|
`and the URL cannot be restored.`)
|
||||||
assert.Contains(t, body,
|
assert.Contains(t, body,
|
||||||
`data-confirm="Delete entrypoint "http://example.com/h/`+
|
`Delete entrypoint "http:\/\/example.com/h/`+
|
||||||
unnamed.Path+`"? `)
|
unnamed.Path+`"?`)
|
||||||
assert.Contains(t, body,
|
assert.Contains(t, body,
|
||||||
`data-confirm="Delete target "t-log"? `+
|
`Delete target "t-log"?\n\n`+
|
||||||
`Nothing more is delivered to it. `+
|
`Nothing more is delivered to it. `+
|
||||||
`Its past deliveries stay in the event log."`)
|
`Its past deliveries stay in the event log.`)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleSourceDetail_DeletePromptKeepsQuotesInName checks that a
|
// TestHandleSourceDetail_DeletePromptKeepsQuotesInName checks that a
|
||||||
// webhook name with quotes, a backslash, a closing script tag and a
|
// webhook name with quotes, a backslash, a closing script tag and a
|
||||||
// newline reaches its delete prompt escaped for the attribute, which
|
// newline reaches its delete prompt escaped for the script, which the
|
||||||
// the browser reads back as the name typed: each quote and angle
|
// browser reads back as the name typed: each quote and angle bracket
|
||||||
// bracket as a character reference, the rest as typed. An unescaped
|
// as a \u escape, the slash as \/, the newline as \n and the backslash
|
||||||
// double quote would end the attribute and cut the prompt short.
|
// doubled. An unescaped newline would break the prompt's script, and
|
||||||
|
// the form would then submit without asking.
|
||||||
func TestHandleSourceDetail_DeletePromptKeepsQuotesInName(t *testing.T) {
|
func TestHandleSourceDetail_DeletePromptKeepsQuotesInName(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -373,6 +373,6 @@ func TestHandleSourceDetail_DeletePromptKeepsQuotesInName(t *testing.T) {
|
|||||||
body := renderSourceDetailPage(t, h, sess, wh.ID)
|
body := renderSourceDetailPage(t, h, sess, wh.ID)
|
||||||
|
|
||||||
assert.Contains(t, body,
|
assert.Contains(t, body,
|
||||||
"Delete webhook "Bob's "best" \\ hook"+
|
"Delete webhook "Bob\\u0027s \\u0022best\\u0022 \\\\ hook"+
|
||||||
"</script>\nline two"?")
|
"\\u003c\\/script\\u003e\\nline two"?")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -137,7 +137,8 @@ func onlyWebhook(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// seedWebhookWithRetention inserts a webhook owned by the test user
|
// seedWebhookWithRetention inserts a webhook owned by the test user
|
||||||
// with the given retention.
|
// with an exact stored retention value, bypassing Webhook.BeforeSave
|
||||||
|
// via a column-level update so that legacy rows can be planted too.
|
||||||
func seedWebhookWithRetention(
|
func seedWebhookWithRetention(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
db *database.Database,
|
db *database.Database,
|
||||||
@@ -154,6 +155,13 @@ func seedWebhookWithRetention(
|
|||||||
t,
|
t,
|
||||||
db.DB().Omit(clause.Associations).Create(wh).Error,
|
db.DB().Omit(clause.Associations).Create(wh).Error,
|
||||||
)
|
)
|
||||||
|
require.NoError(
|
||||||
|
t,
|
||||||
|
db.DB().Model(wh).
|
||||||
|
Update("retention_days", retentionDays).Error,
|
||||||
|
)
|
||||||
|
|
||||||
|
wh.RetentionDays = retentionDays
|
||||||
|
|
||||||
return *wh
|
return *wh
|
||||||
}
|
}
|
||||||
@@ -217,9 +225,10 @@ func setupSourceTest(t *testing.T) *sourceTestEnv {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleSourceCreateSubmit_ZeroRetentionPersistsForever checks
|
// TestHandleSourceCreateSubmit_ZeroRetentionPersistsForever is the core
|
||||||
// that the create form's 0 reaches the database as 0, which means
|
// regression test for the bug: the create form's 0 must reach the
|
||||||
// retain forever.
|
// database as the retain-forever sentinel rather than being replaced by
|
||||||
|
// the column's default of 30.
|
||||||
func TestHandleSourceCreateSubmit_ZeroRetentionPersistsForever(
|
func TestHandleSourceCreateSubmit_ZeroRetentionPersistsForever(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
@@ -232,45 +241,22 @@ func TestHandleSourceCreateSubmit_ZeroRetentionPersistsForever(
|
|||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
|
|
||||||
wh := onlyWebhook(t, env.db)
|
wh := onlyWebhook(t, env.db)
|
||||||
assert.Equal(t, 0, storedRetentionDays(t, env.db, wh.ID))
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
database.RetentionForeverDays,
|
||||||
|
storedRetentionDays(t, env.db, wh.ID),
|
||||||
|
)
|
||||||
assert.True(t, wh.RetainsForever())
|
assert.True(t, wh.RetainsForever())
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandleSourceCreateSubmit_DayCountIsStored(t *testing.T) {
|
func TestHandleSourceCreateSubmit_OmittedRetentionUsesDefault(
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
days := "45"
|
|
||||||
|
|
||||||
w := submitCreate(t, env.handlers, env.cookies, "finite", &days)
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
|
||||||
|
|
||||||
wh := onlyWebhook(t, env.db)
|
|
||||||
assert.Equal(t, 45, storedRetentionDays(t, env.db, wh.ID))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceCreateSubmit_BlankRetentionUsesDefault covers a
|
|
||||||
// retention field left empty and one missing from the form: both
|
|
||||||
// give the default, not forever.
|
|
||||||
func TestHandleSourceCreateSubmit_BlankRetentionUsesDefault(
|
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
blank := ""
|
|
||||||
|
|
||||||
for name, retention := range map[string]*string{
|
|
||||||
"blank": &blank,
|
|
||||||
"omitted": nil,
|
|
||||||
} {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
env := setupSourceTest(t)
|
||||||
|
|
||||||
w := submitCreate(
|
w := submitCreate(t, env.handlers, env.cookies, "defaulted", nil)
|
||||||
t, env.handlers, env.cookies, "defaulted", retention,
|
|
||||||
)
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
|
|
||||||
wh := onlyWebhook(t, env.db)
|
wh := onlyWebhook(t, env.db)
|
||||||
@@ -279,13 +265,11 @@ func TestHandleSourceCreateSubmit_BlankRetentionUsesDefault(
|
|||||||
database.DefaultRetentionDays,
|
database.DefaultRetentionDays,
|
||||||
storedRetentionDays(t, env.db, wh.ID),
|
storedRetentionDays(t, env.db, wh.ID),
|
||||||
)
|
)
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleSourceCreate_PrefillsDefaultFromConstant checks that the
|
// TestHandleSourceCreate_PrefillsDefaultFromConstant keeps the create
|
||||||
// create form pre-fills its retention from database.DefaultRetentionDays
|
// form's pre-filled retention from becoming a third hardcoded copy of
|
||||||
// rather than from a hardcoded 30.
|
// the 30-day policy.
|
||||||
func TestHandleSourceCreate_PrefillsDefaultFromConstant(t *testing.T) {
|
func TestHandleSourceCreate_PrefillsDefaultFromConstant(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -306,7 +290,7 @@ func TestHandleSourceCreate_PrefillsDefaultFromConstant(t *testing.T) {
|
|||||||
)
|
)
|
||||||
assert.NotContains(
|
assert.NotContains(
|
||||||
t, body, `max="365"`,
|
t, body, `max="365"`,
|
||||||
"a max of 365 would block any longer retention",
|
"a max below the sentinel would block retain-forever",
|
||||||
)
|
)
|
||||||
assert.Contains(t, body, `min="0"`)
|
assert.Contains(t, body, `min="0"`)
|
||||||
}
|
}
|
||||||
@@ -384,6 +368,40 @@ func TestHandleSourceCreateSubmit_OverflowingRetentionIsRejected(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestHandleSourceCreateSubmit_SentinelIsAcceptedAsForever guards the
|
||||||
|
// boundary between "too large to represent" and "retain forever": the
|
||||||
|
// sentinel is above MaxFiniteRetentionDays, but it is the value the
|
||||||
|
// edit form pre-fills, so it must be accepted rather than rejected as
|
||||||
|
// out of range. A value above the sentinel is stored as the sentinel.
|
||||||
|
func TestHandleSourceCreateSubmit_SentinelIsAcceptedAsForever(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
for _, days := range []int{
|
||||||
|
database.RetentionForeverDays,
|
||||||
|
database.RetentionForeverDays + 1,
|
||||||
|
} {
|
||||||
|
raw := strconv.Itoa(days)
|
||||||
|
|
||||||
|
t.Run(raw, func(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupSourceTest(t)
|
||||||
|
|
||||||
|
w := submitCreate(t, env.handlers, env.cookies, "forever", &raw)
|
||||||
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
|
|
||||||
|
wh := onlyWebhook(t, env.db)
|
||||||
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
database.RetentionForeverDays,
|
||||||
|
storedRetentionDays(t, env.db, wh.ID),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestHandleSourceCreateSubmit_RejectedFormKeepsUserInput checks that a
|
// TestHandleSourceCreateSubmit_RejectedFormKeepsUserInput checks that a
|
||||||
// validation failure hands the user's typing back, matching what the
|
// validation failure hands the user's typing back, matching what the
|
||||||
// edit form already does. Losing a long description to a mistyped
|
// edit form already does. Losing a long description to a mistyped
|
||||||
@@ -462,7 +480,11 @@ func TestHandleSourceEditSubmit_ZeroRetentionPersistsForever(
|
|||||||
w := submitEdit(t, env, wh, "0")
|
w := submitEdit(t, env, wh, "0")
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
|
|
||||||
assert.Equal(t, 0, storedRetentionDays(t, env.db, wh.ID))
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
database.RetentionForeverDays,
|
||||||
|
storedRetentionDays(t, env.db, wh.ID),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHandleSourceEditSubmit_InvalidRetentionIsRejected(
|
func TestHandleSourceEditSubmit_InvalidRetentionIsRejected(
|
||||||
@@ -845,15 +867,18 @@ func TestHandleSourceEditSubmit_ArchiveNameTaken(t *testing.T) {
|
|||||||
assert.Equal(t, oldName, stored.Name)
|
assert.Equal(t, oldName, stored.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestSourceEditForm_ForeverWebhookRoundTrips renders the edit form for
|
// TestSourceEditForm_ForeverWebhookRoundTrips walks the exact path that
|
||||||
// a retain-forever webhook, confirms it pre-fills 0, then submits that
|
// the removed max="365" cap used to break: render the edit form for a
|
||||||
// pre-filled value straight back and confirms the retention policy
|
// retain-forever webhook, confirm the pre-filled sentinel is not capped
|
||||||
// survives untouched.
|
// by browser validation, then submit that pre-filled value straight
|
||||||
|
// back and confirm the retention policy survives untouched.
|
||||||
func TestSourceEditForm_ForeverWebhookRoundTrips(t *testing.T) {
|
func TestSourceEditForm_ForeverWebhookRoundTrips(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
env := setupSourceTest(t)
|
||||||
wh := seedWebhookWithRetention(t, env.db, 0)
|
wh := seedWebhookWithRetention(
|
||||||
|
t, env.db, database.RetentionForeverDays,
|
||||||
|
)
|
||||||
|
|
||||||
req := getRequest(
|
req := getRequest(
|
||||||
t, "/hook/"+wh.ID+"/edit", env.cookies,
|
t, "/hook/"+wh.ID+"/edit", env.cookies,
|
||||||
@@ -864,12 +889,17 @@ func TestSourceEditForm_ForeverWebhookRoundTrips(t *testing.T) {
|
|||||||
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
require.Equal(t, http.StatusOK, w.Code)
|
||||||
|
|
||||||
|
sentinel := strconv.Itoa(database.RetentionForeverDays)
|
||||||
body := w.Body.String()
|
body := w.Body.String()
|
||||||
|
|
||||||
assert.Contains(
|
assert.Contains(
|
||||||
t, body, `name="retention_days" value="0"`,
|
t, body, `value="`+sentinel+`"`,
|
||||||
"the edit form pre-fills the stored retention",
|
"the edit form pre-fills the stored retention",
|
||||||
)
|
)
|
||||||
|
assert.NotContains(
|
||||||
|
t, body, `max="365"`,
|
||||||
|
"a max below the sentinel would block saving any edit",
|
||||||
|
)
|
||||||
// "Currently forever." is the rendered RetentionLabel, not the
|
// "Currently forever." is the rendered RetentionLabel, not the
|
||||||
// static hint below the input, which says "Enter 0 to retain events
|
// static hint below the input, which says "Enter 0 to retain events
|
||||||
// forever." A bare Contains of "forever" would pass for any
|
// forever." A bare Contains of "forever" would pass for any
|
||||||
@@ -880,20 +910,29 @@ func TestSourceEditForm_ForeverWebhookRoundTrips(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
// Submit the pre-filled value back, exactly as a browser would.
|
// Submit the pre-filled value back, exactly as a browser would.
|
||||||
post := submitEdit(t, env, wh, "0")
|
post := submitEdit(t, env, wh, sentinel)
|
||||||
require.Equal(t, http.StatusSeeOther, post.Code)
|
require.Equal(t, http.StatusSeeOther, post.Code)
|
||||||
|
|
||||||
assert.Equal(t, 0, storedRetentionDays(t, env.db, wh.ID))
|
assert.Equal(
|
||||||
|
t,
|
||||||
|
database.RetentionForeverDays,
|
||||||
|
storedRetentionDays(t, env.db, wh.ID),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestSourceListAndDetail_ShowForeverNotZeroDays checks that a
|
// TestSourceListAndDetail_ShowForeverNotTheSentinelNumber checks that
|
||||||
// retention of 0 is shown as forever, never as a day count, on either
|
// the retain-forever value is never rendered to the user as a raw day
|
||||||
// read-only view.
|
// count on either read-only view.
|
||||||
func TestSourceListAndDetail_ShowForeverNotZeroDays(t *testing.T) {
|
func TestSourceListAndDetail_ShowForeverNotTheSentinelNumber(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
env := setupSourceTest(t)
|
||||||
wh := seedWebhookWithRetention(t, env.db, 0)
|
wh := seedWebhookWithRetention(
|
||||||
|
t, env.db, database.RetentionForeverDays,
|
||||||
|
)
|
||||||
|
sentinel := strconv.Itoa(database.RetentionForeverDays)
|
||||||
|
|
||||||
listW := httptest.NewRecorder()
|
listW := httptest.NewRecorder()
|
||||||
env.handlers.HandleSourceList().ServeHTTP(
|
env.handlers.HandleSourceList().ServeHTTP(
|
||||||
@@ -902,7 +941,7 @@ func TestSourceListAndDetail_ShowForeverNotZeroDays(t *testing.T) {
|
|||||||
|
|
||||||
require.Equal(t, http.StatusOK, listW.Code)
|
require.Equal(t, http.StatusOK, listW.Code)
|
||||||
assert.Contains(t, listW.Body.String(), "Retention: forever")
|
assert.Contains(t, listW.Body.String(), "Retention: forever")
|
||||||
assert.NotContains(t, listW.Body.String(), "Retention: 0 days")
|
assert.NotContains(t, listW.Body.String(), sentinel)
|
||||||
|
|
||||||
detailW := httptest.NewRecorder()
|
detailW := httptest.NewRecorder()
|
||||||
env.handlers.HandleSourceDetail().ServeHTTP(
|
env.handlers.HandleSourceDetail().ServeHTTP(
|
||||||
@@ -915,5 +954,5 @@ func TestSourceListAndDetail_ShowForeverNotZeroDays(t *testing.T) {
|
|||||||
|
|
||||||
require.Equal(t, http.StatusOK, detailW.Code)
|
require.Equal(t, http.StatusOK, detailW.Code)
|
||||||
assert.Contains(t, detailW.Body.String(), "Retention: forever")
|
assert.Contains(t, detailW.Body.String(), "Retention: forever")
|
||||||
assert.NotContains(t, detailW.Body.String(), "Retention: 0 days")
|
assert.NotContains(t, detailW.Body.String(), sentinel)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,81 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
// maxTargetAttempts bounds a target's max_attempts.
|
|
||||||
//
|
|
||||||
// Both target forms already declare max="20" on the input, so this
|
|
||||||
// enforces server-side what the UI has always advertised rather than
|
|
||||||
// introducing a new limit.
|
|
||||||
//
|
|
||||||
// The number is not cosmetic. Every attempt writes a delivery_results
|
|
||||||
// row that the event log then loads and renders, and the engine backs
|
|
||||||
// off by 2^(n-1) seconds, so attempt 20 is already about six days
|
|
||||||
// after the first. A value beyond this buys no additional durability
|
|
||||||
// and only costs rows.
|
|
||||||
const maxTargetAttempts = 20
|
|
||||||
|
|
||||||
// Errors returned when a max_attempts form value cannot be turned into
|
|
||||||
// a number of attempts.
|
|
||||||
var (
|
|
||||||
// errAttemptsInvalid signals a max_attempts form value that is not
|
|
||||||
// a whole number of at least 1.
|
|
||||||
errAttemptsInvalid = errors.New(
|
|
||||||
"must be a whole number, at least 1",
|
|
||||||
)
|
|
||||||
|
|
||||||
// errAttemptsTooLarge signals a max_attempts form value that is a
|
|
||||||
// whole number but above maxTargetAttempts. It is distinguished
|
|
||||||
// from errAttemptsInvalid so the message can name the ceiling
|
|
||||||
// instead of implying the input was not a number.
|
|
||||||
errAttemptsTooLarge = errors.New("out of range")
|
|
||||||
)
|
|
||||||
|
|
||||||
// parseMaxAttempts interprets a max_attempts form value.
|
|
||||||
//
|
|
||||||
// An ABSENT value — the field empty or not submitted — yields
|
|
||||||
// fallback, which lets the create path apply its default and the edit
|
|
||||||
// path leave the stored value alone. A value that is SET BUT INVALID
|
|
||||||
// is an error: unparseable, below 1, or above maxTargetAttempts.
|
|
||||||
//
|
|
||||||
// The distinction is the whole point of this function. Returning a
|
|
||||||
// default for input the operator typed but that did not parse would
|
|
||||||
// silently turn retries off on a store-and-forward proxy — and on the
|
|
||||||
// edit path destroy a working configuration over a typo. A default
|
|
||||||
// answers a question that was not asked; it never answers one that
|
|
||||||
// was asked badly.
|
|
||||||
func parseMaxAttempts(raw string, fallback int) (int, error) {
|
|
||||||
raw = strings.TrimSpace(raw)
|
|
||||||
if raw == "" {
|
|
||||||
return fallback, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
v, err := strconv.Atoi(raw)
|
|
||||||
if err != nil || v < 1 {
|
|
||||||
return 0, errAttemptsInvalid
|
|
||||||
}
|
|
||||||
|
|
||||||
if v > maxTargetAttempts {
|
|
||||||
return 0, errAttemptsTooLarge
|
|
||||||
}
|
|
||||||
|
|
||||||
return v, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// attemptsErrorMessage returns the message the create and edit forms
|
|
||||||
// show for a rejected max_attempts value. Any error other than
|
|
||||||
// errAttemptsTooLarge falls back to the generic wording, so an
|
|
||||||
// unrecognised parse failure still produces a sensible 400.
|
|
||||||
func attemptsErrorMessage(err error) string {
|
|
||||||
if errors.Is(err, errAttemptsTooLarge) {
|
|
||||||
return errAttemptsTooLarge.Error() +
|
|
||||||
": at most " + strconv.Itoa(maxTargetAttempts) +
|
|
||||||
" attempts"
|
|
||||||
}
|
|
||||||
|
|
||||||
return errAttemptsInvalid.Error()
|
|
||||||
}
|
|
||||||
@@ -1,429 +0,0 @@
|
|||||||
package handlers_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"strconv"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
|
||||||
)
|
|
||||||
|
|
||||||
// attemptsTargetURL is the destination the attempts-validation targets
|
|
||||||
// point at. It is a literal public address rather than a hostname so
|
|
||||||
// the SSRF check resolves nothing and a sandbox without DNS cannot
|
|
||||||
// make these cases pass or fail for the wrong reason.
|
|
||||||
const attemptsTargetURL = "https://93.184.216.34/hooks/attempts"
|
|
||||||
|
|
||||||
const (
|
|
||||||
// wayAboveCeiling is the typo'd-extra-zero case from the report.
|
|
||||||
wayAboveCeiling = "999999999"
|
|
||||||
|
|
||||||
// notANumber is the plainest garbage an operator can type, and
|
|
||||||
// the value the report submitted on the edit form.
|
|
||||||
notANumber = "abc"
|
|
||||||
|
|
||||||
// workingAttempts is the number of attempts a seeded target is
|
|
||||||
// already delivering with, which a rejected submission must not
|
|
||||||
// disturb.
|
|
||||||
workingAttempts = 2
|
|
||||||
)
|
|
||||||
|
|
||||||
// aboveCeiling is the smallest rejected whole number.
|
|
||||||
func aboveCeiling() string {
|
|
||||||
return strconv.Itoa(handlers.MaxTargetAttemptsForTest + 1)
|
|
||||||
}
|
|
||||||
|
|
||||||
// overCeilingAttempts is whole-number input past the limit, which is
|
|
||||||
// rejected with the limit named.
|
|
||||||
func overCeilingAttempts() []string {
|
|
||||||
return []string{aboveCeiling(), wayAboveCeiling}
|
|
||||||
}
|
|
||||||
|
|
||||||
// belowOneAttempts is whole-number input under the floor of one
|
|
||||||
// attempt, which is rejected with the floor named.
|
|
||||||
func belowOneAttempts() []string {
|
|
||||||
return []string{"0", "-5"}
|
|
||||||
}
|
|
||||||
|
|
||||||
// unparseableAttempts is input an operator can type into the field
|
|
||||||
// that is not a number of attempts. Each must be REJECTED: silently
|
|
||||||
// reading any of them as a default turns retries off on a
|
|
||||||
// store-and-forward proxy without saying so.
|
|
||||||
//
|
|
||||||
// The twenty-digit case is here because it parses as digits but
|
|
||||||
// overflows int, which is the one failure the field's own min/max
|
|
||||||
// attributes cannot describe.
|
|
||||||
func unparseableAttempts() []string {
|
|
||||||
return []string{
|
|
||||||
notANumber,
|
|
||||||
"2.7",
|
|
||||||
"12345678901234567890",
|
|
||||||
"1e3",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// createAttemptsForm is a complete, otherwise-valid HTTP target
|
|
||||||
// creation, so the only thing any case below varies is max_attempts.
|
|
||||||
func createAttemptsForm(attempts string) url.Values {
|
|
||||||
form := url.Values{}
|
|
||||||
form.Set("name", "attempts-target")
|
|
||||||
form.Set("type", string(database.TargetTypeHTTP))
|
|
||||||
form.Set("url", attemptsTargetURL)
|
|
||||||
|
|
||||||
if attempts != absentField {
|
|
||||||
form.Set("max_attempts", attempts)
|
|
||||||
}
|
|
||||||
|
|
||||||
return form
|
|
||||||
}
|
|
||||||
|
|
||||||
// absentField marks a field the form does not submit at all, which is
|
|
||||||
// the case that legitimately takes a default and must stay distinct
|
|
||||||
// from a field submitted with garbage in it.
|
|
||||||
const absentField = "\x00absent"
|
|
||||||
|
|
||||||
// absentAttempts is every way of saying "the operator did not set
|
|
||||||
// this", each of which takes the default rather than a 400. Blank and
|
|
||||||
// whitespace-only count as absent here because they do in the timeout
|
|
||||||
// and retention controls on the same forms; a rule the fields do not
|
|
||||||
// share would be its own surprise.
|
|
||||||
func absentAttempts() []string {
|
|
||||||
return []string{absentField, "", " "}
|
|
||||||
}
|
|
||||||
|
|
||||||
// createWithAttempts posts the target create form for a fresh webhook
|
|
||||||
// and returns the webhook and the response.
|
|
||||||
func createWithAttempts(
|
|
||||||
t *testing.T,
|
|
||||||
env *sourceTestEnv,
|
|
||||||
attempts string,
|
|
||||||
) (database.Webhook, int, string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
webhook := seedWebhookWithRetention(t, env.db, 30)
|
|
||||||
|
|
||||||
w := serveTarget(
|
|
||||||
env, http.MethodPost,
|
|
||||||
"/hook/"+webhook.ID+"/targets",
|
|
||||||
createAttemptsForm(attempts),
|
|
||||||
)
|
|
||||||
|
|
||||||
return webhook, w.Code, w.Body.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTargetCreate_AttemptsAboveCeilingRejected proves the create form
|
|
||||||
// enforces a ceiling at all, and that the 400 names it — a rejection
|
|
||||||
// that does not say what the limit is leaves the operator guessing.
|
|
||||||
func TestTargetCreate_AttemptsAboveCeilingRejected(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
ceiling := strconv.Itoa(handlers.MaxTargetAttemptsForTest)
|
|
||||||
|
|
||||||
for _, attempts := range overCeilingAttempts() {
|
|
||||||
webhook, code, body := createWithAttempts(t, env, attempts)
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusBadRequest, code,
|
|
||||||
"max_attempts=%s should be rejected", attempts)
|
|
||||||
assert.Contains(t, body, ceiling,
|
|
||||||
"the rejection for %s should name the ceiling",
|
|
||||||
attempts)
|
|
||||||
assert.Empty(t,
|
|
||||||
targetsForWebhook(t, env.db, webhook.ID),
|
|
||||||
"no target should be created for %s", attempts)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTargetCreate_AttemptsBelowOneRejected proves the create form
|
|
||||||
// refuses 0 and less, naming the floor: every delivery is attempted
|
|
||||||
// at least once.
|
|
||||||
func TestTargetCreate_AttemptsBelowOneRejected(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
|
|
||||||
for _, attempts := range belowOneAttempts() {
|
|
||||||
webhook, code, body := createWithAttempts(t, env, attempts)
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusBadRequest, code,
|
|
||||||
"max_attempts=%s should be rejected", attempts)
|
|
||||||
assert.Contains(t, body, "at least 1",
|
|
||||||
"the rejection for %s should name the floor",
|
|
||||||
attempts)
|
|
||||||
assert.Empty(t,
|
|
||||||
targetsForWebhook(t, env.db, webhook.ID),
|
|
||||||
"no target should be created for %s", attempts)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTargetCreate_UnparseableAttemptsRejected is the core of the
|
|
||||||
// defect: each of these was accepted with HTTP 200 and stored as a
|
|
||||||
// default.
|
|
||||||
func TestTargetCreate_UnparseableAttemptsRejected(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
|
|
||||||
for _, attempts := range unparseableAttempts() {
|
|
||||||
webhook, code, body := createWithAttempts(t, env, attempts)
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusBadRequest, code,
|
|
||||||
"max_attempts=%q should be rejected, not coerced",
|
|
||||||
attempts)
|
|
||||||
assert.Contains(t, body, "whole number",
|
|
||||||
"the rejection for %q should say why", attempts)
|
|
||||||
assert.Empty(t,
|
|
||||||
targetsForWebhook(t, env.db, webhook.ID),
|
|
||||||
"no target should be created for %q", attempts)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTargetCreate_ValidAttemptsStored covers the accepting half,
|
|
||||||
// including both bounds: a bound that rejects its own limit would
|
|
||||||
// make the advertised range unreachable.
|
|
||||||
func TestTargetCreate_ValidAttemptsStored(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
|
|
||||||
for _, want := range []int{1, 3, handlers.MaxTargetAttemptsForTest} {
|
|
||||||
webhook, code, body := createWithAttempts(
|
|
||||||
t, env, strconv.Itoa(want),
|
|
||||||
)
|
|
||||||
|
|
||||||
require.Equal(t, http.StatusSeeOther, code, body)
|
|
||||||
|
|
||||||
targets := targetsForWebhook(t, env.db, webhook.ID)
|
|
||||||
require.Len(t, targets, 1)
|
|
||||||
assert.Equal(t, want, targets[0].MaxAttempts)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTargetCreate_AbsentAttemptsTakesDefault keeps the two cases
|
|
||||||
// distinct. An omitted field is not an operator asking for something
|
|
||||||
// invalid, so it still gets the default of one attempt rather than a
|
|
||||||
// 400 — otherwise the fix above would make the form unusable.
|
|
||||||
func TestTargetCreate_AbsentAttemptsTakesDefault(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
|
|
||||||
for _, attempts := range absentAttempts() {
|
|
||||||
webhook, code, body := createWithAttempts(t, env, attempts)
|
|
||||||
|
|
||||||
require.Equal(t, http.StatusSeeOther, code, body)
|
|
||||||
|
|
||||||
targets := targetsForWebhook(t, env.db, webhook.ID)
|
|
||||||
require.Len(t, targets, 1)
|
|
||||||
assert.Equal(t, 1, targets[0].MaxAttempts,
|
|
||||||
"an absent max_attempts should take the default")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// seedAttemptsTarget creates an HTTP target already delivering with
|
|
||||||
// workingAttempts attempts, through the real create handler.
|
|
||||||
func seedAttemptsTarget(
|
|
||||||
t *testing.T,
|
|
||||||
env *sourceTestEnv,
|
|
||||||
) (database.Webhook, database.Target) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
webhook, code, body := createWithAttempts(
|
|
||||||
t, env, strconv.Itoa(workingAttempts),
|
|
||||||
)
|
|
||||||
require.Equal(t, http.StatusSeeOther, code, body)
|
|
||||||
|
|
||||||
targets := targetsForWebhook(t, env.db, webhook.ID)
|
|
||||||
require.Len(t, targets, 1)
|
|
||||||
require.Equal(t, workingAttempts, targets[0].MaxAttempts)
|
|
||||||
|
|
||||||
return webhook, targets[0]
|
|
||||||
}
|
|
||||||
|
|
||||||
// editAttemptsForm is a complete edit submission that changes the
|
|
||||||
// target's name as well, so a rejected submission can be shown to
|
|
||||||
// have written nothing at all rather than merely to have left
|
|
||||||
// max_attempts alone.
|
|
||||||
func editAttemptsForm(attempts string) url.Values {
|
|
||||||
form := url.Values{}
|
|
||||||
form.Set("name", "renamed-by-edit")
|
|
||||||
form.Set("url", attemptsTargetURL)
|
|
||||||
|
|
||||||
if attempts != absentField {
|
|
||||||
form.Set("max_attempts", attempts)
|
|
||||||
}
|
|
||||||
|
|
||||||
return form
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertEditRejectedAndUnchanged submits an edit expected to fail and
|
|
||||||
// checks both halves of the requirement: the 400 explains itself, and
|
|
||||||
// the target it was submitted against is untouched.
|
|
||||||
func assertEditRejectedAndUnchanged(
|
|
||||||
t *testing.T,
|
|
||||||
env *sourceTestEnv,
|
|
||||||
attempts, wantReason string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
webhook, target := seedAttemptsTarget(t, env)
|
|
||||||
|
|
||||||
w := submitTargetEdit(
|
|
||||||
env, webhook.ID, target.ID, editAttemptsForm(attempts),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusBadRequest, w.Code,
|
|
||||||
"max_attempts=%q should be rejected on edit", attempts)
|
|
||||||
assert.Contains(t, w.Body.String(), wantReason,
|
|
||||||
"the rejection for %q should say why", attempts)
|
|
||||||
|
|
||||||
stored := storedTarget(t, env, target.ID)
|
|
||||||
assert.Equal(t, workingAttempts, stored.MaxAttempts,
|
|
||||||
"a rejected edit must not destroy the working number "+
|
|
||||||
"of attempts with %q", attempts)
|
|
||||||
assert.Equal(t, "attempts-target", stored.Name,
|
|
||||||
"a rejected edit must write nothing at all")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTargetEdit_UnparseableAttemptsRejected is the damaging half of
|
|
||||||
// the defect. A target delivering with retries, re-saved with a typo
|
|
||||||
// in the field, returned 200 and was left with retries disabled and
|
|
||||||
// nothing said.
|
|
||||||
func TestTargetEdit_UnparseableAttemptsRejected(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
|
|
||||||
for _, attempts := range unparseableAttempts() {
|
|
||||||
assertEditRejectedAndUnchanged(
|
|
||||||
t, env, attempts, "whole number",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTargetEdit_AttemptsBelowOneRejected proves the floor applies
|
|
||||||
// to the edit path too.
|
|
||||||
func TestTargetEdit_AttemptsBelowOneRejected(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
|
|
||||||
for _, attempts := range belowOneAttempts() {
|
|
||||||
assertEditRejectedAndUnchanged(t, env, attempts, "at least 1")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTargetEdit_AttemptsAboveCeilingRejected proves the ceiling
|
|
||||||
// applies to the edit path too, naming itself, so the two paths
|
|
||||||
// cannot disagree about what is storable.
|
|
||||||
func TestTargetEdit_AttemptsAboveCeilingRejected(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
ceiling := strconv.Itoa(handlers.MaxTargetAttemptsForTest)
|
|
||||||
|
|
||||||
for _, attempts := range overCeilingAttempts() {
|
|
||||||
assertEditRejectedAndUnchanged(t, env, attempts, ceiling)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTargetEdit_ValidAttemptsStored covers the accepting half of the
|
|
||||||
// edit path, so the bounds cannot be enforced by simply refusing
|
|
||||||
// every submission.
|
|
||||||
func TestTargetEdit_ValidAttemptsStored(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
|
|
||||||
for _, want := range []int{1, 9, handlers.MaxTargetAttemptsForTest} {
|
|
||||||
webhook, target := seedAttemptsTarget(t, env)
|
|
||||||
|
|
||||||
w := submitTargetEdit(
|
|
||||||
env, webhook.ID, target.ID,
|
|
||||||
editAttemptsForm(strconv.Itoa(want)),
|
|
||||||
)
|
|
||||||
require.Equal(t,
|
|
||||||
http.StatusSeeOther, w.Code, w.Body.String(),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Equal(t, want,
|
|
||||||
storedTarget(t, env, target.ID).MaxAttempts)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTargetEdit_AbsentAttemptsLeavesStoredValue is the edit path's
|
|
||||||
// absent-versus-invalid case. The field is only offered by the forms
|
|
||||||
// for types that retry, so a submission without it must leave the
|
|
||||||
// stored count alone rather than be rejected or reset.
|
|
||||||
func TestTargetEdit_AbsentAttemptsLeavesStoredValue(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
|
|
||||||
for _, attempts := range absentAttempts() {
|
|
||||||
webhook, target := seedAttemptsTarget(t, env)
|
|
||||||
|
|
||||||
w := submitTargetEdit(
|
|
||||||
env, webhook.ID, target.ID,
|
|
||||||
editAttemptsForm(attempts),
|
|
||||||
)
|
|
||||||
require.Equal(t,
|
|
||||||
http.StatusSeeOther, w.Code, w.Body.String(),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Equal(t, workingAttempts,
|
|
||||||
storedTarget(t, env, target.ID).MaxAttempts,
|
|
||||||
"an absent max_attempts must leave the stored "+
|
|
||||||
"count alone (%q)", attempts)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestTargetAttempts_CreateAndEditAgreeOnEveryCase proves the two
|
|
||||||
// paths cannot disagree, which is what let the create form and the
|
|
||||||
// edit form drift apart in the first place. Every input is submitted
|
|
||||||
// to both and the accept/reject verdicts are compared.
|
|
||||||
func TestTargetAttempts_CreateAndEditAgreeOnEveryCase(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
|
|
||||||
accepted := []string{
|
|
||||||
"1", "2",
|
|
||||||
strconv.Itoa(handlers.MaxTargetAttemptsForTest),
|
|
||||||
}
|
|
||||||
overCeiling := overCeilingAttempts()
|
|
||||||
belowOne := belowOneAttempts()
|
|
||||||
unparseable := unparseableAttempts()
|
|
||||||
|
|
||||||
cases := make(
|
|
||||||
[]string, 0,
|
|
||||||
len(accepted)+len(overCeiling)+len(belowOne)+
|
|
||||||
len(unparseable),
|
|
||||||
)
|
|
||||||
cases = append(cases, accepted...)
|
|
||||||
cases = append(cases, overCeiling...)
|
|
||||||
cases = append(cases, belowOne...)
|
|
||||||
cases = append(cases, unparseable...)
|
|
||||||
|
|
||||||
for _, attempts := range cases {
|
|
||||||
_, createCode, _ := createWithAttempts(t, env, attempts)
|
|
||||||
|
|
||||||
webhook, target := seedAttemptsTarget(t, env)
|
|
||||||
editCode := submitTargetEdit(
|
|
||||||
env, webhook.ID, target.ID,
|
|
||||||
editAttemptsForm(attempts),
|
|
||||||
).Code
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
createCode == http.StatusBadRequest,
|
|
||||||
editCode == http.StatusBadRequest,
|
|
||||||
"create and edit must agree on max_attempts=%q "+
|
|
||||||
"(create %d, edit %d)",
|
|
||||||
attempts, createCode, editCode,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -9,7 +9,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
"github.com/go-chi/chi"
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
)
|
)
|
||||||
@@ -28,8 +27,8 @@ func (h *Handlers) HandleTargetCreate() http.HandlerFunc {
|
|||||||
|
|
||||||
sourceID := chi.URLParam(r, "sourceID")
|
sourceID := chi.URLParam(r, "sourceID")
|
||||||
|
|
||||||
h.archives.RenameLock().Lock()
|
h.renameMu.Lock()
|
||||||
defer h.archives.RenameLock().Unlock()
|
defer h.renameMu.Unlock()
|
||||||
|
|
||||||
var webhook database.Webhook
|
var webhook database.Webhook
|
||||||
|
|
||||||
@@ -106,7 +105,6 @@ func (h *Handlers) newTarget(
|
|||||||
WebhookID: webhookID,
|
WebhookID: webhookID,
|
||||||
Type: in.Type,
|
Type: in.Type,
|
||||||
Active: true,
|
Active: true,
|
||||||
MaxAttempts: 1,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
errMsg, err := h.setTargetFromForm(ctx, target, in)
|
errMsg, err := h.setTargetFromForm(ctx, target, in)
|
||||||
@@ -119,10 +117,9 @@ func (h *Handlers) newTarget(
|
|||||||
|
|
||||||
// setTargetFromForm validates a target form against the target's type
|
// setTargetFromForm validates a target form against the target's type
|
||||||
// and, when it accepts it, sets the target's name, configuration and
|
// and, when it accepts it, sets the target's name, configuration and
|
||||||
// number of delivery attempts from it. It returns the message the form
|
// retry count from it. It returns the message the form shows for
|
||||||
// shows for anything it refuses, an unknown type among them, and then
|
// anything it refuses, an unknown type among them, and then leaves the
|
||||||
// leaves the target unchanged; an error is the server's fault, as for
|
// target unchanged; an error is the server's fault, as for newTarget.
|
||||||
// newTarget.
|
|
||||||
// The add target form and the target edit form both go through here,
|
// The add target form and the target edit form both go through here,
|
||||||
// so the two cannot come to disagree about what a target may be.
|
// so the two cannot come to disagree about what a target may be.
|
||||||
func (h *Handlers) setTargetFromForm(
|
func (h *Handlers) setTargetFromForm(
|
||||||
@@ -139,20 +136,20 @@ func (h *Handlers) setTargetFromForm(
|
|||||||
return errMsg, err
|
return errMsg, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// An empty max_attempts keeps the target's count: the default of
|
// An empty max_retries keeps the target's count: the
|
||||||
// 1 for a new target, and the stored count for an edited one,
|
// fire-and-forget default of 0 for a new target, and the stored
|
||||||
// since the forms for target types that do not retry have no such
|
// count for an edited one, since the forms for target types that
|
||||||
// field. A value that is filled in but invalid is refused rather
|
// do not retry have no such field. A value that is filled in but
|
||||||
// than becoming that count, so a typo cannot destroy the count a
|
// invalid is refused rather than becoming that count, so a typo
|
||||||
// target is delivering with.
|
// cannot destroy the count a target is delivering with.
|
||||||
maxAttempts, err := parseMaxAttempts(in.MaxAttempts, target.MaxAttempts)
|
maxRetries, err := parseMaxRetries(in.MaxRetries, target.MaxRetries)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "Invalid delivery attempts: " + attemptsErrorMessage(err), nil
|
return "Invalid delivery attempts: " + retriesErrorMessage(err), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
target.Name = in.Name
|
target.Name = in.Name
|
||||||
target.Config = configJSON
|
target.Config = configJSON
|
||||||
target.MaxAttempts = maxAttempts
|
target.MaxRetries = maxRetries
|
||||||
|
|
||||||
return "", nil
|
return "", nil
|
||||||
}
|
}
|
||||||
@@ -179,8 +176,8 @@ type targetFormInput struct {
|
|||||||
// ForwardQuery is an HTTP target's checkbox that passes each
|
// ForwardQuery is an HTTP target's checkbox that passes each
|
||||||
// event's query string on to it.
|
// event's query string on to it.
|
||||||
ForwardQuery bool
|
ForwardQuery bool
|
||||||
// MaxAttempts is an HTTP or Slack target's max_attempts.
|
// MaxRetries is an HTTP or Slack target's max_retries.
|
||||||
MaxAttempts string
|
MaxRetries string
|
||||||
// Expiry is a database (archive) target's row expiry.
|
// Expiry is a database (archive) target's row expiry.
|
||||||
Expiry string
|
Expiry string
|
||||||
// Rotation is a database (archive) target's rotation.
|
// Rotation is a database (archive) target's rotation.
|
||||||
@@ -207,7 +204,7 @@ func targetFormInputFrom(r *http.Request) targetFormInput {
|
|||||||
Headers: r.PostFormValue("headers"),
|
Headers: r.PostFormValue("headers"),
|
||||||
Timeout: r.PostFormValue("timeout"),
|
Timeout: r.PostFormValue("timeout"),
|
||||||
ForwardQuery: r.PostFormValue("forward_query") != "",
|
ForwardQuery: r.PostFormValue("forward_query") != "",
|
||||||
MaxAttempts: r.PostFormValue("max_attempts"),
|
MaxRetries: r.PostFormValue("max_retries"),
|
||||||
Expiry: r.PostFormValue("expiry"),
|
Expiry: r.PostFormValue("expiry"),
|
||||||
Rotation: r.PostFormValue("rotation"),
|
Rotation: r.PostFormValue("rotation"),
|
||||||
}
|
}
|
||||||
@@ -362,12 +359,12 @@ func buildDatabaseTargetConfig(
|
|||||||
) (string, string, error) {
|
) (string, string, error) {
|
||||||
expiry = strings.TrimSpace(expiry)
|
expiry = strings.TrimSpace(expiry)
|
||||||
|
|
||||||
err := archive.ValidateExpiry(expiry)
|
err := delivery.ValidateArchiveExpiry(expiry)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Sprintf("Invalid archive expiry: %v", err), nil
|
return "", fmt.Sprintf("Invalid archive expiry: %v", err), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
err = archive.ValidateRotation(rotation)
|
err = delivery.ValidateArchiveRotation(rotation)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Sprintf("Invalid archive rotation: %v", err), nil
|
return "", fmt.Sprintf("Invalid archive rotation: %v", err), nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -196,7 +196,7 @@ func TestHandleTargetCreate_QueryStringCannotSupplyNameOrType(
|
|||||||
|
|
||||||
w, _ := postTargetCreate(
|
w, _ := postTargetCreate(
|
||||||
t, env, webhook.ID,
|
t, env, webhook.ID,
|
||||||
"name=leaky&type=slack&max_attempts=9&expiry=30d&url="+
|
"name=leaky&type=slack&max_retries=9&expiry=30d&url="+
|
||||||
url.QueryEscape(targetSecretURL),
|
url.QueryEscape(targetSecretURL),
|
||||||
url.Values{},
|
url.Values{},
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -24,21 +24,21 @@ func TestHandleTargetCreate_EveryType(t *testing.T) {
|
|||||||
targetType database.TargetType
|
targetType database.TargetType
|
||||||
fields string
|
fields string
|
||||||
wantConfig string
|
wantConfig string
|
||||||
wantAttempts int
|
wantRetries int
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
database.TargetTypeHTTP, "timeout=12&max_attempts=3",
|
database.TargetTypeHTTP, "timeout=12&max_retries=3",
|
||||||
`{"url":"` + editOriginalURL + `","timeout":12}`, 3,
|
`{"url":"` + editOriginalURL + `","timeout":12}`, 3,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
database.TargetTypeSlack, "max_attempts=4",
|
database.TargetTypeSlack, "max_retries=4",
|
||||||
`{"webhookUrl":"` + editOriginalURL + `"}`, 4,
|
`{"webhookUrl":"` + editOriginalURL + `"}`, 4,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
database.TargetTypeDatabase, "expiry=720h",
|
database.TargetTypeDatabase, "expiry=720h",
|
||||||
`{"expiry":"720h"}`, 1,
|
`{"expiry":"720h"}`, 0,
|
||||||
},
|
},
|
||||||
{database.TargetTypeLog, "", "", 1},
|
{database.TargetTypeLog, "", "", 0},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range cases {
|
for _, tc := range cases {
|
||||||
@@ -62,7 +62,7 @@ func TestHandleTargetCreate_EveryType(t *testing.T) {
|
|||||||
targets := targetsForWebhook(t, env.db, webhook.ID)
|
targets := targetsForWebhook(t, env.db, webhook.ID)
|
||||||
require.Len(t, targets, 1)
|
require.Len(t, targets, 1)
|
||||||
assert.Equal(t, tc.targetType, targets[0].Type)
|
assert.Equal(t, tc.targetType, targets[0].Type)
|
||||||
assert.Equal(t, tc.wantAttempts, targets[0].MaxAttempts)
|
assert.Equal(t, tc.wantRetries, targets[0].MaxRetries)
|
||||||
|
|
||||||
if tc.wantConfig == "" {
|
if tc.wantConfig == "" {
|
||||||
assert.Empty(t, targets[0].Config)
|
assert.Empty(t, targets[0].Config)
|
||||||
@@ -90,11 +90,11 @@ func TestHandleTargetCreate_RefusedFormComesBack(t *testing.T) {
|
|||||||
{
|
{
|
||||||
database.TargetTypeHTTP,
|
database.TargetTypeHTTP,
|
||||||
"name=private&url=" + editBlockedURL +
|
"name=private&url=" + editBlockedURL +
|
||||||
"&timeout=12&max_attempts=3",
|
"&timeout=12&max_retries=3",
|
||||||
"Invalid target URL",
|
"Invalid target URL",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
database.TargetTypeSlack, "name=no-url&max_attempts=4",
|
database.TargetTypeSlack, "name=no-url&max_retries=4",
|
||||||
"Webhook URL is required for Slack targets",
|
"Webhook URL is required for Slack targets",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -133,8 +133,8 @@ func TestHandleTargetCreate_RefusedFormComesBack(t *testing.T) {
|
|||||||
assert.Contains(t, page, html.EscapeString(tc.reason))
|
assert.Contains(t, page, html.EscapeString(tc.reason))
|
||||||
|
|
||||||
// Each value comes back in a data attribute of the targets
|
// Each value comes back in a data attribute of the targets
|
||||||
// section named after its field (max_attempts as
|
// section named after its field (max_retries as
|
||||||
// data-max-attempts), except url, which comes back in
|
// data-max-retries), except url, which comes back in
|
||||||
// data-destination; templates/source_detail.html says why.
|
// data-destination; templates/source_detail.html says why.
|
||||||
for field := range typed {
|
for field := range typed {
|
||||||
attr := "data-" + strings.ReplaceAll(field, "_", "-")
|
attr := "data-" + strings.ReplaceAll(field, "_", "-")
|
||||||
|
|||||||
@@ -6,8 +6,8 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// downloadWriteTimeout is how long one write of a download may wait
|
// downloadWriteTimeout is how long one write of a download may wait
|
||||||
@@ -16,7 +16,7 @@ const downloadWriteTimeout = 60 * time.Second
|
|||||||
|
|
||||||
// HandleTargetDownload serves a database target's archive as one
|
// HandleTargetDownload serves a database target's archive as one
|
||||||
// gzipped JSON file, named for the webhook, the target and the time;
|
// gzipped JSON file, named for the webhook, the target and the time;
|
||||||
// see archive.Export.WriteGzipJSON for what it holds. Other
|
// see delivery.ArchiveExport.WriteGzipJSON for what it holds. Other
|
||||||
// target types have no archive and are a 404.
|
// target types have no archive and are a 404.
|
||||||
//
|
//
|
||||||
// A download runs for as long as the client keeps reading: it reads
|
// A download runs for as long as the client keeps reading: it reads
|
||||||
@@ -37,7 +37,7 @@ func (h *Handlers) HandleTargetDownload() http.HandlerFunc {
|
|||||||
w.Header().Set("Content-Type", "application/gzip")
|
w.Header().Set("Content-Type", "application/gzip")
|
||||||
w.Header().Set(
|
w.Header().Set(
|
||||||
"Content-Disposition",
|
"Content-Disposition",
|
||||||
`attachment; filename="`+archive.ExportFileName(
|
`attachment; filename="`+delivery.ArchiveExportFileName(
|
||||||
webhook.Name, target.Name, now,
|
webhook.Name, target.Name, now,
|
||||||
)+`"`,
|
)+`"`,
|
||||||
)
|
)
|
||||||
@@ -82,17 +82,17 @@ func (d downloadWriter) Write(b []byte) (int, error) {
|
|||||||
// listTargetArchive lists the archive files of the request's database
|
// listTargetArchive lists the archive files of the request's database
|
||||||
// target for export. It reports false once it has written the response.
|
// target for export. It reports false once it has written the response.
|
||||||
//
|
//
|
||||||
// It holds the rename lock, which every archive rename runs under,
|
// It holds renameMu, which every archive rename runs under, while it
|
||||||
// while it reads the stored names and lists the files, so the files it
|
// reads the stored names and lists the files, so the files it lists
|
||||||
// lists are the ones those names give. It lets go before the export is
|
// are the ones those names give. It lets go before the export is
|
||||||
// streamed, which takes the lock again for each file only while it
|
// streamed, which takes renameMu again for each file only while it
|
||||||
// finds the file under the names stored then and opens it.
|
// finds the file under the names stored then and opens it.
|
||||||
func (h *Handlers) listTargetArchive(
|
func (h *Handlers) listTargetArchive(
|
||||||
w http.ResponseWriter,
|
w http.ResponseWriter,
|
||||||
r *http.Request,
|
r *http.Request,
|
||||||
) (database.Webhook, *database.Target, *archive.Export, bool) {
|
) (database.Webhook, *database.Target, *delivery.ArchiveExport, bool) {
|
||||||
h.archives.RenameLock().Lock()
|
h.renameMu.Lock()
|
||||||
defer h.archives.RenameLock().Unlock()
|
defer h.renameMu.Unlock()
|
||||||
|
|
||||||
webhook, target, ok := h.ownedTarget(w, r)
|
webhook, target, ok := h.ownedTarget(w, r)
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -105,9 +105,9 @@ func (h *Handlers) listTargetArchive(
|
|||||||
return database.Webhook{}, nil, nil, false
|
return database.Webhook{}, nil, nil, false
|
||||||
}
|
}
|
||||||
|
|
||||||
export, err := archive.NewExport(
|
export, err := delivery.NewArchiveExport(
|
||||||
archive.Path(h.dbMgr, &webhook, target),
|
delivery.ArchivePath(h.dbMgr, &webhook, target),
|
||||||
h.archives.RenameLock(),
|
&h.renameMu,
|
||||||
func() (string, error) {
|
func() (string, error) {
|
||||||
return h.storedArchivePath(webhook.ID, target.ID)
|
return h.storedArchivePath(webhook.ID, target.ID)
|
||||||
},
|
},
|
||||||
@@ -122,9 +122,9 @@ func (h *Handlers) listTargetArchive(
|
|||||||
return webhook, target, export, true
|
return webhook, target, export, true
|
||||||
}
|
}
|
||||||
|
|
||||||
// storedArchivePath returns the path archive.Path gives a
|
// storedArchivePath returns the path delivery.ArchivePath gives a
|
||||||
// database target under the names stored for it and its webhook now.
|
// database target under the names stored for it and its webhook now.
|
||||||
// Its caller holds the rename lock. A webhook or target deleted since is still
|
// Its caller holds renameMu. A webhook or target deleted since is still
|
||||||
// found, since deleting one leaves its archive files under their names.
|
// found, since deleting one leaves its archive files under their names.
|
||||||
func (h *Handlers) storedArchivePath(
|
func (h *Handlers) storedArchivePath(
|
||||||
webhookID, targetID string,
|
webhookID, targetID string,
|
||||||
@@ -143,5 +143,5 @@ func (h *Handlers) storedArchivePath(
|
|||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
return archive.Path(h.dbMgr, &webhook, &target), nil
|
return delivery.ArchivePath(h.dbMgr, &webhook, &target), nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,9 +21,9 @@ import (
|
|||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
"sneak.berlin/go/webhooker/internal/middleware/middlewaretest"
|
"sneak.berlin/go/webhooker/internal/middleware/middlewaretest"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -223,8 +223,8 @@ func TestHandleTargetDownload_FindsFilesAfterRename(t *testing.T) {
|
|||||||
|
|
||||||
env := setupSourceTest(t)
|
env := setupSourceTest(t)
|
||||||
wh := seedWebhookWithRetention(t, env.db, 7)
|
wh := seedWebhookWithRetention(t, env.db, 7)
|
||||||
target := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
archive := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
||||||
oldPath := archive.Path(env.dbMgr, &wh, target)
|
oldPath := delivery.ArchivePath(env.dbMgr, &wh, archive)
|
||||||
|
|
||||||
month := func(path string) string {
|
month := func(path string) string {
|
||||||
return strings.TrimSuffix(path, ".db") + "-2026-10.db"
|
return strings.TrimSuffix(path, ".db") + "-2026-10.db"
|
||||||
@@ -233,19 +233,19 @@ func TestHandleTargetDownload_FindsFilesAfterRename(t *testing.T) {
|
|||||||
seedArchive(t, oldPath, 1, 16)
|
seedArchive(t, oldPath, 1, 16)
|
||||||
seedArchive(t, month(oldPath), 1, 16)
|
seedArchive(t, month(oldPath), 1, 16)
|
||||||
|
|
||||||
sw, downloaded := startStalledDownload(t, env, wh.ID, target.ID)
|
sw, downloaded := startStalledDownload(t, env, wh.ID, archive.ID)
|
||||||
|
|
||||||
require.Equal(t,
|
require.Equal(t,
|
||||||
http.StatusSeeOther, renameTarget(env, wh.ID, target.ID).Code,
|
http.StatusSeeOther, renameTarget(env, wh.ID, archive.ID).Code,
|
||||||
)
|
)
|
||||||
|
|
||||||
// The test's archives record a rename without moving any file, so
|
// The test's archives record a rename without moving any file, so
|
||||||
// the files are moved here, as the archive writers move them.
|
// the files are moved here, as the delivery engine moves them.
|
||||||
var renamed database.Target
|
var renamed database.Target
|
||||||
|
|
||||||
require.NoError(t, env.db.DB().First(&renamed, "id = ?", target.ID).Error)
|
require.NoError(t, env.db.DB().First(&renamed, "id = ?", archive.ID).Error)
|
||||||
|
|
||||||
newPath := archive.Path(env.dbMgr, &wh, &renamed)
|
newPath := delivery.ArchivePath(env.dbMgr, &wh, &renamed)
|
||||||
|
|
||||||
require.NoError(t, os.Rename(oldPath, newPath))
|
require.NoError(t, os.Rename(oldPath, newPath))
|
||||||
require.NoError(t, os.Rename(month(oldPath), month(newPath)))
|
require.NoError(t, os.Rename(month(oldPath), month(newPath)))
|
||||||
@@ -355,16 +355,16 @@ func TestHandleTargetDownload_OutlastsTheRequestLimit(t *testing.T) {
|
|||||||
|
|
||||||
env := setupSourceTest(t)
|
env := setupSourceTest(t)
|
||||||
wh := seedWebhookWithRetention(t, env.db, 7)
|
wh := seedWebhookWithRetention(t, env.db, 7)
|
||||||
target := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
archive := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
||||||
seedArchive(
|
seedArchive(
|
||||||
t, archive.Path(env.dbMgr, &wh, target), rows, bodySize,
|
t, delivery.ArchivePath(env.dbMgr, &wh, archive), rows, bodySize,
|
||||||
)
|
)
|
||||||
|
|
||||||
srv, accessLog := limitedServer(t, env, limit)
|
srv, accessLog := limitedServer(t, env, limit)
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(
|
req, err := http.NewRequestWithContext(
|
||||||
t.Context(), http.MethodGet,
|
t.Context(), http.MethodGet,
|
||||||
srv.URL+downloadPath(wh.ID, target.ID), nil,
|
srv.URL+downloadPath(wh.ID, archive.ID), nil,
|
||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
"github.com/go-chi/chi"
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
)
|
)
|
||||||
@@ -83,7 +82,7 @@ func (h *Handlers) HandleTargetEdit() http.HandlerFunc {
|
|||||||
Headers: cfg.Headers,
|
Headers: cfg.Headers,
|
||||||
Timeout: cfg.Timeout,
|
Timeout: cfg.Timeout,
|
||||||
ForwardQuery: cfg.ForwardQuery,
|
ForwardQuery: cfg.ForwardQuery,
|
||||||
MaxAttempts: strconv.Itoa(target.MaxAttempts),
|
MaxRetries: strconv.Itoa(target.MaxRetries),
|
||||||
Expiry: cfg.Expiry,
|
Expiry: cfg.Expiry,
|
||||||
Rotation: cfg.Rotation,
|
Rotation: cfg.Rotation,
|
||||||
}
|
}
|
||||||
@@ -97,8 +96,8 @@ func (h *Handlers) HandleTargetEdit() http.HandlerFunc {
|
|||||||
// HandleTargetEditSubmit handles the target edit form submission.
|
// HandleTargetEditSubmit handles the target edit form submission.
|
||||||
func (h *Handlers) HandleTargetEditSubmit() http.HandlerFunc {
|
func (h *Handlers) HandleTargetEditSubmit() http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
h.archives.RenameLock().Lock()
|
h.renameMu.Lock()
|
||||||
defer h.archives.RenameLock().Unlock()
|
defer h.renameMu.Unlock()
|
||||||
|
|
||||||
webhook, target, ok := h.ownedTarget(w, r)
|
webhook, target, ok := h.ownedTarget(w, r)
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -158,7 +157,7 @@ func (h *Handlers) applyTargetEdit(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// A new name renames the archive file before it is saved (see
|
// A new name renames the archive file before it is saved (see
|
||||||
// archive.Writers.Rename). If either step fails, it goes back to
|
// delivery.Engine.Rename). If either step fails, it goes back to
|
||||||
// the name that is still stored.
|
// the name that is still stored.
|
||||||
err = h.renameTargetArchive(
|
err = h.renameTargetArchive(
|
||||||
target, webhook.Name, target.Name, edited.Name,
|
target, webhook.Name, target.Name, edited.Name,
|
||||||
@@ -179,7 +178,7 @@ func (h *Handlers) applyTargetEdit(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if errors.Is(err, archive.ErrNameTaken) {
|
if errors.Is(err, delivery.ErrArchiveNameTaken) {
|
||||||
h.renderTargetEdit(
|
h.renderTargetEdit(
|
||||||
w, r, webhook, target, in,
|
w, r, webhook, target, in,
|
||||||
"Not saved: "+err.Error()+
|
"Not saved: "+err.Error()+
|
||||||
|
|||||||
@@ -117,7 +117,7 @@ func seedHTTPTarget(
|
|||||||
form.Set("url", editOriginalURL)
|
form.Set("url", editOriginalURL)
|
||||||
form.Set("headers", headers)
|
form.Set("headers", headers)
|
||||||
form.Set("timeout", timeout)
|
form.Set("timeout", timeout)
|
||||||
form.Set("max_attempts", "3")
|
form.Set("max_retries", "3")
|
||||||
|
|
||||||
w := serveTarget(
|
w := serveTarget(
|
||||||
env, http.MethodPost,
|
env, http.MethodPost,
|
||||||
@@ -179,7 +179,7 @@ func editForm(targetURL, headers, timeout string) url.Values {
|
|||||||
form.Set("url", targetURL)
|
form.Set("url", targetURL)
|
||||||
form.Set("headers", headers)
|
form.Set("headers", headers)
|
||||||
form.Set("timeout", timeout)
|
form.Set("timeout", timeout)
|
||||||
form.Set("max_attempts", "5")
|
form.Set("max_retries", "5")
|
||||||
|
|
||||||
return form
|
return form
|
||||||
}
|
}
|
||||||
@@ -280,9 +280,9 @@ func TestHandleTargetEditSubmit_Saves(t *testing.T) {
|
|||||||
assertEditClearingRemovesThem(t, env)
|
assertEditClearingRemovesThem(t, env)
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("absent max_attempts is kept", func(t *testing.T) {
|
t.Run("absent max_retries is not zeroed", func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
assertEditKeepsAbsentMaxAttempts(t, env)
|
assertEditKeepsAbsentMaxRetries(t, env)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -307,7 +307,7 @@ func assertEditChangesDestination(
|
|||||||
|
|
||||||
reloaded := storedTarget(t, env, target.ID)
|
reloaded := storedTarget(t, env, target.ID)
|
||||||
assert.Equal(t, "edited-name", reloaded.Name)
|
assert.Equal(t, "edited-name", reloaded.Name)
|
||||||
assert.Equal(t, 5, reloaded.MaxAttempts)
|
assert.Equal(t, 5, reloaded.MaxRetries)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, database.TargetTypeHTTP, reloaded.Type,
|
t, database.TargetTypeHTTP, reloaded.Type,
|
||||||
"the edit form must not change a target's type",
|
"the edit form must not change a target's type",
|
||||||
@@ -368,25 +368,25 @@ func assertEditClearingRemovesThem(
|
|||||||
assert.Zero(t, cfg.Timeout)
|
assert.Zero(t, cfg.Timeout)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delivery attempts are offered only by the forms for target types
|
// Retries are offered only by the forms for target types that retry.
|
||||||
// that retry. An absent field means the form does not edit them, not
|
// An absent field means the form does not edit retries, not that
|
||||||
// that retries should be turned off.
|
// they should be turned off.
|
||||||
func assertEditKeepsAbsentMaxAttempts(
|
func assertEditKeepsAbsentMaxRetries(
|
||||||
t *testing.T, env *sourceTestEnv,
|
t *testing.T, env *sourceTestEnv,
|
||||||
) {
|
) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
webhook, target := seedHTTPTarget(t, env, "", "")
|
webhook, target := seedHTTPTarget(t, env, "", "")
|
||||||
require.Equal(t, 3, target.MaxAttempts)
|
require.Equal(t, 3, target.MaxRetries)
|
||||||
|
|
||||||
form := editForm(editOriginalURL, "", "")
|
form := editForm(editOriginalURL, "", "")
|
||||||
form.Del("max_attempts")
|
form.Del("max_retries")
|
||||||
|
|
||||||
w := submitTargetEdit(env, webhook.ID, target.ID, form)
|
w := submitTargetEdit(env, webhook.ID, target.ID, form)
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, 3, storedTarget(t, env, target.ID).MaxAttempts,
|
t, 3, storedTarget(t, env, target.ID).MaxRetries,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -659,12 +659,12 @@ func TestHandleTargetEditSubmit_RefusedFormComesBack(t *testing.T) {
|
|||||||
{
|
{
|
||||||
database.TargetTypeHTTP,
|
database.TargetTypeHTTP,
|
||||||
"name=edited&url=" + editBlockedURL +
|
"name=edited&url=" + editBlockedURL +
|
||||||
"&headers=X-Edited:+kept&timeout=12&max_attempts=3",
|
"&headers=X-Edited:+kept&timeout=12&max_retries=3",
|
||||||
"Invalid target URL",
|
"Invalid target URL",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
database.TargetTypeSlack,
|
database.TargetTypeSlack,
|
||||||
"name=edited&url=" + editOriginalURL + "&max_attempts=25",
|
"name=edited&url=" + editOriginalURL + "&max_retries=25",
|
||||||
"Invalid delivery attempts",
|
"Invalid delivery attempts",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ import (
|
|||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
)
|
)
|
||||||
@@ -243,7 +242,7 @@ func readTargetDeliveries(
|
|||||||
func (h *Handlers) archiveFileView(
|
func (h *Handlers) archiveFileView(
|
||||||
webhook *database.Webhook, target *database.Target, now time.Time,
|
webhook *database.Webhook, target *database.Target, now time.Time,
|
||||||
) *ArchiveFileView {
|
) *ArchiveFileView {
|
||||||
current, err := archive.PathAt(h.dbMgr, webhook, target, now)
|
current, err := delivery.ArchivePathAt(h.dbMgr, webhook, target, now)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return h.archiveUnreadable(&ArchiveFileView{}, target, err)
|
return h.archiveUnreadable(&ArchiveFileView{}, target, err)
|
||||||
}
|
}
|
||||||
@@ -255,8 +254,8 @@ func (h *Handlers) archiveFileView(
|
|||||||
view.Note = "not created yet"
|
view.Note = "not created yet"
|
||||||
}
|
}
|
||||||
|
|
||||||
files, err := archive.Stat(
|
files, err := delivery.StatArchive(
|
||||||
archive.Path(h.dbMgr, webhook, target),
|
delivery.ArchivePath(h.dbMgr, webhook, target),
|
||||||
)
|
)
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
|
|||||||
@@ -10,8 +10,8 @@ import (
|
|||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
"sneak.berlin/go/webhooker/internal/handlers"
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
"sneak.berlin/go/webhooker/internal/session"
|
||||||
@@ -38,10 +38,10 @@ func TestHandleSourceDetail_ShowsArchiveFile(t *testing.T) {
|
|||||||
t.Cleanup(app.RequireStop)
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
wh := seedWebhook(t, db)
|
||||||
target := seedTarget(t, db, wh.ID, database.TargetTypeDatabase)
|
archive := seedTarget(t, db, wh.ID, database.TargetTypeDatabase)
|
||||||
seedTarget(t, db, wh.ID, database.TargetTypeLog)
|
seedTarget(t, db, wh.ID, database.TargetTypeLog)
|
||||||
|
|
||||||
path := archive.Path(dbMgr, wh, target)
|
path := delivery.ArchivePath(dbMgr, wh, archive)
|
||||||
|
|
||||||
body := renderSourceDetailPage(t, h, sess, wh.ID)
|
body := renderSourceDetailPage(t, h, sess, wh.ID)
|
||||||
assert.Equal(t, 1, strings.Count(body, "Archive file:"))
|
assert.Equal(t, 1, strings.Count(body, "Archive file:"))
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
package handlers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// maxTargetRetries bounds a target's max_retries.
|
||||||
|
//
|
||||||
|
// Both target forms already declare max="20" on the input, so this
|
||||||
|
// enforces server-side what the UI has always advertised rather than
|
||||||
|
// introducing a new limit.
|
||||||
|
//
|
||||||
|
// The number is not cosmetic. Every attempt writes a delivery_results
|
||||||
|
// row that the event log then loads and renders, and the engine backs
|
||||||
|
// off by 2^(n-1) seconds, so attempt 20 is already about six days
|
||||||
|
// after the first. A value beyond this buys no additional durability
|
||||||
|
// and only costs rows.
|
||||||
|
const maxTargetRetries = 20
|
||||||
|
|
||||||
|
// Errors returned when a max_retries form value cannot be turned into
|
||||||
|
// a retry count.
|
||||||
|
var (
|
||||||
|
// errRetriesInvalid signals a max_retries form value that is not
|
||||||
|
// a non-negative whole number.
|
||||||
|
errRetriesInvalid = errors.New(
|
||||||
|
"must be a whole number",
|
||||||
|
)
|
||||||
|
|
||||||
|
// errRetriesTooLarge signals a max_retries form value that is a
|
||||||
|
// whole number but above maxTargetRetries. It is distinguished
|
||||||
|
// from errRetriesInvalid so the message can name the ceiling
|
||||||
|
// instead of implying the input was not a number.
|
||||||
|
errRetriesTooLarge = errors.New("out of range")
|
||||||
|
)
|
||||||
|
|
||||||
|
// parseMaxRetries interprets a max_retries form value.
|
||||||
|
//
|
||||||
|
// An ABSENT value — the field empty or not submitted — yields
|
||||||
|
// fallback, which lets the create path apply its default and the edit
|
||||||
|
// path leave the stored value alone. A value that is SET BUT INVALID
|
||||||
|
// is an error: unparseable, negative, or above maxTargetRetries.
|
||||||
|
//
|
||||||
|
// The distinction is the whole point of this function. max_retries=0
|
||||||
|
// means fire-and-forget, so returning 0 for input the operator typed
|
||||||
|
// but that did not parse silently disables retries on a
|
||||||
|
// store-and-forward proxy — and on the edit path it destroys a
|
||||||
|
// working retry configuration over a typo. A default answers a
|
||||||
|
// question that was not asked; it never answers one that was asked
|
||||||
|
// badly.
|
||||||
|
func parseMaxRetries(raw string, fallback int) (int, error) {
|
||||||
|
raw = strings.TrimSpace(raw)
|
||||||
|
if raw == "" {
|
||||||
|
return fallback, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
v, err := strconv.Atoi(raw)
|
||||||
|
if err != nil || v < 0 {
|
||||||
|
return 0, errRetriesInvalid
|
||||||
|
}
|
||||||
|
|
||||||
|
if v > maxTargetRetries {
|
||||||
|
return 0, errRetriesTooLarge
|
||||||
|
}
|
||||||
|
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// retriesErrorMessage returns the message the create and edit forms
|
||||||
|
// show for a rejected max_retries value. Any error other than
|
||||||
|
// errRetriesTooLarge falls back to the generic wording, so an
|
||||||
|
// unrecognised parse failure still produces a sensible 400.
|
||||||
|
func retriesErrorMessage(err error) string {
|
||||||
|
if errors.Is(err, errRetriesTooLarge) {
|
||||||
|
return errRetriesTooLarge.Error() +
|
||||||
|
": at most " + strconv.Itoa(maxTargetRetries) +
|
||||||
|
" attempts"
|
||||||
|
}
|
||||||
|
|
||||||
|
return errRetriesInvalid.Error() +
|
||||||
|
", or 0 for fire-and-forget"
|
||||||
|
}
|
||||||
@@ -0,0 +1,385 @@
|
|||||||
|
package handlers_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strconv"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
|
"sneak.berlin/go/webhooker/internal/handlers"
|
||||||
|
)
|
||||||
|
|
||||||
|
// retriesTargetURL is the destination the retry-validation targets
|
||||||
|
// point at. It is a literal public address rather than a hostname so
|
||||||
|
// the SSRF check resolves nothing and a sandbox without DNS cannot
|
||||||
|
// make these cases pass or fail for the wrong reason.
|
||||||
|
const retriesTargetURL = "https://93.184.216.34/hooks/retries"
|
||||||
|
|
||||||
|
const (
|
||||||
|
// wayAboveCeiling is the typo'd-extra-zero case from the report.
|
||||||
|
wayAboveCeiling = "999999999"
|
||||||
|
|
||||||
|
// notANumber is the plainest garbage an operator can type, and
|
||||||
|
// the value the report submitted on the edit form.
|
||||||
|
notANumber = "abc"
|
||||||
|
|
||||||
|
// workingRetries is the retry count a seeded target is already
|
||||||
|
// delivering with, which a rejected submission must not disturb.
|
||||||
|
workingRetries = 2
|
||||||
|
)
|
||||||
|
|
||||||
|
// aboveCeiling is the smallest rejected whole number.
|
||||||
|
func aboveCeiling() string {
|
||||||
|
return strconv.Itoa(handlers.MaxTargetRetriesForTest + 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// overCeilingRetries is whole-number input past the limit, which is
|
||||||
|
// rejected with the limit named.
|
||||||
|
func overCeilingRetries() []string {
|
||||||
|
return []string{aboveCeiling(), wayAboveCeiling}
|
||||||
|
}
|
||||||
|
|
||||||
|
// unparseableRetries is input an operator can type into the field
|
||||||
|
// that is not a retry count. Each must be REJECTED: silently reading
|
||||||
|
// any of them as 0 turns a store-and-forward proxy into
|
||||||
|
// fire-and-forget without saying so.
|
||||||
|
//
|
||||||
|
// The twenty-digit case is here because it parses as digits but
|
||||||
|
// overflows int, which is the one failure the field's own min/max
|
||||||
|
// attributes cannot describe.
|
||||||
|
func unparseableRetries() []string {
|
||||||
|
return []string{
|
||||||
|
notANumber,
|
||||||
|
"2.7",
|
||||||
|
"-5",
|
||||||
|
"12345678901234567890",
|
||||||
|
"1e3",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// createRetriesForm is a complete, otherwise-valid HTTP target
|
||||||
|
// creation, so the only thing any case below varies is max_retries.
|
||||||
|
func createRetriesForm(retries string) url.Values {
|
||||||
|
form := url.Values{}
|
||||||
|
form.Set("name", "retries-target")
|
||||||
|
form.Set("type", string(database.TargetTypeHTTP))
|
||||||
|
form.Set("url", retriesTargetURL)
|
||||||
|
|
||||||
|
if retries != absentField {
|
||||||
|
form.Set("max_retries", retries)
|
||||||
|
}
|
||||||
|
|
||||||
|
return form
|
||||||
|
}
|
||||||
|
|
||||||
|
// absentField marks a field the form does not submit at all, which is
|
||||||
|
// the case that legitimately takes a default and must stay distinct
|
||||||
|
// from a field submitted with garbage in it.
|
||||||
|
const absentField = "\x00absent"
|
||||||
|
|
||||||
|
// absentRetries is every way of saying "the operator did not set
|
||||||
|
// this", each of which takes the default rather than a 400. Blank and
|
||||||
|
// whitespace-only count as absent here because they do in the timeout
|
||||||
|
// and retention controls on the same forms; a rule the fields do not
|
||||||
|
// share would be its own surprise.
|
||||||
|
func absentRetries() []string {
|
||||||
|
return []string{absentField, "", " "}
|
||||||
|
}
|
||||||
|
|
||||||
|
// createWithRetries posts the target create form for a fresh webhook
|
||||||
|
// and returns the webhook and the response.
|
||||||
|
func createWithRetries(
|
||||||
|
t *testing.T,
|
||||||
|
env *sourceTestEnv,
|
||||||
|
retries string,
|
||||||
|
) (database.Webhook, int, string) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
webhook := seedWebhookWithRetention(t, env.db, 30)
|
||||||
|
|
||||||
|
w := serveTarget(
|
||||||
|
env, http.MethodPost,
|
||||||
|
"/hook/"+webhook.ID+"/targets",
|
||||||
|
createRetriesForm(retries),
|
||||||
|
)
|
||||||
|
|
||||||
|
return webhook, w.Code, w.Body.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTargetCreate_RetriesAboveCeilingRejected proves the create form
|
||||||
|
// enforces a ceiling at all, and that the 400 names it — a rejection
|
||||||
|
// that does not say what the limit is leaves the operator guessing.
|
||||||
|
func TestTargetCreate_RetriesAboveCeilingRejected(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupSourceTest(t)
|
||||||
|
ceiling := strconv.Itoa(handlers.MaxTargetRetriesForTest)
|
||||||
|
|
||||||
|
for _, retries := range overCeilingRetries() {
|
||||||
|
webhook, code, body := createWithRetries(t, env, retries)
|
||||||
|
|
||||||
|
assert.Equal(t, http.StatusBadRequest, code,
|
||||||
|
"max_retries=%s should be rejected", retries)
|
||||||
|
assert.Contains(t, body, ceiling,
|
||||||
|
"the rejection for %s should name the ceiling",
|
||||||
|
retries)
|
||||||
|
assert.Empty(t,
|
||||||
|
targetsForWebhook(t, env.db, webhook.ID),
|
||||||
|
"no target should be created for %s", retries)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTargetCreate_UnparseableRetriesRejected is the core of the
|
||||||
|
// defect: each of these was accepted with HTTP 200 and stored as 0.
|
||||||
|
func TestTargetCreate_UnparseableRetriesRejected(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupSourceTest(t)
|
||||||
|
|
||||||
|
for _, retries := range unparseableRetries() {
|
||||||
|
webhook, code, body := createWithRetries(t, env, retries)
|
||||||
|
|
||||||
|
assert.Equal(t, http.StatusBadRequest, code,
|
||||||
|
"max_retries=%q should be rejected, not coerced",
|
||||||
|
retries)
|
||||||
|
assert.Contains(t, body, "whole number",
|
||||||
|
"the rejection for %q should say why", retries)
|
||||||
|
assert.Empty(t,
|
||||||
|
targetsForWebhook(t, env.db, webhook.ID),
|
||||||
|
"no target should be created for %q", retries)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTargetCreate_ValidRetriesStored covers the accepting half,
|
||||||
|
// including the ceiling itself: a bound that rejects its own limit
|
||||||
|
// would make the advertised maximum unreachable.
|
||||||
|
func TestTargetCreate_ValidRetriesStored(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupSourceTest(t)
|
||||||
|
|
||||||
|
for _, want := range []int{0, 3, handlers.MaxTargetRetriesForTest} {
|
||||||
|
webhook, code, body := createWithRetries(
|
||||||
|
t, env, strconv.Itoa(want),
|
||||||
|
)
|
||||||
|
|
||||||
|
require.Equal(t, http.StatusSeeOther, code, body)
|
||||||
|
|
||||||
|
targets := targetsForWebhook(t, env.db, webhook.ID)
|
||||||
|
require.Len(t, targets, 1)
|
||||||
|
assert.Equal(t, want, targets[0].MaxRetries)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTargetCreate_AbsentRetriesTakesDefault keeps the two cases
|
||||||
|
// distinct. An omitted field is not an operator asking for something
|
||||||
|
// invalid, so it still gets the fire-and-forget default rather than a
|
||||||
|
// 400 — otherwise the fix above would make the form unusable.
|
||||||
|
func TestTargetCreate_AbsentRetriesTakesDefault(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupSourceTest(t)
|
||||||
|
|
||||||
|
for _, retries := range absentRetries() {
|
||||||
|
webhook, code, body := createWithRetries(t, env, retries)
|
||||||
|
|
||||||
|
require.Equal(t, http.StatusSeeOther, code, body)
|
||||||
|
|
||||||
|
targets := targetsForWebhook(t, env.db, webhook.ID)
|
||||||
|
require.Len(t, targets, 1)
|
||||||
|
assert.Equal(t, 0, targets[0].MaxRetries,
|
||||||
|
"an absent max_retries should take the default")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// seedRetriesTarget creates an HTTP target already delivering with
|
||||||
|
// workingRetries retries, through the real create handler.
|
||||||
|
func seedRetriesTarget(
|
||||||
|
t *testing.T,
|
||||||
|
env *sourceTestEnv,
|
||||||
|
) (database.Webhook, database.Target) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
webhook, code, body := createWithRetries(
|
||||||
|
t, env, strconv.Itoa(workingRetries),
|
||||||
|
)
|
||||||
|
require.Equal(t, http.StatusSeeOther, code, body)
|
||||||
|
|
||||||
|
targets := targetsForWebhook(t, env.db, webhook.ID)
|
||||||
|
require.Len(t, targets, 1)
|
||||||
|
require.Equal(t, workingRetries, targets[0].MaxRetries)
|
||||||
|
|
||||||
|
return webhook, targets[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
// editRetriesForm is a complete edit submission that changes the
|
||||||
|
// target's name as well, so a rejected submission can be shown to
|
||||||
|
// have written nothing at all rather than merely to have left
|
||||||
|
// max_retries alone.
|
||||||
|
func editRetriesForm(retries string) url.Values {
|
||||||
|
form := url.Values{}
|
||||||
|
form.Set("name", "renamed-by-edit")
|
||||||
|
form.Set("url", retriesTargetURL)
|
||||||
|
|
||||||
|
if retries != absentField {
|
||||||
|
form.Set("max_retries", retries)
|
||||||
|
}
|
||||||
|
|
||||||
|
return form
|
||||||
|
}
|
||||||
|
|
||||||
|
// assertEditRejectedAndUnchanged submits an edit expected to fail and
|
||||||
|
// checks both halves of the requirement: the 400 explains itself, and
|
||||||
|
// the target it was submitted against is untouched.
|
||||||
|
func assertEditRejectedAndUnchanged(
|
||||||
|
t *testing.T,
|
||||||
|
env *sourceTestEnv,
|
||||||
|
retries, wantReason string,
|
||||||
|
) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
webhook, target := seedRetriesTarget(t, env)
|
||||||
|
|
||||||
|
w := submitTargetEdit(
|
||||||
|
env, webhook.ID, target.ID, editRetriesForm(retries),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert.Equal(t, http.StatusBadRequest, w.Code,
|
||||||
|
"max_retries=%q should be rejected on edit", retries)
|
||||||
|
assert.Contains(t, w.Body.String(), wantReason,
|
||||||
|
"the rejection for %q should say why", retries)
|
||||||
|
|
||||||
|
stored := storedTarget(t, env, target.ID)
|
||||||
|
assert.Equal(t, workingRetries, stored.MaxRetries,
|
||||||
|
"a rejected edit must not destroy the working retry "+
|
||||||
|
"count with %q", retries)
|
||||||
|
assert.Equal(t, "retries-target", stored.Name,
|
||||||
|
"a rejected edit must write nothing at all")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTargetEdit_UnparseableRetriesRejected is the damaging half of
|
||||||
|
// the defect. A target delivering with two retries, re-saved with a
|
||||||
|
// typo in the field, returned 200 and was left with retries disabled
|
||||||
|
// and nothing said.
|
||||||
|
func TestTargetEdit_UnparseableRetriesRejected(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupSourceTest(t)
|
||||||
|
|
||||||
|
for _, retries := range unparseableRetries() {
|
||||||
|
assertEditRejectedAndUnchanged(
|
||||||
|
t, env, retries, "whole number",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTargetEdit_RetriesAboveCeilingRejected proves the ceiling
|
||||||
|
// applies to the edit path too, naming itself, so the two paths
|
||||||
|
// cannot disagree about what is storable.
|
||||||
|
func TestTargetEdit_RetriesAboveCeilingRejected(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupSourceTest(t)
|
||||||
|
ceiling := strconv.Itoa(handlers.MaxTargetRetriesForTest)
|
||||||
|
|
||||||
|
for _, retries := range overCeilingRetries() {
|
||||||
|
assertEditRejectedAndUnchanged(t, env, retries, ceiling)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTargetEdit_ValidRetriesStored covers the accepting half of the
|
||||||
|
// edit path, so the ceiling cannot be enforced by simply refusing
|
||||||
|
// every submission.
|
||||||
|
func TestTargetEdit_ValidRetriesStored(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupSourceTest(t)
|
||||||
|
|
||||||
|
for _, want := range []int{0, 9, handlers.MaxTargetRetriesForTest} {
|
||||||
|
webhook, target := seedRetriesTarget(t, env)
|
||||||
|
|
||||||
|
w := submitTargetEdit(
|
||||||
|
env, webhook.ID, target.ID,
|
||||||
|
editRetriesForm(strconv.Itoa(want)),
|
||||||
|
)
|
||||||
|
require.Equal(t,
|
||||||
|
http.StatusSeeOther, w.Code, w.Body.String(),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert.Equal(t, want,
|
||||||
|
storedTarget(t, env, target.ID).MaxRetries)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTargetEdit_AbsentRetriesLeavesStoredValue is the edit path's
|
||||||
|
// absent-versus-invalid case. Retries are only offered by the forms
|
||||||
|
// for types that retry, so a submission without the field must leave
|
||||||
|
// the stored count alone rather than be rejected or zeroed.
|
||||||
|
func TestTargetEdit_AbsentRetriesLeavesStoredValue(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupSourceTest(t)
|
||||||
|
|
||||||
|
for _, retries := range absentRetries() {
|
||||||
|
webhook, target := seedRetriesTarget(t, env)
|
||||||
|
|
||||||
|
w := submitTargetEdit(
|
||||||
|
env, webhook.ID, target.ID,
|
||||||
|
editRetriesForm(retries),
|
||||||
|
)
|
||||||
|
require.Equal(t,
|
||||||
|
http.StatusSeeOther, w.Code, w.Body.String(),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert.Equal(t, workingRetries,
|
||||||
|
storedTarget(t, env, target.ID).MaxRetries,
|
||||||
|
"an absent max_retries must leave the stored "+
|
||||||
|
"count alone (%q)", retries)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestTargetRetries_CreateAndEditAgreeOnEveryCase proves the two
|
||||||
|
// paths cannot disagree, which is what let the create form and the
|
||||||
|
// edit form drift apart in the first place. Every input is submitted
|
||||||
|
// to both and the accept/reject verdicts are compared.
|
||||||
|
func TestTargetRetries_CreateAndEditAgreeOnEveryCase(t *testing.T) {
|
||||||
|
t.Parallel()
|
||||||
|
|
||||||
|
env := setupSourceTest(t)
|
||||||
|
|
||||||
|
accepted := []string{
|
||||||
|
"0", "1",
|
||||||
|
strconv.Itoa(handlers.MaxTargetRetriesForTest),
|
||||||
|
}
|
||||||
|
overCeiling := overCeilingRetries()
|
||||||
|
unparseable := unparseableRetries()
|
||||||
|
|
||||||
|
cases := make(
|
||||||
|
[]string, 0,
|
||||||
|
len(accepted)+len(overCeiling)+len(unparseable),
|
||||||
|
)
|
||||||
|
cases = append(cases, accepted...)
|
||||||
|
cases = append(cases, overCeiling...)
|
||||||
|
cases = append(cases, unparseable...)
|
||||||
|
|
||||||
|
for _, retries := range cases {
|
||||||
|
_, createCode, _ := createWithRetries(t, env, retries)
|
||||||
|
|
||||||
|
webhook, target := seedRetriesTarget(t, env)
|
||||||
|
editCode := submitTargetEdit(
|
||||||
|
env, webhook.ID, target.ID,
|
||||||
|
editRetriesForm(retries),
|
||||||
|
).Code
|
||||||
|
|
||||||
|
assert.Equal(t,
|
||||||
|
createCode == http.StatusBadRequest,
|
||||||
|
editCode == http.StatusBadRequest,
|
||||||
|
"create and edit must agree on max_retries=%q "+
|
||||||
|
"(create %d, edit %d)",
|
||||||
|
retries, createCode, editCode,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -59,7 +59,7 @@ func TestHandleTargetToggle_DoesNotUndoAnEdit(t *testing.T) {
|
|||||||
stored := storedTarget(t, env, tgt.ID)
|
stored := storedTarget(t, env, tgt.ID)
|
||||||
assert.False(t, stored.Active)
|
assert.False(t, stored.Active)
|
||||||
assert.Equal(t, "edited-name", stored.Name)
|
assert.Equal(t, "edited-name", stored.Name)
|
||||||
assert.Equal(t, 5, stored.MaxAttempts)
|
assert.Equal(t, 5, stored.MaxRetries)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, editReplacedURL, storedHTTPConfig(t, env, tgt.ID).URL,
|
t, editReplacedURL, storedHTTPConfig(t, env, tgt.ID).URL,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -184,7 +184,8 @@ func TestEventLogPageIsCalledFullEventLog(t *testing.T) {
|
|||||||
// TestCreateFormRetentionCopyMatchesBehaviour pins the create form's
|
// TestCreateFormRetentionCopyMatchesBehaviour pins the create form's
|
||||||
// retention copy to what the code does: the reaper permanently deletes
|
// retention copy to what the code does: the reaper permanently deletes
|
||||||
// events past the cutoff, an empty field falls back to
|
// events past the cutoff, an empty field falls back to
|
||||||
// DefaultRetentionDays, and 0 means retain forever.
|
// DefaultRetentionDays, and 0 is rewritten to the retain-forever
|
||||||
|
// sentinel by Webhook.BeforeSave.
|
||||||
func TestCreateFormRetentionCopyMatchesBehaviour(t *testing.T) {
|
func TestCreateFormRetentionCopyMatchesBehaviour(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -252,7 +253,10 @@ func TestEditFormRetentionCopyMatchesBehaviour(t *testing.T) {
|
|||||||
"blank means unchanged, not forever",
|
"blank means unchanged, not forever",
|
||||||
)
|
)
|
||||||
|
|
||||||
forever := &database.Webhook{Name: "wh", RetentionDays: 0}
|
forever := &database.Webhook{
|
||||||
|
Name: "wh",
|
||||||
|
RetentionDays: database.RetentionForeverDays,
|
||||||
|
}
|
||||||
forever.ID = "wh-2"
|
forever.ID = "wh-2"
|
||||||
|
|
||||||
foreverBody := renderPage(
|
foreverBody := renderPage(
|
||||||
@@ -344,19 +348,21 @@ func TestEntrypointCopyButtonIsProgressiveEnhancement(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// maxAttemptsHelp is the wording both target forms must carry. The
|
// maxRetriesHelp is the wording both target forms must carry. The
|
||||||
// delivery core makes max_attempts attempts in total, not that many
|
// delivery core makes max_retries attempts in total, not that many
|
||||||
// retries on top of a first try (a fresh delivery starts at attempt 1
|
// retries on top of a first try (a fresh delivery starts at attempt 1
|
||||||
// and target_http gives up once the attempt number reaches
|
// and target_http gives up once the attempt number reaches
|
||||||
// max_attempts).
|
// max_retries), and 0 is special-cased to a single fire-and-forget
|
||||||
const maxAttemptsHelp = "How many times each delivery is attempted in " +
|
// attempt with no circuit breaker.
|
||||||
"all, the first attempt included."
|
const maxRetriesHelp = "How many times each delivery is attempted in " +
|
||||||
|
"all, the first attempt included. 0 means a single attempt with no " +
|
||||||
|
"retries and no circuit breaker."
|
||||||
|
|
||||||
// TestTargetFormMaxAttemptsCopyMatchesBehaviour pins the max_attempts
|
// TestTargetFormMaxRetriesCopyMatchesBehaviour pins the max_retries
|
||||||
// label, "Delivery attempts", and help text on both the create form
|
// label, "Delivery attempts", and help text on both the create form
|
||||||
// (the add-target form on the webhook detail page) and the edit form,
|
// (the add-target form on the webhook detail page) and the edit form,
|
||||||
// so the copy cannot drift back to calling the number a retry count.
|
// so the copy cannot drift back to calling the number a retry count.
|
||||||
func TestTargetFormMaxAttemptsCopyMatchesBehaviour(t *testing.T) {
|
func TestTargetFormMaxRetriesCopyMatchesBehaviour(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
var h *handlers.Handlers
|
var h *handlers.Handlers
|
||||||
@@ -388,11 +394,11 @@ func TestTargetFormMaxAttemptsCopyMatchesBehaviour(t *testing.T) {
|
|||||||
|
|
||||||
assert.Contains(t, createBody, "Delivery attempts:</label>")
|
assert.Contains(t, createBody, "Delivery attempts:</label>")
|
||||||
assert.Contains(
|
assert.Contains(
|
||||||
t, createBody, maxAttemptsHelp,
|
t, createBody, maxRetriesHelp,
|
||||||
"the add-target form must explain max_attempts as total attempts",
|
"the add-target form must explain max_retries as total attempts",
|
||||||
)
|
)
|
||||||
|
|
||||||
// A slack target exercises the same max_attempts field while needing
|
// A slack target exercises the same max_retries field while needing
|
||||||
// only a URL from the edit template, so the test data stays
|
// only a URL from the edit template, so the test data stays
|
||||||
// minimal. The Target and TargetForm keys mirror the field names
|
// minimal. The Target and TargetForm keys mirror the field names
|
||||||
// the template reads off the handler's values.
|
// the template reads off the handler's values.
|
||||||
@@ -407,7 +413,7 @@ func TestTargetFormMaxAttemptsCopyMatchesBehaviour(t *testing.T) {
|
|||||||
},
|
},
|
||||||
"TargetForm": map[string]any{
|
"TargetForm": map[string]any{
|
||||||
"URL": "https://hooks.slack.com/services/x",
|
"URL": "https://hooks.slack.com/services/x",
|
||||||
"MaxAttempts": "3",
|
"MaxRetries": "3",
|
||||||
},
|
},
|
||||||
dataKeyError: "",
|
dataKeyError: "",
|
||||||
},
|
},
|
||||||
@@ -415,8 +421,8 @@ func TestTargetFormMaxAttemptsCopyMatchesBehaviour(t *testing.T) {
|
|||||||
|
|
||||||
assert.Contains(t, editBody, `class="label">Delivery attempts</label>`)
|
assert.Contains(t, editBody, `class="label">Delivery attempts</label>`)
|
||||||
assert.Contains(
|
assert.Contains(
|
||||||
t, editBody, maxAttemptsHelp,
|
t, editBody, maxRetriesHelp,
|
||||||
"the target edit form must explain max_attempts as total attempts",
|
"the target edit form must explain max_retries as total attempts",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
|
"sneak.berlin/go/webhooker/internal/logfield"
|
||||||
"sneak.berlin/go/webhooker/internal/middleware"
|
"sneak.berlin/go/webhooker/internal/middleware"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -48,13 +49,14 @@ func (h *Handlers) HandleWebhook() http.HandlerFunc {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Logged only once the lookup has found the entrypoint, so
|
// Logged only once the UUID is known to name a real
|
||||||
// an invented path writes no INFO line; a miss is logged at
|
// entrypoint. The UUID comes straight out of the path on
|
||||||
// DEBUG. The entrypoint is named by its ID: the UUID is its
|
// the one unauthenticated endpoint, so logging it before
|
||||||
// credential and is never logged.
|
// the lookup let a client write an INFO line per invented
|
||||||
|
// path; the request itself is already in the access log
|
||||||
|
// and a miss is already logged at DEBUG.
|
||||||
h.log.Info("webhook request received",
|
h.log.Info("webhook request received",
|
||||||
"entrypoint_id", entrypoint.ID,
|
"entrypoint_uuid", entrypointUUID,
|
||||||
"webhook_id", entrypoint.WebhookID,
|
|
||||||
"method", r.Method,
|
"method", r.Method,
|
||||||
"remoteIP", middleware.RemoteIP(r),
|
"remoteIP", middleware.RemoteIP(r),
|
||||||
"clientIP", middleware.ClientIP(r),
|
"clientIP", middleware.ClientIP(r),
|
||||||
@@ -131,10 +133,17 @@ func (h *Handlers) lookupEntrypoint(
|
|||||||
"path = ?", entrypointUUID,
|
"path = ?", entrypointUUID,
|
||||||
).First(&entrypoint)
|
).First(&entrypoint)
|
||||||
if result.Error != nil {
|
if result.Error != nil {
|
||||||
// Not even an unknown UUID is logged. The receiver's log
|
// The receiver is unauthenticated and /h/{uuid}
|
||||||
// lines never carry the path segment, which is a credential
|
// matches any single segment, so this value is entirely
|
||||||
// whenever it names an entrypoint.
|
// client-chosen on exactly the branch where the lookup
|
||||||
h.log.Debug("entrypoint not found")
|
// failed. DEBUG is off by default; the cap is what keeps
|
||||||
|
// turning it on from restoring an unbounded write.
|
||||||
|
h.log.Debug(
|
||||||
|
"entrypoint not found",
|
||||||
|
"path", logfield.Truncate(
|
||||||
|
entrypointUUID, logfield.MaxBytes,
|
||||||
|
),
|
||||||
|
)
|
||||||
http.NotFound(w, r)
|
http.NotFound(w, r)
|
||||||
|
|
||||||
return entrypoint, false
|
return entrypoint, false
|
||||||
@@ -433,7 +442,7 @@ func buildDeliveryTasks(
|
|||||||
TargetName: targets[i].Name,
|
TargetName: targets[i].Name,
|
||||||
TargetType: targets[i].Type,
|
TargetType: targets[i].Type,
|
||||||
TargetConfig: targets[i].Config,
|
TargetConfig: targets[i].Config,
|
||||||
MaxAttempts: targets[i].MaxAttempts,
|
MaxRetries: targets[i].MaxRetries,
|
||||||
Method: event.Method,
|
Method: event.Method,
|
||||||
RawQuery: event.RawQuery,
|
RawQuery: event.RawQuery,
|
||||||
Headers: event.Headers,
|
Headers: event.Headers,
|
||||||
|
|||||||
@@ -77,8 +77,8 @@ func (h *Handlers) deleteWebhookResources(
|
|||||||
"user_id", userID,
|
"user_id", userID,
|
||||||
)
|
)
|
||||||
|
|
||||||
// Release the webhook's archive writers so a deleted
|
// Release the delivery engine's per-webhook archiving state
|
||||||
// webhook's archive writer (and any handle open
|
// so a deleted webhook's archive writer (and any handle open
|
||||||
// within its debounce window) does not linger for the
|
// within its debounce window) does not linger for the
|
||||||
// process lifetime. The archive file itself is deliberately
|
// process lifetime. The archive file itself is deliberately
|
||||||
// left on disk; see evictArchiveWriter.
|
// left on disk; see evictArchiveWriter.
|
||||||
@@ -150,9 +150,9 @@ func (h *Handlers) commitWebhookDeletion(
|
|||||||
return tx.Commit().Error
|
return tx.Commit().Error
|
||||||
}
|
}
|
||||||
|
|
||||||
// evictArchiveWriter asks the archive writers to drop the cached
|
// evictArchiveWriter asks the delivery engine to drop the cached
|
||||||
// writers of a webhook's database targets, closing their archive
|
// archive writers of a webhook's database targets, closing their
|
||||||
// file handles.
|
// archive file handles.
|
||||||
//
|
//
|
||||||
// The archive database files are NOT deleted. Unlike the event
|
// The archive database files are NOT deleted. Unlike the event
|
||||||
// database — which is per-webhook working storage and is
|
// database — which is per-webhook working storage and is
|
||||||
|
|||||||
@@ -6,8 +6,8 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
"github.com/go-chi/chi"
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// HandleSourceEdit shows the form to edit a webhook.
|
// HandleSourceEdit shows the form to edit a webhook.
|
||||||
@@ -59,8 +59,8 @@ func (h *Handlers) HandleSourceEditSubmit() http.HandlerFunc {
|
|||||||
|
|
||||||
sourceID := chi.URLParam(r, "sourceID")
|
sourceID := chi.URLParam(r, "sourceID")
|
||||||
|
|
||||||
h.archives.RenameLock().Lock()
|
h.renameMu.Lock()
|
||||||
defer h.archives.RenameLock().Unlock()
|
defer h.renameMu.Unlock()
|
||||||
|
|
||||||
var webhook database.Webhook
|
var webhook database.Webhook
|
||||||
|
|
||||||
@@ -130,7 +130,7 @@ func (h *Handlers) applyWebhookEdit(
|
|||||||
edited.RetentionDays = retentionDays
|
edited.RetentionDays = retentionDays
|
||||||
|
|
||||||
// A new name renames the archive files before it is saved (see
|
// A new name renames the archive files before it is saved (see
|
||||||
// archive.Writers.Rename). If either step fails, the same targets'
|
// delivery.Engine.Rename). If either step fails, the same targets'
|
||||||
// archives go back to the name that is still stored, without
|
// archives go back to the name that is still stored, without
|
||||||
// reading the main database again.
|
// reading the main database again.
|
||||||
targets, err := h.renameWebhookArchives(
|
targets, err := h.renameWebhookArchives(
|
||||||
@@ -150,7 +150,7 @@ func (h *Handlers) applyWebhookEdit(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if errors.Is(err, archive.ErrNameTaken) {
|
if errors.Is(err, delivery.ErrArchiveNameTaken) {
|
||||||
h.renderWebhookEdit(
|
h.renderWebhookEdit(
|
||||||
w, r, webhook, name, description, retention,
|
w, r, webhook, name, description, retention,
|
||||||
"Not saved: "+err.Error()+
|
"Not saved: "+err.Error()+
|
||||||
|
|||||||
@@ -500,7 +500,7 @@ func TestWebhookStats_PaneShowsRetentionPeriod(t *testing.T) {
|
|||||||
want string
|
want string
|
||||||
}{
|
}{
|
||||||
{30, "30 days"},
|
{30, "30 days"},
|
||||||
{0, "forever"},
|
{database.RetentionForeverDays, "forever"},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
|
|||||||
@@ -12,15 +12,12 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
"github.com/go-chi/chi"
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"gorm.io/gorm/clause"
|
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
"sneak.berlin/go/webhooker/internal/handlers"
|
||||||
"sneak.berlin/go/webhooker/internal/middleware"
|
"sneak.berlin/go/webhooker/internal/middleware"
|
||||||
"sneak.berlin/go/webhooker/internal/middleware/middlewaretest"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestHandleWebhook_LogsClientNextToThePeer checks that the
|
// TestHandleWebhook_LogsClientNextToThePeer checks that the
|
||||||
@@ -102,87 +99,6 @@ func TestHandleWebhook_LogsClientNextToThePeer(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleWebhook_LogsNoEntrypointUUID sends an accepted event, a
|
|
||||||
// request over the receiver rate limit and a request for an unknown
|
|
||||||
// UUID through the receiver as routes.go mounts it, and checks that
|
|
||||||
// nothing logged, at any level, carries either UUID.
|
|
||||||
func TestHandleWebhook_LogsNoEntrypointUUID(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
db *database.Database
|
|
||||||
)
|
|
||||||
|
|
||||||
cfg := &config.Config{DataDir: t.TempDir(), ReceiverRateLimit: 1}
|
|
||||||
|
|
||||||
app := newTestAppWithConfig(t, cfg, &h, &db)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
buf := new(bytes.Buffer)
|
|
||||||
log := slog.New(slog.NewJSONHandler(
|
|
||||||
buf, &slog.HandlerOptions{Level: slog.LevelDebug},
|
|
||||||
))
|
|
||||||
h.SetLogForTest(log)
|
|
||||||
mw := middlewaretest.New(t, log, cfg, nil)
|
|
||||||
|
|
||||||
webhook := seedWebhook(t, db)
|
|
||||||
entrypoint := &database.Entrypoint{
|
|
||||||
WebhookID: webhook.ID,
|
|
||||||
Path: uuid.New().String(),
|
|
||||||
Active: true,
|
|
||||||
}
|
|
||||||
require.NoError(t, db.DB().Omit(clause.Associations).
|
|
||||||
Create(entrypoint).Error)
|
|
||||||
|
|
||||||
unknownUUID := uuid.New().String()
|
|
||||||
|
|
||||||
router := chi.NewRouter()
|
|
||||||
router.Use(mw.Logging())
|
|
||||||
router.With(mw.ReceiverRateLimit()).HandleFunc(
|
|
||||||
middleware.ReceiverRoute, h.HandleWebhook(),
|
|
||||||
)
|
|
||||||
|
|
||||||
// In order: the limit is one request per minute per path, so the
|
|
||||||
// second request for the known UUID is the rejected one.
|
|
||||||
for _, step := range []struct {
|
|
||||||
entrypointUUID string
|
|
||||||
wantStatus int
|
|
||||||
}{
|
|
||||||
{entrypoint.Path, http.StatusOK},
|
|
||||||
{entrypoint.Path, http.StatusTooManyRequests},
|
|
||||||
{unknownUUID, http.StatusNotFound},
|
|
||||||
} {
|
|
||||||
req := httptest.NewRequestWithContext(
|
|
||||||
context.Background(), http.MethodPost,
|
|
||||||
"/h/"+step.entrypointUUID, strings.NewReader("{}"),
|
|
||||||
)
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
router.ServeHTTP(w, req)
|
|
||||||
|
|
||||||
require.Equal(t, step.wantStatus, w.Code)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The lines this test is about were all written, so the checks
|
|
||||||
// below are not passing on an empty log.
|
|
||||||
for _, msg := range []string{
|
|
||||||
"webhook request received",
|
|
||||||
"webhook receiver rate limit exceeded",
|
|
||||||
"entrypoint not found",
|
|
||||||
} {
|
|
||||||
assert.Contains(t, buf.String(), `"msg":"`+msg+`"`)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.NotContains(t, buf.String(), entrypoint.Path)
|
|
||||||
assert.NotContains(t, buf.String(), unknownUUID)
|
|
||||||
|
|
||||||
line := receivedLine(t, buf)
|
|
||||||
assert.Equal(t, entrypoint.ID, line["entrypoint_id"])
|
|
||||||
assert.Equal(t, webhook.ID, line["webhook_id"])
|
|
||||||
}
|
|
||||||
|
|
||||||
// receivedLine returns the one "webhook request received" line in the
|
// receivedLine returns the one "webhook request received" line in the
|
||||||
// captured JSON log.
|
// captured JSON log.
|
||||||
func receivedLine(t *testing.T, buf *bytes.Buffer) map[string]any {
|
func receivedLine(t *testing.T, buf *bytes.Buffer) map[string]any {
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -28,13 +27,6 @@ const floodRequests = 64
|
|||||||
// line for a redirected or rejected request may contain it.
|
// line for a redirected or rejected request may contain it.
|
||||||
const attackerMarker = "QQATTACKERTEXTQQ"
|
const attackerMarker = "QQATTACKERTEXTQQ"
|
||||||
|
|
||||||
// knownEntrypointUUID and failingEntrypointUUID are the UUIDs the
|
|
||||||
// stand-in receiver in accessLogRouter answers 200 and 500 to.
|
|
||||||
const (
|
|
||||||
knownEntrypointUUID = "6f1e2a9c-4b7d-4e3a-9c2f-1d8b5a7e3c60"
|
|
||||||
failingEntrypointUUID = "0b7c3e5a-2d9f-4a61-8e4b-7c1d6f2a9e35"
|
|
||||||
)
|
|
||||||
|
|
||||||
// maxLineBytes bounds a single access log line whose client-supplied
|
// maxLineBytes bounds a single access log line whose client-supplied
|
||||||
// fields are of ordinary size. Well above what the fixed fields need,
|
// fields are of ordinary size. Well above what the fixed fields need,
|
||||||
// well below the length of the oversized input the amplification tests
|
// well below the length of the oversized input the amplification tests
|
||||||
@@ -140,21 +132,17 @@ func accessLogRouter(m *middleware.Middleware) *chi.Mux {
|
|||||||
)
|
)
|
||||||
|
|
||||||
router.HandleFunc(
|
router.HandleFunc(
|
||||||
middleware.ReceiverRoute,
|
"/h/{uuid}",
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
func(w http.ResponseWriter, r *http.Request) {
|
||||||
// Stands in for the real handler: a known entrypoint
|
// Stands in for the real handler: an unknown entrypoint
|
||||||
// UUID succeeds, a failing one is a server error, and
|
// UUID 404s, a known one succeeds.
|
||||||
// any other 404s.
|
if chi.URLParam(r, "uuid") != "known" {
|
||||||
switch chi.URLParam(r, "uuid") {
|
|
||||||
case knownEntrypointUUID:
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
case failingEntrypointUUID:
|
|
||||||
http.Error(
|
|
||||||
w, "boom", http.StatusInternalServerError,
|
|
||||||
)
|
|
||||||
default:
|
|
||||||
http.Error(w, "not found", http.StatusNotFound)
|
http.Error(w, "not found", http.StatusNotFound)
|
||||||
|
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -347,13 +335,11 @@ func oversizedValue(ch string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// oversizedHeaders fills every client-supplied header the access log
|
// oversizedHeaders fills every client-supplied header the access log
|
||||||
// reads with the same value. The Referer carries it as the host of an
|
// reads with the same value.
|
||||||
// absolute URL, because the access log keeps only a Referer's scheme
|
|
||||||
// and host.
|
|
||||||
func oversizedHeaders(value string) map[string]string {
|
func oversizedHeaders(value string) map[string]string {
|
||||||
return map[string]string{
|
return map[string]string{
|
||||||
"User-Agent": value,
|
"User-Agent": value,
|
||||||
"Referer": "http://" + value,
|
"Referer": value,
|
||||||
"X-Request-Id": value,
|
"X-Request-Id": value,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -663,111 +649,17 @@ func TestAccessLog_SuccessKeepsConcretePathAndRedactsQuery(
|
|||||||
router := accessLogRouter(m)
|
router := accessLogRouter(m)
|
||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, http.StatusOK,
|
t, http.StatusOK, get(t, router, "/h/known?src=ci"),
|
||||||
get(t, router, "/.well-known/healthcheck?src=ci"),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// The path resolved against a stored entrypoint, so it stays. The
|
||||||
|
// query never does: see TestAccessLog_UnauthenticatedSuccess...
|
||||||
entries := accessLogEntries(t, buf)
|
entries := accessLogEntries(t, buf)
|
||||||
require.Len(t, entries, 1)
|
require.Len(t, entries, 1)
|
||||||
assert.Equal(
|
assert.Equal(t, "/h/known?(redacted)", entries[0]["url"])
|
||||||
t, "/.well-known/healthcheck?(redacted)", entries[0]["url"],
|
|
||||||
)
|
|
||||||
assert.NotContains(t, buf.String(), "src=ci")
|
assert.NotContains(t, buf.String(), "src=ci")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestAccessLog_ReceiverNeverLogsTheUUID covers the two statuses that
|
|
||||||
// keep the concrete path on every other route. On the receiver that
|
|
||||||
// path carries the entrypoint UUID, so the line names the route
|
|
||||||
// pattern instead.
|
|
||||||
func TestAccessLog_ReceiverNeverLogsTheUUID(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for entrypointUUID, wantStatus := range map[string]int{
|
|
||||||
knownEntrypointUUID: http.StatusOK,
|
|
||||||
failingEntrypointUUID: http.StatusInternalServerError,
|
|
||||||
} {
|
|
||||||
t.Run(strconv.Itoa(wantStatus), func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
m, buf := capturingMiddleware(t)
|
|
||||||
router := accessLogRouter(m)
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t, wantStatus,
|
|
||||||
get(t, router, "/h/"+entrypointUUID+"?src=ci"),
|
|
||||||
)
|
|
||||||
|
|
||||||
entries := accessLogEntries(t, buf)
|
|
||||||
require.Len(t, entries, 1)
|
|
||||||
assert.Equal(t, middleware.ReceiverRoute, entries[0]["url"])
|
|
||||||
assert.NotContains(t, buf.String(), entrypointUUID)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestAccessLog_ReceiverPreflightNeverLogsTheUUID sends a CORS
|
|
||||||
// preflight to the receiver in dev, in the production middleware
|
|
||||||
// order. CORS answers it with 200 before chi routes it, so the line
|
|
||||||
// names no route and carries no UUID.
|
|
||||||
func TestAccessLog_ReceiverPreflightNeverLogsTheUUID(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
m, buf := capturingMiddleware(t)
|
|
||||||
|
|
||||||
router := chi.NewRouter()
|
|
||||||
router.Use(m.Logging())
|
|
||||||
router.Use(m.CORS())
|
|
||||||
router.HandleFunc(
|
|
||||||
middleware.ReceiverRoute,
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(
|
|
||||||
context.Background(), http.MethodOptions,
|
|
||||||
"/h/"+knownEntrypointUUID, nil,
|
|
||||||
)
|
|
||||||
req.Header.Set("Origin", "https://sender.example")
|
|
||||||
req.Header.Set("Access-Control-Request-Method", http.MethodPost)
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
router.ServeHTTP(w, req)
|
|
||||||
assert.Equal(t, http.StatusOK, w.Code)
|
|
||||||
|
|
||||||
entries := accessLogEntries(t, buf)
|
|
||||||
require.Len(t, entries, 1)
|
|
||||||
assert.Equal(t, unmatchedRouteLiteral, entries[0]["url"])
|
|
||||||
assert.NotContains(t, buf.String(), knownEntrypointUUID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestAccessLog_RefererNeverLogsTheUUID sends receiver requests whose
|
|
||||||
// Referer is the entrypoint URL, as a sender's is after it follows a
|
|
||||||
// redirect from http:// to https://. The referer field keeps only the
|
|
||||||
// scheme and host, and a Referer without them is logged as a marker.
|
|
||||||
func TestAccessLog_RefererNeverLogsTheUUID(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
m, buf := capturingMiddleware(t)
|
|
||||||
router := accessLogRouter(m)
|
|
||||||
|
|
||||||
for _, referer := range []string{
|
|
||||||
"http://webhooker.example/h/" + knownEntrypointUUID + "?src=ci",
|
|
||||||
"/h/" + knownEntrypointUUID,
|
|
||||||
} {
|
|
||||||
assert.Equal(t, http.StatusOK, getWithHeaders(
|
|
||||||
t, router, "/h/"+knownEntrypointUUID,
|
|
||||||
map[string]string{"Referer": referer},
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
entries := accessLogEntries(t, buf)
|
|
||||||
require.Len(t, entries, 2)
|
|
||||||
assert.Equal(t, "http://webhooker.example", entries[0]["referer"])
|
|
||||||
assert.Equal(t, "(redacted)", entries[1]["referer"])
|
|
||||||
assert.NotContains(t, buf.String(), knownEntrypointUUID)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAccessLog_ServerErrorKeepsConcreteURL(t *testing.T) {
|
func TestAccessLog_ServerErrorKeepsConcreteURL(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -10,10 +10,8 @@ package middleware_test
|
|||||||
// the client's choosing.
|
// the client's choosing.
|
||||||
// - CSRF's 403 rejection, at WARN, also registered ahead of
|
// - CSRF's 403 rejection, at WARN, also registered ahead of
|
||||||
// RequireAuth.
|
// RequireAuth.
|
||||||
// - The rate limiters' 429 rejection, at WARN. On the receiver
|
// - The rate limiters' 429 rejection, at WARN, on the
|
||||||
// route it names the route pattern rather than the path, and
|
// unauthenticated receiver among others.
|
||||||
// the limiters that log a path sit behind RequireAuth, so its cap
|
|
||||||
// is defensive.
|
|
||||||
// - RequireAuth's own unauthenticated-request line, at DEBUG.
|
// - RequireAuth's own unauthenticated-request line, at DEBUG.
|
||||||
// - RecordLoginFailure's throttle rejection, at WARN. Its cap is
|
// - RecordLoginFailure's throttle rejection, at WARN. Its cap is
|
||||||
// defensive rather than load-bearing today: chi pins the one
|
// defensive rather than load-bearing today: chi pins the one
|
||||||
@@ -270,14 +268,10 @@ func logSites() map[string]logSite {
|
|||||||
send: postNoToken,
|
send: postNoToken,
|
||||||
wantStatus: http.StatusForbidden,
|
wantStatus: http.StatusForbidden,
|
||||||
},
|
},
|
||||||
// The 429 line every limiter but the aggregate one shares.
|
// The per-entrypoint receiver limiter, unauthenticated. Its
|
||||||
// Mounted outside any route, as here, it logs the path, as
|
// bucket is keyed on the path, so the first request through a
|
||||||
// it does for the limiters behind RequireAuth; on
|
// fresh path is served and only the ones after it are
|
||||||
// ReceiverRoute it names the pattern instead. The
|
// rejected; sendUntilLimited absorbs that.
|
||||||
// per-entrypoint receiver limiter's bucket is keyed on the
|
|
||||||
// path, so the first request through a fresh path is served
|
|
||||||
// and only the ones after it are rejected; sendUntilLimited
|
|
||||||
// absorbs that.
|
|
||||||
"receiver rate limit 429": {
|
"receiver rate limit 429": {
|
||||||
build: func(
|
build: func(
|
||||||
t *testing.T, m *middleware.Middleware,
|
t *testing.T, m *middleware.Middleware,
|
||||||
|
|||||||
@@ -41,8 +41,7 @@ const unmatchedMethod = unmatchedRoute
|
|||||||
// after /h/ is client-chosen, so labelling by path lets any
|
// after /h/ is client-chosen, so labelling by path lets any
|
||||||
// unauthenticated client mint permanent series at will, and publishes
|
// unauthenticated client mint permanent series at will, and publishes
|
||||||
// the entrypoint UUID (the receiver's only credential) in the scrape
|
// the entrypoint UUID (the receiver's only credential) in the scrape
|
||||||
// while doing it. The access log and the 429 line read it for the
|
// while doing it.
|
||||||
// same reason: see ReceiverRoute.
|
|
||||||
//
|
//
|
||||||
// chi populates the route context during routeHTTP, so this is only
|
// chi populates the route context during routeHTTP, so this is only
|
||||||
// valid once routing has run. Every caller below is on the recording
|
// valid once routing has run. Every caller below is on the recording
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
basicauth "github.com/99designs/basicauth-go"
|
basicauth "github.com/99designs/basicauth-go"
|
||||||
|
"github.com/go-chi/chi"
|
||||||
"github.com/go-chi/chi/middleware"
|
"github.com/go-chi/chi/middleware"
|
||||||
"github.com/go-chi/cors"
|
"github.com/go-chi/cors"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
@@ -22,7 +23,6 @@ import (
|
|||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/globals"
|
"sneak.berlin/go/webhooker/internal/globals"
|
||||||
"sneak.berlin/go/webhooker/internal/logfield"
|
"sneak.berlin/go/webhooker/internal/logfield"
|
||||||
"sneak.berlin/go/webhooker/internal/referer"
|
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
"sneak.berlin/go/webhooker/internal/session"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -96,14 +96,14 @@ const (
|
|||||||
// It is also the ceiling on every OTHER line this service writes
|
// It is also the ceiling on every OTHER line this service writes
|
||||||
// THROUGH SLOG that carries text an UNAUTHENTICATED client
|
// THROUGH SLOG that carries text an UNAUTHENTICATED client
|
||||||
// supplies. Those lines — the MaxBodySize rejection, the CSRF
|
// supplies. Those lines — the MaxBodySize rejection, the CSRF
|
||||||
// rejection, the unauthenticated-request DEBUG line, the
|
// rejection, the rate-limit rejection, the unauthenticated-request
|
||||||
// failed-login DEBUG lines, and the two login-throttle WARN
|
// and unknown-entrypoint DEBUG lines, the failed-login DEBUG
|
||||||
// lines ("login failure limit exceeded" in loginguard.go and
|
// lines, and the two login-throttle WARN lines ("login failure
|
||||||
// "password verification capacity exhausted" in
|
// limit exceeded" in loginguard.go and "password verification
|
||||||
// internal/handlers/auth.go) — spend the same per-field budgets,
|
// capacity exhausted" in internal/handlers/auth.go) — spend the
|
||||||
// and each carries strictly fewer client-supplied fields than the
|
// same per-field budgets, and each carries strictly fewer
|
||||||
// access log does, so none of them can reach a width the access
|
// client-supplied fields than the access log does,
|
||||||
// log cannot. That is
|
// so none of them can reach a width the access log cannot. That is
|
||||||
// asserted directly, per line and under both handlers, rather than
|
// asserted directly, per line and under both handlers, rather than
|
||||||
// left to the reasoning: see logbound_test.go in this package and
|
// left to the reasoning: see logbound_test.go in this package and
|
||||||
// in internal/handlers.
|
// in internal/handlers.
|
||||||
@@ -115,15 +115,16 @@ const (
|
|||||||
// a caller on a parameterised route would supply, which is the
|
// a caller on a parameterised route would supply, which is the
|
||||||
// only way those caps can be pinned at all.
|
// only way those caps can be pinned at all.
|
||||||
//
|
//
|
||||||
// The GORM adapter in internal/gormlog writes through slog as well.
|
// One writer in that set is not a handler's own slog call. The
|
||||||
// It logs each statement with its placeholders rather than the
|
// GORM adapter in internal/gormlog logs the statement with the
|
||||||
// values bound to them (see its ParamsFilter), so the key a client
|
// client-chosen parameter already interpolated into it, which on
|
||||||
// sends to the receiver or login lookup does not reach its lines.
|
// the receiver and login lookups is exactly the value the budgets
|
||||||
// Its widest line spends two logfield.MaxBytes budgets — the
|
// above exist for. Its widest line spends two logfield.MaxBytes
|
||||||
// statement and the driver error — against a fixed portion smaller
|
// budgets — the statement and the driver error — against a fixed
|
||||||
// than this one's, and internal/gormlog/gormlog_test.go asserts
|
// portion smaller than this one's, and
|
||||||
// every line it emits against this constant directly rather than
|
// internal/gormlog/gormlog_test.go asserts every line it emits
|
||||||
// leaving it as arithmetic.
|
// against this constant directly rather than leaving it as
|
||||||
|
// arithmetic.
|
||||||
//
|
//
|
||||||
// What it does NOT cover, so that the figure above is not read as
|
// What it does NOT cover, so that the figure above is not read as
|
||||||
// more than it is:
|
// more than it is:
|
||||||
@@ -150,12 +151,6 @@ const (
|
|||||||
MaxAccessLogLineBytes = 2560
|
MaxAccessLogLineBytes = 2560
|
||||||
)
|
)
|
||||||
|
|
||||||
// ReceiverRoute is the route pattern the webhook receiver is served
|
|
||||||
// at. Its path carries the entrypoint UUID, which is the receiver's
|
|
||||||
// only credential, so the log lines this package writes name a
|
|
||||||
// request on this route by the pattern, never by its path.
|
|
||||||
const ReceiverRoute = "/h/{uuid}"
|
|
||||||
|
|
||||||
//nolint:revive // MiddlewareParams is a standard fx naming convention.
|
//nolint:revive // MiddlewareParams is a standard fx naming convention.
|
||||||
type MiddlewareParams struct {
|
type MiddlewareParams struct {
|
||||||
fx.In
|
fx.In
|
||||||
@@ -273,14 +268,14 @@ func (lrw *loggingResponseWriter) Unwrap() http.ResponseWriter {
|
|||||||
// marker.
|
// marker.
|
||||||
//
|
//
|
||||||
// The path on those branches is bounded by the service's routes or by
|
// The path on those branches is bounded by the service's routes or by
|
||||||
// the operator's data — a 2xx under /s means the file is in the
|
// the operator's data — a 2xx on the receiver means the UUID named a
|
||||||
// embedded tree. The query is not bounded by anything:
|
// stored entrypoint, a 2xx under /s means the file is in the embedded
|
||||||
// /.well-known/healthcheck and /s/* take no authentication and sit
|
// tree. The query is not bounded by anything: /.well-known/healthcheck
|
||||||
// behind no rate limiter, and /pages/login behind only the login
|
// and /s/* take no authentication and sit behind no rate limiter, and
|
||||||
// limiter, so any of them will answer 200 to a URL carrying an
|
// /pages/login behind only the login limiter, so any of them will
|
||||||
// arbitrary number of arbitrary bytes after the '?'. Keeping the path
|
// answer 200 to a URL carrying an arbitrary number of arbitrary bytes
|
||||||
// and dropping the query is what makes this branch as bounded as the
|
// after the '?'. Keeping the path and dropping the query is what makes
|
||||||
// pattern branches below.
|
// this branch as bounded as the pattern branches below.
|
||||||
//
|
//
|
||||||
// Nothing debuggable is lost. The only query parameters the service
|
// Nothing debuggable is lost. The only query parameters the service
|
||||||
// reads are the login page's `next`, the page to return to,
|
// reads are the login page's `next`, the page to return to,
|
||||||
@@ -302,40 +297,36 @@ func concreteLogURL(r *http.Request) string {
|
|||||||
|
|
||||||
// accessLogURL returns the value for the access log's url field.
|
// accessLogURL returns the value for the access log's url field.
|
||||||
//
|
//
|
||||||
// On ReceiverRoute it is the route pattern whatever the status, so
|
// 2xx and 5xx responses get the concrete path (see concreteLogURL). A
|
||||||
// the entrypoint UUID in the path is never logged. A request chi never
|
// success resolved against a static route or against the operator's
|
||||||
// routed gets unmatchedRoute whatever the status, for the same reason:
|
// own data — on the receiver, a 2xx means the UUID named a stored
|
||||||
// in dev, CORS answers a preflight to the receiver before routing.
|
// entrypoint — and a server error is our own bug, where the exact URL
|
||||||
//
|
// is the primary evidence and which no client can provoke at will.
|
||||||
// On every other route, 2xx and 5xx responses get the concrete path
|
|
||||||
// (see concreteLogURL). A success resolved against a static route or
|
|
||||||
// against the operator's own data, and a server error is our own bug,
|
|
||||||
// where the exact URL is the primary evidence and which no client can
|
|
||||||
// provoke at will.
|
|
||||||
//
|
//
|
||||||
// 3xx and 4xx responses get the chi route pattern instead. Those are
|
// 3xx and 4xx responses get the chi route pattern instead. Those are
|
||||||
// the outcomes an unauthenticated client drives for free, such as a
|
// the outcomes an unauthenticated client drives for free: 404 or 429
|
||||||
// 303 to the login page on any invented /user/ path. Logging the
|
// on any invented /h/ path, 303 to the login page on any
|
||||||
// concrete URL there lets a flood write attacker-chosen text, of
|
// invented /user/ path. Logging the concrete URL there lets a flood
|
||||||
// attacker-chosen length, into the operator's log at one line per
|
// write attacker-chosen text, of attacker-chosen length, into the
|
||||||
// request. The pattern comes from the router's own table, so it is
|
// operator's log at one line per request. The pattern comes from the
|
||||||
// bounded by the service's routes while still naming which class of
|
// router's own table, so it is bounded by the service's routes while
|
||||||
// request was rejected.
|
// still naming which class of request was rejected.
|
||||||
//
|
//
|
||||||
// The pattern is only populated once routing has run, so this must be
|
// The pattern is only populated once routing has run, so this must be
|
||||||
// called after the handler returns, not before.
|
// called after the handler returns, not before.
|
||||||
func accessLogURL(r *http.Request, status int) string {
|
func accessLogURL(r *http.Request, status int) string {
|
||||||
pattern := routePatternID(r.Context())
|
|
||||||
if pattern == ReceiverRoute || pattern == unmatchedRoute {
|
|
||||||
return pattern
|
|
||||||
}
|
|
||||||
|
|
||||||
if status < http.StatusMultipleChoices ||
|
if status < http.StatusMultipleChoices ||
|
||||||
status >= http.StatusInternalServerError {
|
status >= http.StatusInternalServerError {
|
||||||
return concreteLogURL(r)
|
return concreteLogURL(r)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if rc := chi.RouteContext(r.Context()); rc != nil {
|
||||||
|
if pattern := rc.RoutePattern(); pattern != "" {
|
||||||
return pattern
|
return pattern
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return unmatchedRoute
|
||||||
}
|
}
|
||||||
|
|
||||||
// Logging returns middleware that logs each HTTP request with
|
// Logging returns middleware that logs each HTTP request with
|
||||||
@@ -388,8 +379,7 @@ func (s *Middleware) Logging() func(http.Handler) http.Handler {
|
|||||||
requestID, maxLogRequestIDBytes,
|
requestID, maxLogRequestIDBytes,
|
||||||
),
|
),
|
||||||
"referer", logfield.Truncate(
|
"referer", logfield.Truncate(
|
||||||
referer.SchemeAndHost(r.Referer()),
|
r.Referer(), logfield.MaxBytes,
|
||||||
logfield.MaxBytes,
|
|
||||||
),
|
),
|
||||||
"proto", r.Proto,
|
"proto", r.Proto,
|
||||||
"remoteIP", RemoteIP(r),
|
"remoteIP", RemoteIP(r),
|
||||||
@@ -548,9 +538,7 @@ func (s *Middleware) MetricsAuth() func(http.Handler) http.Handler {
|
|||||||
// SecurityHeaders returns middleware that sets production security
|
// SecurityHeaders returns middleware that sets production security
|
||||||
// headers on every response: HSTS (unless HSTS_MAX_AGE is zero),
|
// headers on every response: HSTS (unless HSTS_MAX_AGE is zero),
|
||||||
// X-Content-Type-Options, X-Frame-Options, CSP, Referrer-Policy, and
|
// X-Content-Type-Options, X-Frame-Options, CSP, Referrer-Policy, and
|
||||||
// Permissions-Policy. The CSP takes scripts and styles only from
|
// Permissions-Policy.
|
||||||
// files on this origin, so a template may hold no inline script,
|
|
||||||
// event handler or style attribute.
|
|
||||||
func (s *Middleware) SecurityHeaders() func(http.Handler) http.Handler {
|
func (s *Middleware) SecurityHeaders() func(http.Handler) http.Handler {
|
||||||
maxAge := s.params.Config.HSTSMaxAge
|
maxAge := s.params.Config.HSTSMaxAge
|
||||||
hsts := "max-age=" + strconv.FormatInt(int64(maxAge/time.Second), 10) +
|
hsts := "max-age=" + strconv.FormatInt(int64(maxAge/time.Second), 10) +
|
||||||
@@ -572,8 +560,8 @@ func (s *Middleware) SecurityHeaders() func(http.Handler) http.Handler {
|
|||||||
w.Header().Set(
|
w.Header().Set(
|
||||||
"Content-Security-Policy",
|
"Content-Security-Policy",
|
||||||
"default-src 'self'; "+
|
"default-src 'self'; "+
|
||||||
"script-src 'self'; "+
|
"script-src 'self' 'unsafe-inline'; "+
|
||||||
"style-src 'self'",
|
"style-src 'self' 'unsafe-inline'",
|
||||||
)
|
)
|
||||||
w.Header().Set(
|
w.Header().Set(
|
||||||
"Referrer-Policy",
|
"Referrer-Policy",
|
||||||
|
|||||||
@@ -747,10 +747,10 @@ func TestNoCache_SetsHeaders(t *testing.T) {
|
|||||||
|
|
||||||
// --- SecurityHeaders Middleware Tests ---
|
// --- SecurityHeaders Middleware Tests ---
|
||||||
|
|
||||||
// securityHeaders serves one request through SecurityHeaders built
|
// hstsHeaders serves one request through SecurityHeaders built over
|
||||||
// over a Config with the given HSTSMaxAge, and returns the response's
|
// a Config with the given HSTSMaxAge, and returns every
|
||||||
// headers.
|
// Strict-Transport-Security value on the response.
|
||||||
func securityHeaders(t *testing.T, maxAge time.Duration) http.Header {
|
func hstsHeaders(t *testing.T, maxAge time.Duration) []string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
cfg := &config.Config{HSTSMaxAge: maxAge}
|
cfg := &config.Config{HSTSMaxAge: maxAge}
|
||||||
@@ -772,7 +772,7 @@ func securityHeaders(t *testing.T, maxAge time.Duration) http.Header {
|
|||||||
|
|
||||||
handler.ServeHTTP(w, req)
|
handler.ServeHTTP(w, req)
|
||||||
|
|
||||||
return w.Header()
|
return w.Header().Values("Strict-Transport-Security")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSecurityHeaders_HSTS(t *testing.T) {
|
func TestSecurityHeaders_HSTS(t *testing.T) {
|
||||||
@@ -804,28 +804,11 @@ func TestSecurityHeaders_HSTS(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
assert.Equal(
|
assert.Equal(t, tt.expected, hstsHeaders(t, tt.maxAge))
|
||||||
t, tt.expected,
|
|
||||||
securityHeaders(t, tt.maxAge).Values(
|
|
||||||
"Strict-Transport-Security",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestSecurityHeaders_CSP pins the Content-Security-Policy: a page
|
|
||||||
// takes its scripts and styles only from files on its own origin,
|
|
||||||
// never from an inline script, event handler or style.
|
|
||||||
func TestSecurityHeaders_CSP(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t, "default-src 'self'; script-src 'self'; style-src 'self'",
|
|
||||||
securityHeaders(t, 0).Get("Content-Security-Policy"),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- MaxBodySize Middleware Tests ---
|
// --- MaxBodySize Middleware Tests ---
|
||||||
|
|
||||||
const testBodyLimit int64 = 64
|
const testBodyLimit int64 = 64
|
||||||
|
|||||||
@@ -259,28 +259,26 @@ func (m *Middleware) clientAddr(r *http.Request) (netip.Addr, bool) {
|
|||||||
return peer, true
|
return peer, true
|
||||||
}
|
}
|
||||||
|
|
||||||
// tooManyRequests returns the 429 handler used by the postRateLimit
|
// tooManyRequests returns the 429 handler used by the
|
||||||
// limiters and the per-entrypoint receiver limiter: it logs the
|
// password-change and per-entrypoint receiver limiters: it logs the
|
||||||
// rejection with logMessage and answers with responseMessage.
|
// rejection with logMessage and answers with responseMessage.
|
||||||
// httprate adds the Retry-After header (RFC 6585). The aggregate
|
// httprate adds the Retry-After header (RFC 6585). The aggregate
|
||||||
// receiver limiter uses floodTooManyRequests instead.
|
// receiver limiter uses floodTooManyRequests instead.
|
||||||
//
|
//
|
||||||
// On ReceiverRoute the line names the route pattern, so the
|
// The path is capped against the same budget as the access log's url
|
||||||
// entrypoint UUID in the path is never logged. Elsewhere it names the
|
// field. The per-entrypoint receiver limiter is unauthenticated and
|
||||||
// path, capped against the same budget as the access log's url field;
|
// its path is a client-chosen segment of client-chosen length, so at
|
||||||
// the limiters there sit behind RequireAuth, so the cap is defensive.
|
// WARN an uncapped path would let a sender pick the size of the line
|
||||||
|
// it writes — the same defect the access log capping closed.
|
||||||
func (m *Middleware) tooManyRequests(
|
func (m *Middleware) tooManyRequests(
|
||||||
logMessage, responseMessage string,
|
logMessage, responseMessage string,
|
||||||
) http.HandlerFunc {
|
) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
path := r.URL.Path
|
|
||||||
if routePatternID(r.Context()) == ReceiverRoute {
|
|
||||||
path = ReceiverRoute
|
|
||||||
}
|
|
||||||
|
|
||||||
m.log.Warn(
|
m.log.Warn(
|
||||||
logMessage,
|
logMessage,
|
||||||
"path", logfield.Truncate(path, logfield.MaxBytes),
|
"path", logfield.Truncate(
|
||||||
|
r.URL.Path, logfield.MaxBytes,
|
||||||
|
),
|
||||||
"remoteIP", RemoteIP(r),
|
"remoteIP", RemoteIP(r),
|
||||||
"clientIP", ClientIP(r),
|
"clientIP", ClientIP(r),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,32 +0,0 @@
|
|||||||
// Package referer reduces a Referer header to its scheme and host, so
|
|
||||||
// that a URL path, which on the receiver route carries the entrypoint
|
|
||||||
// UUID, never reaches a log line or an error report.
|
|
||||||
package referer
|
|
||||||
|
|
||||||
import "net/url"
|
|
||||||
|
|
||||||
// redacted stands in for a Referer that cannot be reduced to its
|
|
||||||
// scheme and host.
|
|
||||||
const redacted = "(redacted)"
|
|
||||||
|
|
||||||
// SchemeAndHost returns a Referer's scheme and host, never its path or
|
|
||||||
// query. The path can be an entrypoint URL, which carries the UUID: a
|
|
||||||
// sender that follows a redirect from http:// to https:// sends the
|
|
||||||
// URL it was redirected from, and a browser sends the 404 page's URL
|
|
||||||
// under /h/ when it fetches that page's stylesheet. A Referer without
|
|
||||||
// both a scheme and a host comes back as "(redacted)".
|
|
||||||
//
|
|
||||||
// It is the access log's referer field, and the Referer header the
|
|
||||||
// Sentry hook sends.
|
|
||||||
func SchemeAndHost(referer string) string {
|
|
||||||
if referer == "" {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
parsed, err := url.Parse(referer)
|
|
||||||
if err != nil || parsed.Scheme == "" || parsed.Host == "" {
|
|
||||||
return redacted
|
|
||||||
}
|
|
||||||
|
|
||||||
return parsed.Scheme + "://" + parsed.Host
|
|
||||||
}
|
|
||||||
@@ -1,67 +0,0 @@
|
|||||||
package referer_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"sneak.berlin/go/webhooker/internal/referer"
|
|
||||||
)
|
|
||||||
|
|
||||||
// entrypointUUID is the secret part of an entrypoint URL; no case may
|
|
||||||
// return it.
|
|
||||||
const entrypointUUID = "0b6f7d3e-6a39-4e3c-9f0a-2c4d8e1b5a77"
|
|
||||||
|
|
||||||
// redacted is what SchemeAndHost returns for a Referer it cannot
|
|
||||||
// reduce to a scheme and host.
|
|
||||||
const redacted = "(redacted)"
|
|
||||||
|
|
||||||
func TestSchemeAndHost(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
referer string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "no Referer",
|
|
||||||
referer: "",
|
|
||||||
want: "",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "entrypoint URL with a query",
|
|
||||||
referer: "http://webhooker.example/h/" + entrypointUUID +
|
|
||||||
"?src=ci",
|
|
||||||
want: "http://webhooker.example",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "host with a port",
|
|
||||||
referer: "https://webhooker.example:8443/h/" + entrypointUUID,
|
|
||||||
want: "https://webhooker.example:8443",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "path without a scheme or host",
|
|
||||||
referer: "/h/" + entrypointUUID,
|
|
||||||
want: redacted,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "host without a scheme",
|
|
||||||
referer: "webhooker.example/h/" + entrypointUUID,
|
|
||||||
want: redacted,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "not a URL",
|
|
||||||
referer: "http://[::1/h/" + entrypointUUID,
|
|
||||||
want: redacted,
|
|
||||||
},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
got := referer.SchemeAndHost(tc.referer)
|
|
||||||
|
|
||||||
assert.Equal(t, tc.want, got)
|
|
||||||
assert.NotContains(t, got, entrypointUUID)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -10,7 +10,6 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -18,7 +17,6 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"go.uber.org/fx/fxtest"
|
"go.uber.org/fx/fxtest"
|
||||||
"sneak.berlin/go/webhooker/internal/archive"
|
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/datadir"
|
"sneak.berlin/go/webhooker/internal/datadir"
|
||||||
@@ -135,13 +133,7 @@ type noopNotifier struct{}
|
|||||||
|
|
||||||
func (n *noopNotifier) Notify([]delivery.Task) {}
|
func (n *noopNotifier) Notify([]delivery.Task) {}
|
||||||
|
|
||||||
type noopArchives struct {
|
type noopArchives struct{}
|
||||||
renameMu sync.Mutex
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *noopArchives) RenameLock() *sync.Mutex {
|
|
||||||
return &n.renameMu
|
|
||||||
}
|
|
||||||
|
|
||||||
func (n *noopArchives) EvictWebhook(string) {}
|
func (n *noopArchives) EvictWebhook(string) {}
|
||||||
|
|
||||||
@@ -191,7 +183,7 @@ func newServerApp(
|
|||||||
healthcheck.New,
|
healthcheck.New,
|
||||||
session.New,
|
session.New,
|
||||||
func() delivery.Notifier { return &noopNotifier{} },
|
func() delivery.Notifier { return &noopNotifier{} },
|
||||||
func() archive.Archives { return &noopArchives{} },
|
func() delivery.Archives { return &noopArchives{} },
|
||||||
func() delivery.CircuitBreakers {
|
func() delivery.CircuitBreakers {
|
||||||
return &noopCircuitBreakers{}
|
return &noopCircuitBreakers{}
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -60,8 +60,8 @@ const (
|
|||||||
// TestAlpineRunsUnderTheSecurityPolicy loads the webhook page and the
|
// TestAlpineRunsUnderTheSecurityPolicy loads the webhook page and the
|
||||||
// event log in a headless browser, served by the real router and so
|
// event log in a headless browser, served by the real router and so
|
||||||
// under the real Content-Security-Policy, and checks that the pages'
|
// under the real Content-Security-Policy, and checks that the pages'
|
||||||
// Alpine.js directives, the copy control and the delete prompts work,
|
// Alpine.js directives and the copy control work, and that a target's
|
||||||
// and that a target's row shows its delivery counts.
|
// row shows its delivery counts.
|
||||||
func TestAlpineRunsUnderTheSecurityPolicy(t *testing.T) {
|
func TestAlpineRunsUnderTheSecurityPolicy(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -74,10 +74,6 @@ func TestAlpineRunsUnderTheSecurityPolicy(t *testing.T) {
|
|||||||
userID, _ := env.seedUser(t, "browser", "browser-password")
|
userID, _ := env.seedUser(t, "browser", "browser-password")
|
||||||
webhook, older, event, target := seedBrowserWebhook(t, env, userID)
|
webhook, older, event, target := seedBrowserWebhook(t, env, userID)
|
||||||
|
|
||||||
// The sign-in page is loaded before the browser signs in, since a
|
|
||||||
// signed-in visit to it is sent on to another page.
|
|
||||||
checkOtherPages(ctx, t, srv.URL+"/pages/login")
|
|
||||||
|
|
||||||
require.NoError(t, chromedp.Run(
|
require.NoError(t, chromedp.Run(
|
||||||
ctx, setCookies(srv.URL, env.authCookies(t, userID, "browser")),
|
ctx, setCookies(srv.URL, env.authCookies(t, userID, "browser")),
|
||||||
))
|
))
|
||||||
@@ -103,11 +99,6 @@ func TestAlpineRunsUnderTheSecurityPolicy(t *testing.T) {
|
|||||||
checkEventLog(ctx, t, page+"/events", event.ID, older.ID, target.Name)
|
checkEventLog(ctx, t, page+"/events", event.ID, older.ID, target.Name)
|
||||||
checkMobileMenu(ctx, t, page)
|
checkMobileMenu(ctx, t, page)
|
||||||
checkPhoneWidth(ctx, t, page, page+"/events", target.Name)
|
checkPhoneWidth(ctx, t, page, page+"/events", target.Name)
|
||||||
checkOtherPages(ctx, t,
|
|
||||||
srv.URL+"/hooks", srv.URL+"/settings", srv.URL+"/user/browser",
|
|
||||||
// A path that does not exist, which gets the 404 error page.
|
|
||||||
srv.URL+"/no-such-page")
|
|
||||||
checkDeletePrompt(ctx, t, page, target)
|
|
||||||
|
|
||||||
assert.Empty(t, problems(), "the browser reported problems")
|
assert.Empty(t, problems(), "the browser reported problems")
|
||||||
}
|
}
|
||||||
@@ -363,11 +354,11 @@ func checkAddEachTargetType(ctx context.Context, t *testing.T, url string) {
|
|||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
"http", "http",
|
"http", "http",
|
||||||
"csrf_token name type url headers timeout max_attempts",
|
"csrf_token name type url headers timeout max_retries",
|
||||||
map[string]string{"url": publicTargetURL},
|
map[string]string{"url": publicTargetURL},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"slack", "slack", "csrf_token name type url max_attempts",
|
"slack", "slack", "csrf_token name type url max_retries",
|
||||||
map[string]string{"url": publicTargetURL},
|
map[string]string{"url": publicTargetURL},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -643,7 +634,7 @@ func checkRefusedEdits(
|
|||||||
"#url": "http://127.0.0.1/hook",
|
"#url": "http://127.0.0.1/hook",
|
||||||
"#headers": "X-Edited: kept",
|
"#headers": "X-Edited: kept",
|
||||||
"#timeout": "12",
|
"#timeout": "12",
|
||||||
"#max_attempts": "3",
|
"#max_retries": "3",
|
||||||
}},
|
}},
|
||||||
{page + "/edit", map[string]string{
|
{page + "/edit", map[string]string{
|
||||||
"#name": "edited-webhook",
|
"#name": "edited-webhook",
|
||||||
@@ -1397,75 +1388,3 @@ func checkPhoneWidth(
|
|||||||
assert.False(t, sideways, "the event log scrolls sideways on a phone")
|
assert.False(t, sideways, "the event log scrolls sideways on a phone")
|
||||||
assert.Empty(t, cutOff, "the event log cuts these off on a phone")
|
assert.Empty(t, cutOff, "the event log cuts these off on a phone")
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkOtherPages loads the pages at urls, which no other check opens,
|
|
||||||
// so that anything the browser reports going wrong on them fails the
|
|
||||||
// test too.
|
|
||||||
func checkOtherPages(ctx context.Context, t *testing.T, urls ...string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for _, url := range urls {
|
|
||||||
require.NoError(t, chromedp.Run(ctx, loadPage(url)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// answerPrompt returns a script that stands in for the user at the
|
|
||||||
// browser's prompt: it makes window.confirm record the question in
|
|
||||||
// window.asked and give answer. It also records in window.sent whether
|
|
||||||
// a form is sent once its submit event is handled, from a listener on
|
|
||||||
// window, which the event reaches after the listener static/js/app.js
|
|
||||||
// adds to document.
|
|
||||||
func answerPrompt(answer bool) string {
|
|
||||||
return fmt.Sprintf(`
|
|
||||||
window.asked = "";
|
|
||||||
window.sent = false;
|
|
||||||
window.confirm = (question) => {
|
|
||||||
window.asked = question;
|
|
||||||
return %t;
|
|
||||||
};
|
|
||||||
window.addEventListener("submit", (event) => {
|
|
||||||
window.sent = !event.defaultPrevented;
|
|
||||||
});`, answer)
|
|
||||||
}
|
|
||||||
|
|
||||||
// checkDeletePrompt loads a webhook page and clicks Delete beside
|
|
||||||
// target. It checks that the browser asks first, naming the target,
|
|
||||||
// that answering no leaves the form unsent, and that answering yes
|
|
||||||
// sends it and the target is deleted.
|
|
||||||
func checkDeletePrompt(
|
|
||||||
ctx context.Context, t *testing.T, url string, target *database.Target,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
form := `form[action$="/targets/` + target.ID + `/delete"]`
|
|
||||||
question := `Delete target "` + target.Name + `"?` + "\n\n" +
|
|
||||||
"Nothing more is delivered to it. " +
|
|
||||||
"Its past deliveries stay in the event log."
|
|
||||||
|
|
||||||
var (
|
|
||||||
asked string
|
|
||||||
sent bool
|
|
||||||
listed bool
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, chromedp.Run(
|
|
||||||
ctx, loadPage(url), chromedp.Evaluate(answerPrompt(false), nil),
|
|
||||||
))
|
|
||||||
click(ctx, t, form+" button")
|
|
||||||
require.NoError(t, chromedp.Run(
|
|
||||||
ctx,
|
|
||||||
chromedp.Evaluate(`window.asked`, &asked),
|
|
||||||
chromedp.Evaluate(`window.sent`, &sent),
|
|
||||||
))
|
|
||||||
assert.Equal(t, question, asked, "Delete does not ask about the target")
|
|
||||||
assert.False(t, sent, "answering no still sends the delete form")
|
|
||||||
|
|
||||||
require.NoError(t, chromedp.Run(
|
|
||||||
ctx, chromedp.Evaluate(answerPrompt(true), nil),
|
|
||||||
))
|
|
||||||
clickAndLoad(ctx, t, form+" button")
|
|
||||||
require.NoError(t, chromedp.Run(ctx, chromedp.Evaluate(
|
|
||||||
`document.querySelector('`+form+`') !== null`, &listed,
|
|
||||||
)))
|
|
||||||
assert.False(t, listed, "answering yes does not delete the target")
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -31,7 +31,6 @@ func TestHook_DownloadArchive(t *testing.T) {
|
|||||||
Name: "kept",
|
Name: "kept",
|
||||||
Type: database.TargetTypeDatabase,
|
Type: database.TargetTypeDatabase,
|
||||||
Active: true,
|
Active: true,
|
||||||
MaxAttempts: 1,
|
|
||||||
}
|
}
|
||||||
require.NoError(t,
|
require.NoError(t,
|
||||||
env.db.DB().Omit(clause.Associations).Create(archive).Error,
|
env.db.DB().Omit(clause.Associations).Create(archive).Error,
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user