Compare commits
1
Commits
next
..
e82b1cc3df
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e82b1cc3df |
+2
-2
@@ -15,8 +15,8 @@ bin/
|
|||||||
# Extracted from 3p/ by `make assets` inside the build; a host copy is not
|
# Extracted from 3p/ by `make assets` inside the build; a host copy is not
|
||||||
# needed. The tarball in 3p/ must stay in the context.
|
# needed. The tarball in 3p/ must stay in the context.
|
||||||
static/js/alpine.min.js
|
static/js/alpine.min.js
|
||||||
# The js-deps stage installs ESLint and prettier; a host copy would overwrite
|
# The js-deps stage installs ESLint; a host copy would overwrite it at the
|
||||||
# them at the `COPY . .` of the stages built on it.
|
# js-lint stage's `COPY . .`.
|
||||||
node_modules/
|
node_modules/
|
||||||
.env
|
.env
|
||||||
.env.*
|
.env.*
|
||||||
|
|||||||
@@ -33,5 +33,5 @@ jobs:
|
|||||||
# and built cannot report success from cache.
|
# and built cannot report success from cache.
|
||||||
run: git rev-parse HEAD > .ci-fingerprint
|
run: git rev-parse HEAD > .ci-fingerprint
|
||||||
|
|
||||||
- name: Build Docker image (runs the gofmt check, golangci-lint, the stylesheet check, ESLint, the Markdown check, make test, make build)
|
- name: Build Docker image (runs make fmt-check, golangci-lint, the stylesheet check, ESLint, make test, make build)
|
||||||
run: script/cibuild
|
run: script/cibuild
|
||||||
|
|||||||
+1
-1
@@ -15,7 +15,7 @@ bin/
|
|||||||
# Go vendor directory
|
# Go vendor directory
|
||||||
vendor/
|
vendor/
|
||||||
|
|
||||||
# ESLint, prettier and their dependencies, installed from yarn.lock
|
# ESLint and its dependencies, installed from yarn.lock
|
||||||
node_modules/
|
node_modules/
|
||||||
|
|
||||||
# IDE specific files
|
# IDE specific files
|
||||||
|
|||||||
@@ -1,4 +0,0 @@
|
|||||||
{
|
|
||||||
"tabWidth": 4,
|
|
||||||
"proseWrap": "always"
|
|
||||||
}
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
# Install into node_modules/: the Dockerfile's lint and Markdown stages run
|
|
||||||
# ESLint and prettier from node_modules/.bin.
|
|
||||||
nodeLinker: node-modules
|
|
||||||
+16
-42
@@ -4,6 +4,8 @@
|
|||||||
# compile on Alpine musl (off64_t is a glibc type).
|
# compile on Alpine musl (off64_t is a glibc type).
|
||||||
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
|
FROM golangci/golangci-lint:v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 AS lint
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends make && rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
|
|
||||||
# Copy go mod files first for better layer caching
|
# Copy go mod files first for better layer caching
|
||||||
@@ -17,14 +19,12 @@ RUN go mod download
|
|||||||
# .dockerignore.
|
# .dockerignore.
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# Run the Go formatting check and the linter. gofmt and golangci-lint are
|
# Run formatting check and linter. golangci-lint is invoked directly rather
|
||||||
# invoked directly rather than through `make fmt-check` and `make lint`: this
|
# than through `make lint`: this stage is already the pinned linter image, and
|
||||||
# stage is already the pinned linter image, and both scripts build docker
|
# script/lint is a wrapper that builds Dockerfile.lint, so calling it here
|
||||||
# stages, so calling them here would need a docker daemon inside the build.
|
# would need a docker daemon inside the build. Keep these steps in step with
|
||||||
# The Markdown half of `make fmt-check` is the markdown-check stage below.
|
# Dockerfile.lint, including --network=none (see its header for why).
|
||||||
# Keep the golangci-lint steps in step with Dockerfile.lint, including
|
RUN make fmt-check
|
||||||
# --network=none (see its header for why).
|
|
||||||
RUN if [ -n "$(gofmt -s -l .)" ]; then echo "gofmt needed on:"; gofmt -s -l .; exit 1; fi
|
|
||||||
RUN script/assets
|
RUN script/assets
|
||||||
RUN --network=none golangci-lint config verify --config .golangci.yml
|
RUN --network=none golangci-lint config verify --config .golangci.yml
|
||||||
RUN --network=none golangci-lint run --config .golangci.yml --build-tags browser ./...
|
RUN --network=none golangci-lint run --config .golangci.yml --build-tags browser ./...
|
||||||
@@ -66,56 +66,30 @@ RUN sed 's/}/}\n/g' static/css/tailwind.css > /tmp/committed.css \
|
|||||||
}
|
}
|
||||||
|
|
||||||
# JavaScript lint stages: ESLint, at the version package.json and yarn.lock
|
# JavaScript lint stages: ESLint, at the version package.json and yarn.lock
|
||||||
# pin, checks static/js/ against eslint.config.mjs. js-deps installs it, and
|
# pin, checks static/js/ against eslint.config.mjs. js-deps installs it and
|
||||||
# prettier for the Markdown stages below, and stays cached until package.json,
|
# stays cached until those two files change. script/lint forces only js-lint
|
||||||
# yarn.lock or .yarnrc.yml changes. script/lint forces only js-lint to re-run,
|
# to re-run, and the build stage below runs it too. COPY . . brings in the CI
|
||||||
# and the build stage below runs it too. COPY . . brings in the CI cache
|
# cache barrier described in the lint stage above.
|
||||||
# barrier described in the lint stage above.
|
# node:24.21.0-alpine (LTS, with yarn 1.22.22), 2026-09-18
|
||||||
#
|
|
||||||
# The image's own corepack runs the yarn that package.json's packageManager
|
|
||||||
# field names, yarn 4.18.1 (released 2026-09-24), and checks it against the
|
|
||||||
# hash there. The image also ships yarn 1, which `corepack enable yarn`
|
|
||||||
# replaces.
|
|
||||||
# node:24.21.0-alpine (LTS), 2026-09-18
|
|
||||||
FROM node:24.21.0-alpine@sha256:ebfe2f90462722a7a4de65e91990e97fe0d401c70e0e762c5b53302f905ec1c1 AS js-deps
|
FROM node:24.21.0-alpine@sha256:ebfe2f90462722a7a4de65e91990e97fe0d401c70e0e762c5b53302f905ec1c1 AS js-deps
|
||||||
WORKDIR /src
|
WORKDIR /src
|
||||||
COPY package.json yarn.lock .yarnrc.yml ./
|
COPY package.json yarn.lock ./
|
||||||
RUN corepack enable yarn && yarn install --immutable --mode=skip-build
|
RUN yarn install --frozen-lockfile --ignore-scripts
|
||||||
|
|
||||||
FROM js-deps AS js-lint
|
FROM js-deps AS js-lint
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN --network=none node_modules/.bin/eslint static/js
|
RUN --network=none node_modules/.bin/eslint static/js
|
||||||
|
|
||||||
# Markdown stages: prettier, at the version package.json and yarn.lock pin,
|
|
||||||
# formats every Markdown file in the tree with the settings in .prettierrc.
|
|
||||||
# `make fmt` (script/fmt) writes the formatted files out from markdown-output.
|
|
||||||
# markdown-check fails on any file prettier would change; `make fmt-check`
|
|
||||||
# runs it, and so does the build stage below.
|
|
||||||
FROM js-deps AS markdown
|
|
||||||
COPY . .
|
|
||||||
RUN --network=none node_modules/.bin/prettier --write '**/*.md' \
|
|
||||||
&& mkdir /out \
|
|
||||||
&& find . -name '*.md' ! -path './node_modules/*' -exec cp -p --parents {} /out \;
|
|
||||||
|
|
||||||
FROM scratch AS markdown-output
|
|
||||||
COPY --from=markdown /out /
|
|
||||||
|
|
||||||
FROM js-deps AS markdown-check
|
|
||||||
COPY . .
|
|
||||||
RUN --network=none node_modules/.bin/prettier --check '**/*.md'
|
|
||||||
|
|
||||||
# Build stage
|
# Build stage
|
||||||
# golang:1.26.1-bookworm (Debian-based), 2026-03-17
|
# golang:1.26.1-bookworm (Debian-based), 2026-03-17
|
||||||
# Using Debian-based image because gorm.io/driver/sqlite pulls in
|
# Using Debian-based image because gorm.io/driver/sqlite pulls in
|
||||||
# mattn/go-sqlite3 (CGO), which does not compile on Alpine musl.
|
# mattn/go-sqlite3 (CGO), which does not compile on Alpine musl.
|
||||||
FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a3492282a6c820bf4755fd64a4 AS builder
|
FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a3492282a6c820bf4755fd64a4 AS builder
|
||||||
|
|
||||||
# Depend on the lint, stylesheet check, JavaScript lint and Markdown check
|
# Depend on the lint, stylesheet check and JavaScript lint stages passing
|
||||||
# stages passing
|
|
||||||
COPY --from=lint /src/go.sum /dev/null
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
COPY --from=css-check /out/tailwind.css /dev/null
|
COPY --from=css-check /out/tailwind.css /dev/null
|
||||||
COPY --from=js-lint /src/yarn.lock /dev/null
|
COPY --from=js-lint /src/yarn.lock /dev/null
|
||||||
COPY --from=markdown-check /src/yarn.lock /dev/null
|
|
||||||
|
|
||||||
# jq is a runtime dependency of script/ci-mark-superseded, which the test
|
# jq is a runtime dependency of script/ci-mark-superseded, which the test
|
||||||
# suite executes. git is what script/version derives the version with.
|
# suite executes. git is what script/version derives the version with.
|
||||||
|
|||||||
@@ -2,367 +2,403 @@
|
|||||||
|
|
||||||
One issue per unit of work, one branch and one PR per issue:
|
One issue per unit of work, one branch and one PR per issue:
|
||||||
|
|
||||||
- ensure a tracked issue exists with a definition of done
|
* ensure a tracked issue exists with a definition of done
|
||||||
- branch from `next` (never from `main`)
|
* branch from `next` (never from `main`)
|
||||||
- do the work; open a PR based on `next` (never on `main`)
|
* do the work; open a PR based on `next` (never on `main`)
|
||||||
- pass an independent review, then the manager squash-merges into `next`
|
* pass an independent review, then the manager squash-merges into `next`
|
||||||
- push; nothing stays local-only
|
* push; nothing stays local-only
|
||||||
|
|
||||||
`next` is the branch for the next milestone and must stay green and mergeable to
|
`next` is the branch for the next milestone and must stay green and
|
||||||
`main` without notice. One `next` -> `main` PR accumulates the milestone;
|
mergeable to `main` without notice. One `next` -> `main` PR accumulates
|
||||||
releases are cut from `main` separately.
|
the milestone; releases are cut from `main` separately.
|
||||||
|
|
||||||
Issue branches do NOT touch this file — the manager maintains it on `next`.
|
Issue branches do NOT touch this file — the manager maintains it on
|
||||||
Every branch editing `TODO.md` conflicts with every other (#112).
|
`next`. Every branch editing `TODO.md` conflicts with every other
|
||||||
|
(#112).
|
||||||
|
|
||||||
# Status
|
# Status
|
||||||
|
|
||||||
The milestone (https://git.eeqj.de/sneak/webhooker/milestone/9) is the
|
The milestone (https://git.eeqj.de/sneak/webhooker/milestone/9) is the
|
||||||
authoritative list, and the only place to read a count or a state of play from.
|
authoritative list, and the only place to read a count or a state of
|
||||||
This file records where the project is, not what is in flight: a sentence whose
|
play from. This file records where the project is, not what is in
|
||||||
truth depends on a branch being unmerged is wrong the moment it merges, and this
|
flight: a sentence whose truth depends on a branch being unmerged is
|
||||||
file has been wrong that way before.
|
wrong the moment it merges, and this file has been wrong that way
|
||||||
|
before.
|
||||||
|
|
||||||
The durability defect that held the tag has landed
|
The durability defect that held the tag has landed
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/256, commit `8d64259`). Every SQLite
|
(https://git.eeqj.de/sneak/webhooker/issues/256, commit `8d64259`).
|
||||||
handle opens with WAL journaling and a busy timeout, a bookkeeping write that
|
Every SQLite handle opens with WAL journaling and a busy timeout, a
|
||||||
fails leaves its delivery in a recoverable state rather than a lying one, and
|
bookkeeping write that fails leaves its delivery in a recoverable
|
||||||
recovery skips a delivery that already has a successful result row. Final
|
state rather than a lying one, and recovery skips a delivery that
|
||||||
pre-tag verification exercised it and confirmed it holds. Whatever the milestone
|
already has a successful result row. Final pre-tag verification
|
||||||
still shows open is what remains before `v1.0.0`.
|
exercised it and confirmed it holds. Whatever the milestone still
|
||||||
|
shows open is what remains before `v1.0.0`.
|
||||||
|
|
||||||
Delivery is at-least-once by design, not by accident: a send whose result row
|
Delivery is at-least-once by design, not by accident: a send whose
|
||||||
does not land is attempted again, so a receiver can see a duplicate. That is
|
result row does not land is attempted again, so a receiver can see a
|
||||||
deliberate — the alternative is a silent lost delivery — and the README says so
|
duplicate. That is deliberate — the alternative is a silent lost
|
||||||
under Rationale. It is not a defect to re-file.
|
delivery — and the README says so under Rationale. It is not a defect
|
||||||
|
to re-file.
|
||||||
|
|
||||||
# Next Step
|
# Next Step
|
||||||
|
|
||||||
Clear the rest of the open 1.0.0 milestone
|
Clear the rest of the open 1.0.0 milestone
|
||||||
(https://git.eeqj.de/sneak/webhooker/milestone/9) and tag `v1.0.0`. Merging
|
(https://git.eeqj.de/sneak/webhooker/milestone/9) and tag `v1.0.0`.
|
||||||
`next` into `main` is a separate act from tagging and waits on neither of those:
|
Merging `next` into `main` is a separate act from tagging and waits on
|
||||||
`next` is kept mergeable at all times, which is the point of the branch.
|
neither of those: `next` is kept mergeable at all times, which is the
|
||||||
|
point of the branch.
|
||||||
|
|
||||||
# Completed Steps
|
# Completed Steps
|
||||||
|
|
||||||
- 2026-08-24 Bind the plaintext HTTP listener deliberately, via `BIND_ADDRESS`
|
- 2026-08-24 Bind the plaintext HTTP listener deliberately, via
|
||||||
defaulting to `127.0.0.1`, and document the reverse-proxy deployment. A
|
`BIND_ADDRESS` defaulting to `127.0.0.1`, and document the
|
||||||
hostname, an empty value or a value carrying a port is a startup error, and
|
reverse-proxy deployment. A hostname, an empty value or a value
|
||||||
the `Dockerfile` sets `0.0.0.0` because a loopback bind inside a container is
|
carrying a port is a startup error, and the `Dockerfile` sets
|
||||||
unreachable (https://git.eeqj.de/sneak/webhooker/issues/268). The same commit
|
`0.0.0.0` because a loopback bind inside a container is unreachable
|
||||||
removed the shutdown race: `httpServer` is built in the constructor rather
|
(https://git.eeqj.de/sneak/webhooker/issues/268). The same commit
|
||||||
than assigned from the serving goroutine, which orders the write before every
|
removed the shutdown race: `httpServer` is built in the constructor
|
||||||
fx hook and rules out the nil dereference a SIGTERM arriving first would have
|
rather than assigned from the serving goroutine, which orders the
|
||||||
caused, and `sentryEnabled` is an `atomic.Bool`
|
write before every fx hook and rules out the nil dereference a
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/226)
|
SIGTERM arriving first would have caused, and `sentryEnabled` is an
|
||||||
- 2026-08-24 Remove inbound request signature verification. The entrypoint UUID
|
`atomic.Bool` (https://git.eeqj.de/sneak/webhooker/issues/226)
|
||||||
is the authentication secret, so the per-entrypoint shared secret, the
|
- 2026-08-24 Remove inbound request signature verification. The
|
||||||
`internal/signature` package, the receiver check, the model fields and the
|
entrypoint UUID is the authentication secret, so the per-entrypoint
|
||||||
forms are all gone. This reverses the feature that landed earlier in the same
|
shared secret, the `internal/signature` package, the receiver check,
|
||||||
milestone (https://git.eeqj.de/sneak/webhooker/issues/67,
|
the model fields and the forms are all gone. This reverses the
|
||||||
|
feature that landed earlier in the same milestone
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/67,
|
||||||
https://git.eeqj.de/sneak/webhooker/issues/279)
|
https://git.eeqj.de/sneak/webhooker/issues/279)
|
||||||
- 2026-08-24 Stamp the build version into the binary and render it in the UI
|
- 2026-08-24 Stamp the build version into the binary and render it in
|
||||||
footer. `script/version` is the single source — `$VERSION`, else
|
the UI footer. `script/version` is the single source — `$VERSION`,
|
||||||
`git describe --tags --always --dirty`, else `unknown` — so a `make build`
|
else `git describe --tags --always --dirty`, else `unknown` — so a
|
||||||
binary and a `make docker` image from one checkout report the same thing, and
|
`make build` binary and a `make docker` image from one checkout
|
||||||
nothing in it varies between two builds of the same commit, which the release
|
report the same thing, and nothing in it varies between two builds
|
||||||
gate's byte-identical assertion would catch
|
of the same commit, which the release gate's byte-identical
|
||||||
|
assertion would catch
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/253)
|
(https://git.eeqj.de/sneak/webhooker/issues/253)
|
||||||
- 2026-08-24 Derive cookie `Secure` and CSRF strictness from the request
|
- 2026-08-24 Derive cookie `Secure` and CSRF strictness from the
|
||||||
transport rather than from `WEBHOOKER_ENVIRONMENT`. Behind a real TLS proxy
|
request transport rather than from `WEBHOOKER_ENVIRONMENT`. Behind a
|
||||||
with the environment left at its `dev` default, the session cookie silently
|
real TLS proxy with the environment left at its `dev` default, the
|
||||||
lost `Secure` while the CSRF cookie on the same response kept it.
|
session cookie silently lost `Secure` while the CSRF cookie on the
|
||||||
`X-Forwarded-Proto` is now matched case-insensitively on its first
|
same response kept it. `X-Forwarded-Proto` is now matched
|
||||||
comma-separated element, so `HTTPS` and `https, http` no longer fall to the
|
case-insensitively on its first comma-separated element, so `HTTPS`
|
||||||
relaxed CSRF path (https://git.eeqj.de/sneak/webhooker/issues/269)
|
and `https, http` no longer fall to the relaxed CSRF path
|
||||||
- 2026-08-24 Roll back a failed webhook deletion instead of committing it. A
|
(https://git.eeqj.de/sneak/webhooker/issues/269)
|
||||||
failing delete committed whatever had already succeeded, hard-deleted the
|
- 2026-08-24 Roll back a failed webhook deletion instead of committing
|
||||||
per-webhook event database anyway, and redirected as though it had worked —
|
it. A failing delete committed whatever had already succeeded,
|
||||||
orphaned config plus permanently destroyed history, reported as success. All
|
hard-deleted the per-webhook event database anyway, and redirected as
|
||||||
three delete positions now roll back with the event database intact
|
though it had worked — orphaned config plus permanently destroyed
|
||||||
|
history, reported as success. All three delete positions now roll
|
||||||
|
back with the event database intact
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/262)
|
(https://git.eeqj.de/sneak/webhooker/issues/262)
|
||||||
- 2026-08-24 Name a deleted target on its historical deliveries, marked
|
- 2026-08-24 Name a deleted target on its historical deliveries, marked
|
||||||
`(deleted)`, rather than leaving the event log unable to say where a delivery
|
`(deleted)`, rather than leaving the event log unable to say where a
|
||||||
went. A deleted target's credentials stay masked exactly as a live one's, and
|
delivery went. A deleted target's credentials stay masked exactly as
|
||||||
it cannot become deliverable again through the receiver, resubmit, replay, the
|
a live one's, and it cannot become deliverable again through the
|
||||||
edit form or the toggle (https://git.eeqj.de/sneak/webhooker/issues/211)
|
receiver, resubmit, replay, the edit form or the toggle
|
||||||
- 2026-08-24 Bound both request-controlled `/metrics` label dimensions, so the
|
(https://git.eeqj.de/sneak/webhooker/issues/211)
|
||||||
unauthenticated receiver is no longer a memory-exhaustion vector: `handler`
|
- 2026-08-24 Bound both request-controlled `/metrics` label dimensions,
|
||||||
carries the chi route pattern, and `method` folds anything chi cannot route
|
so the unauthenticated receiver is no longer a memory-exhaustion
|
||||||
onto a single `(unmatched)` sentinel. Both were reproduced before the fix —
|
vector: `handler` carries the chi route pattern, and `method` folds
|
||||||
300 random method tokens took the series count from 106 to 7,631, and path
|
anything chi cannot route onto a single `(unmatched)` sentinel. Both
|
||||||
flooding reached 62,532 — and a label audit across a live scrape found no
|
were reproduced before the fix — 300 random method tokens took the
|
||||||
third unbounded dimension (https://git.eeqj.de/sneak/webhooker/issues/254,
|
series count from 106 to 7,631, and path flooding reached 62,532 —
|
||||||
|
and a label audit across a live scrape found no third unbounded
|
||||||
|
dimension (https://git.eeqj.de/sneak/webhooker/issues/254,
|
||||||
https://git.eeqj.de/sneak/webhooker/issues/261)
|
https://git.eeqj.de/sneak/webhooker/issues/261)
|
||||||
- 2026-08-24 Validate `max_retries` on both target forms. `abc`, `2.7` and `-5`
|
- 2026-08-24 Validate `max_retries` on both target forms. `abc`, `2.7`
|
||||||
silently became 0 — fire-and-forget — including on the edit path, where it
|
and `-5` silently became 0 — fire-and-forget — including on the edit
|
||||||
destroyed a working value, and `999999999` stored verbatim. The ceiling of 20
|
path, where it destroyed a working value, and `999999999` stored
|
||||||
is the `max` both templates already declared
|
verbatim. The ceiling of 20 is the `max` both templates already
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/221)
|
declared (https://git.eeqj.de/sneak/webhooker/issues/221)
|
||||||
- 2026-08-24 Resubmit a stored event as a new undelivered event, so a backend
|
- 2026-08-24 Resubmit a stored event as a new undelivered event, so a
|
||||||
under development can be tested against real captured traffic. Per-delivery
|
backend under development can be tested against real captured
|
||||||
replay cannot serve that: it re-sends one finished delivery to its own
|
traffic. Per-delivery replay cannot serve that: it re-sends one
|
||||||
original target, and a target created for a dev backend has no prior delivery
|
finished delivery to its own original target, and a target created
|
||||||
to replay. Resubmit re-injects the stored event at the top of the receiver
|
for a dev backend has no prior delivery to replay. Resubmit
|
||||||
path and fans it out to whatever targets are active now
|
re-injects the stored event at the top of the receiver path and fans
|
||||||
|
it out to whatever targets are active now
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/250)
|
(https://git.eeqj.de/sneak/webhooker/issues/250)
|
||||||
- 2026-08-20 Take an exclusive lock on `DATA_DIR` at startup, so two instances
|
- 2026-08-20 Take an exclusive lock on `DATA_DIR` at startup, so two
|
||||||
on one directory cannot both deliver
|
instances on one directory cannot both deliver
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/201)
|
(https://git.eeqj.de/sneak/webhooker/issues/201)
|
||||||
- 2026-08-20 Shut down the app when the HTTP listener fails. The `OnStart` hook
|
- 2026-08-20 Shut down the app when the HTTP listener fails. The
|
||||||
returned as soon as the serving goroutine was spawned, so a failed listen left
|
`OnStart` hook returned as soon as the serving goroutine was
|
||||||
fx reporting RUNNING and a live process with nothing bound — invisible to
|
spawned, so a failed listen left fx reporting RUNNING and a live
|
||||||
systemd and Docker restart policies
|
process with nothing bound — invisible to systemd and Docker restart
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/200)
|
policies (https://git.eeqj.de/sneak/webhooker/issues/200)
|
||||||
- 2026-08-20 Stop target credentials leaking into the per-webhook event
|
- 2026-08-20 Stop target credentials leaking into the per-webhook event
|
||||||
databases (https://git.eeqj.de/sneak/webhooker/issues/206), log SQL with
|
databases (https://git.eeqj.de/sneak/webhooker/issues/206), log SQL
|
||||||
placeholders rather than bound values
|
with placeholders rather than bound values
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/207), and fail loudly on half-set
|
(https://git.eeqj.de/sneak/webhooker/issues/207), and fail loudly on
|
||||||
metrics auth credentials (https://git.eeqj.de/sneak/webhooker/issues/205)
|
half-set metrics auth credentials
|
||||||
- 2026-08-20 Read queue depths with `Find`, not `Scan`. `Scan` swaps GORM's own
|
(https://git.eeqj.de/sneak/webhooker/issues/205)
|
||||||
trace recorder in for the logging adapter, and that recorder does not
|
- 2026-08-20 Read queue depths with `Find`, not `Scan`. `Scan` swaps
|
||||||
implement `gorm.ParamsFilter`, so those statements logged their bound values
|
GORM's own trace recorder in for the logging adapter, and that
|
||||||
interpolated and bypassed the suppression above. The two units gated green
|
recorder does not implement `gorm.ParamsFilter`, so those statements
|
||||||
against a `next` that lacked the other, and `next` went red when both landed
|
logged their bound values interpolated and bypassed the suppression
|
||||||
|
above. The two units gated green against a `next` that lacked the
|
||||||
|
other, and `next` went red when both landed
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/234)
|
(https://git.eeqj.de/sneak/webhooker/issues/234)
|
||||||
- 2026-08-20 Render per-attempt delivery detail in the event log
|
- 2026-08-20 Render per-attempt delivery detail in the event log
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/202) and add replay of a
|
(https://git.eeqj.de/sneak/webhooker/issues/202) and add replay of a
|
||||||
terminally failed delivery (https://git.eeqj.de/sneak/webhooker/issues/203)
|
terminally failed delivery
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/203)
|
||||||
- 2026-08-20 Expose delivery metrics on `/metrics`
|
- 2026-08-20 Expose delivery metrics on `/metrics`
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/209) and document the backup,
|
(https://git.eeqj.de/sneak/webhooker/issues/209) and document the
|
||||||
restore and upgrade procedures
|
backup, restore and upgrade procedures
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/210)
|
(https://git.eeqj.de/sneak/webhooker/issues/210)
|
||||||
- 2026-08-20 Add a `webhooker resetpw` subcommand and a bootstrap banner. The
|
- 2026-08-20 Add a `webhooker resetpw` subcommand and a bootstrap
|
||||||
admin bootstrap password was printed once among roughly 45 fx lines, and under
|
banner. The admin bootstrap password was printed once among roughly
|
||||||
`docker run -d` went to container logs subject to rotation; there was no reset
|
45 fx lines, and under `docker run -d` went to container logs subject
|
||||||
path at all, so recovery meant hand-deleting the users row, documented
|
to rotation; there was no reset path at all, so recovery meant
|
||||||
nowhere. The password is read from stdin or generated, never from argv where
|
hand-deleting the users row, documented nowhere. The password is read
|
||||||
`/proc` would publish it (https://git.eeqj.de/sneak/webhooker/issues/208)
|
from stdin or generated, never from argv where `/proc` would publish
|
||||||
- 2026-08-20 Add `ALLOWED_EGRESS_CIDRS`, an allowlist-only escape hatch for the
|
it (https://git.eeqj.de/sneak/webhooker/issues/208)
|
||||||
SSRF guard, so a self-hosted proxy can forward into the operator's own
|
- 2026-08-20 Add `ALLOWED_EGRESS_CIDRS`, an allowlist-only escape hatch
|
||||||
network. The guard's always-blocked set cannot be reopened by configuration
|
for the SSRF guard, so a self-hosted proxy can forward into the
|
||||||
|
operator's own network. The guard's always-blocked set cannot be
|
||||||
|
reopened by configuration
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/204)
|
(https://git.eeqj.de/sneak/webhooker/issues/204)
|
||||||
- 2026-08-20 Harden operator-set target headers, which were carried unsafely
|
- 2026-08-20 Harden operator-set target headers, which were carried
|
||||||
across a redirect (https://git.eeqj.de/sneak/webhooker/issues/233)
|
unsafely across a redirect
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/233)
|
||||||
- 2026-08-20 Add a target edit form with headers and timeout fields
|
- 2026-08-20 Add a target edit form with headers and timeout fields
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/127)
|
(https://git.eeqj.de/sneak/webhooker/issues/127)
|
||||||
- 2026-08-18 Raise `script/test`'s per-package timeout from 30s to 90s, matching
|
- 2026-08-18 Raise `script/test`'s per-package timeout from 30s to 90s,
|
||||||
the org-wide backstop. `go test` applies `-timeout` per package, and
|
matching the org-wide backstop. `go test` applies `-timeout` per
|
||||||
`internal/handlers` had grown past the old budget: a cache-defeated build
|
package, and `internal/handlers` had grown past the old budget: a
|
||||||
failed outright at `GOMAXPROCS=4`, and every run under deliberate host load
|
cache-defeated build failed outright at `GOMAXPROCS=4`, and every run
|
||||||
breached 30s. The measurement table lives in the script (#194)
|
under deliberate host load breached 30s. The measurement table lives
|
||||||
- 2026-08-18 Re-sync `REPO_POLICIES.md` from `prompts`. The local copy was stale
|
in the script (#194)
|
||||||
and still mandated a 20s test target with a 30s timeout, which the org
|
- 2026-08-18 Re-sync `REPO_POLICIES.md` from `prompts`. The local copy
|
||||||
replaced with a 60s cap and a 90s backstop. A synced copy is not a source;
|
was stale and still mandated a 20s test target with a 30s timeout,
|
||||||
reading it as one nearly produced a PR against `prompts` proposing a change
|
which the org replaced with a 60s cap and a 90s backstop. A synced
|
||||||
already merged there (#196)
|
copy is not a source; reading it as one nearly produced a PR against
|
||||||
- 2026-08-18 Report handler panics through the logger and answer 500. chi
|
`prompts` proposing a change already merged there (#196)
|
||||||
v1.5.5's `Recoverer` scans for a `panic(0x` frame the runtime no longer emits,
|
- 2026-08-18 Report handler panics through the logger and answer 500.
|
||||||
then indexes `pkg[-1:]`, so it panicked inside its own stack printer before
|
chi v1.5.5's `Recoverer` scans for a `panic(0x` frame the runtime no
|
||||||
writing a byte: the recovery never ran, the client got a dropped connection
|
longer emits, then indexes `pkg[-1:]`, so it panicked inside its own
|
||||||
instead of a 500, and the original panic was lost. A local middleware replaces
|
stack printer before writing a byte: the recovery never ran, the
|
||||||
it, bounded by `MaxPanicLogLineBytes` (#187)
|
client got a dropped connection instead of a 500, and the original
|
||||||
- 2026-08-18 Route GORM's logger through `slog` and bound it. Every `gorm.Open`
|
panic was lost. A local middleware replaces it, bounded by
|
||||||
left `logger.Default` in place at `Warn` with `IgnoreRecordNotFoundError`
|
`MaxPanicLogLineBytes` (#187)
|
||||||
false, so **every record-not-found printed the fully interpolated SQL to
|
- 2026-08-18 Route GORM's logger through `slog` and bound it. Every
|
||||||
stdout** — including the client-chosen path on `/webhook/{uuid}` and the
|
`gorm.Open` left `logger.Default` in place at `Warn` with
|
||||||
submitted username on the login form, at no level the operator set and outside
|
`IgnoreRecordNotFoundError` false, so **every record-not-found
|
||||||
`internal/logger` entirely. Three call sites, not the two the issue named
|
printed the fully interpolated SQL to stdout** — including the
|
||||||
(#178)
|
client-chosen path on `/webhook/{uuid}` and the submitted username on
|
||||||
- 2026-08-18 Bound every `slog` line against client-chosen text. Eight sites
|
the login form, at no level the operator set and outside
|
||||||
reachable unauthenticated, found by reading every `slog` call in the tree
|
`internal/logger` entirely. Three call sites, not the two the issue
|
||||||
rather than only the one reported; the budget moved to a shared
|
named (#178)
|
||||||
`internal/logfield` so no second truncation exists. `DEBUG` being off by
|
- 2026-08-18 Bound every `slog` line against client-chosen text. Eight
|
||||||
default is not a bound and is not treated as one (#176)
|
sites reachable unauthenticated, found by reading every `slog` call in
|
||||||
- 2026-08-18 Stop a slow host turning a login-guard test into a segfault. A
|
the tree rather than only the one reported; the budget moved to a
|
||||||
non-fatal `assert` on an acquire result was dereferenced on the next line, so
|
shared `internal/logfield` so no second truncation exists. `DEBUG`
|
||||||
one timing miss killed the whole `internal/middleware` binary and reddened CI
|
being off by default is not a bound and is not treated as one (#176)
|
||||||
for unrelated PRs. The fix also removed a real production race — `acquire`
|
- 2026-08-18 Stop a slow host turning a login-guard test into a
|
||||||
could shed a request with a slot standing free, because Go picks uniformly
|
segfault. A non-fatal `assert` on an acquire result was dereferenced
|
||||||
among ready `select` cases (#186)
|
on the next line, so one timing miss killed the whole
|
||||||
- 2026-08-18 Send the chi route pattern to Sentry rather than the concrete path.
|
`internal/middleware` binary and reddened CI for unrelated PRs. The
|
||||||
The receiver's path carries the entrypoint capability token, so every Sentry
|
fix also removed a real production race — `acquire` could shed a
|
||||||
event from `/webhook/{uuid}` shipped a live credential to a third party.
|
request with a slot standing free, because Go picks uniformly among
|
||||||
Request `Data`, `QueryString`, `Cookies` and `Env` are dropped and headers
|
ready `select` cases (#186)
|
||||||
reduced to an allowlist (#179)
|
- 2026-08-18 Send the chi route pattern to Sentry rather than the
|
||||||
- 2026-08-18 Read form fields from the POST body only. `r.FormValue` merges the
|
concrete path. The receiver's path carries the entrypoint capability
|
||||||
query string, so a login could be driven by URL parameters — putting the
|
token, so every Sentry event from `/webhook/{uuid}` shipped a live
|
||||||
password somewhere that lands in access logs, proxy logs and browser history
|
credential to a third party. Request `Data`, `QueryString`, `Cookies`
|
||||||
(#160)
|
and `Env` are dropped and headers reduced to an allowlist (#179)
|
||||||
- 2026-08-18 Verify login credentials before spending rate-limit budget, so a
|
- 2026-08-18 Read form fields from the POST body only. `r.FormValue`
|
||||||
flood of wrong passwords cannot lock out the account it is guessing at. The
|
merges the query string, so a login could be driven by URL parameters
|
||||||
manager took this decision rather than stall the queue; it is flagged on the
|
— putting the password somewhere that lands in access logs, proxy
|
||||||
issue for reversal (#150)
|
logs and browser history (#160)
|
||||||
- 2026-08-18 Run all linting in Docker via `Dockerfile.lint`. Host lint was
|
- 2026-08-18 Verify login credentials before spending rate-limit
|
||||||
wrong in both directions from version skew and shared caches. `script/lint`
|
budget, so a flood of wrong passwords cannot lock out the account it
|
||||||
asserts the summary line, because `--no-cache-filter` silently ignores a stage
|
is guessing at. The manager took this decision rather than stall the
|
||||||
name it does not match — the flag that makes the gate meaningful fails open
|
queue; it is flagged on the issue for reversal (#150)
|
||||||
(#109)
|
- 2026-08-18 Run all linting in Docker via `Dockerfile.lint`. Host lint
|
||||||
- 2026-08-18 Serve an event's full stored body over HTTP. The list query
|
was wrong in both directions from version skew and shared caches.
|
||||||
truncates for rendering, and that truncated value was the only way to read a
|
`script/lint` asserts the summary line, because `--no-cache-filter`
|
||||||
body, so the full payload was unreachable (#157)
|
silently ignores a stage name it does not match — the flag that makes
|
||||||
|
the gate meaningful fails open (#109)
|
||||||
|
- 2026-08-18 Serve an event's full stored body over HTTP. The list
|
||||||
|
query truncates for rendering, and that truncated value was the only
|
||||||
|
way to read a body, so the full payload was unreachable (#157)
|
||||||
- 2026-08-18 Bound the access log line against client-chosen text.
|
- 2026-08-18 Bound the access log line against client-chosen text.
|
||||||
`internal/logfield` budgets by _encoded_ bytes, not runes, so a handler's JSON
|
`internal/logfield` budgets by *encoded* bytes, not runes, so a
|
||||||
escaping cannot multiply a field past its allowance (#146)
|
handler's JSON escaping cannot multiply a field past its allowance
|
||||||
- 2026-08-18 Mark superseded CI commits `failure` rather than `skipped`. A
|
(#146)
|
||||||
skipped run rolls up green, so a commit that was never tested reported success
|
- 2026-08-18 Mark superseded CI commits `failure` rather than
|
||||||
(#152)
|
`skipped`. A skipped run rolls up green, so a commit that was never
|
||||||
- 2026-08-18 Set `fx.StopTimeout` inside the container stop grace, so shutdown
|
tested reported success (#152)
|
||||||
hooks are bounded by a deadline the orchestrator will actually honour rather
|
- 2026-08-18 Set `fx.StopTimeout` inside the container stop grace, so
|
||||||
than being killed mid-flush (#134)
|
shutdown hooks are bounded by a deadline the orchestrator will
|
||||||
- 2026-08-17 Bucket IPv6 rate-limit keys by `/64`. A single allocation hands out
|
actually honour rather than being killed mid-flush (#134)
|
||||||
2^64 addresses, so per-address keying let one client mint unlimited buckets.
|
- 2026-08-17 Bucket IPv6 rate-limit keys by `/64`. A single allocation
|
||||||
Manager decision, recorded on the issue (#125)
|
hands out 2^64 addresses, so per-address keying let one client mint
|
||||||
- 2026-08-17 Correct release-blocking README and startup-warning inaccuracies,
|
unlimited buckets. Manager decision, recorded on the issue (#125)
|
||||||
including claims about behaviour the code does not have (#151)
|
- 2026-08-17 Correct release-blocking README and startup-warning
|
||||||
|
inaccuracies, including claims about behaviour the code does not have
|
||||||
|
(#151)
|
||||||
- 2026-08-17 Fetch and verify Alpine.js at build time against
|
- 2026-08-17 Fetch and verify Alpine.js at build time against
|
||||||
`static/vendor.sha256` instead of committing the minified blob, so the
|
`static/vendor.sha256` instead of committing the minified blob, so
|
||||||
dependency is pinned by hash rather than by trust (#145)
|
the dependency is pinned by hash rather than by trust (#145)
|
||||||
- 2026-08-17 Bound the event log's rendered bodies in the query itself, so a
|
- 2026-08-17 Bound the event log's rendered bodies in the query itself,
|
||||||
large stored payload cannot be read into memory just to be truncated for
|
so a large stored payload cannot be read into memory just to be
|
||||||
display (#135)
|
truncated for display (#135)
|
||||||
- 2026-08-17 Mask the `http` target's destination URL in the UI: it can carry a
|
- 2026-08-17 Mask the `http` target's destination URL in the UI: it can
|
||||||
bearer credential in its path or query, and was rendered verbatim. Manager
|
carry a bearer credential in its path or query, and was rendered
|
||||||
decision to mask unconditionally (#115)
|
verbatim. Manager decision to mask unconditionally (#115)
|
||||||
- 2026-08-14 Bound shutdown hooks by their stop context, so a hook that hangs
|
- 2026-08-14 Bound shutdown hooks by their stop context, so a hook that
|
||||||
cannot hold the process past its grace period (#102)
|
hangs cannot hold the process past its grace period (#102)
|
||||||
- 2026-08-14 Render templates via a buffer rather than the `ResponseWriter`, so
|
- 2026-08-14 Render templates via a buffer rather than the
|
||||||
a template error part-way through cannot commit a 200 and then fail — the
|
`ResponseWriter`, so a template error part-way through cannot commit
|
||||||
response is written only once it is whole (#123)
|
a 200 and then fail — the response is written only once it is whole
|
||||||
- 2026-08-14 Align the session codec's max-age with the 7-day absolute cap. The
|
(#123)
|
||||||
codec accepted cookies the session layer considered expired, so the cap was
|
- 2026-08-14 Align the session codec's max-age with the 7-day absolute
|
||||||
enforced in one place and not the other (#108)
|
cap. The codec accepted cookies the session layer considered expired,
|
||||||
- 2026-08-12 Warn when `TRUSTED_PROXIES` is empty in production, where the safe
|
so the cap was enforced in one place and not the other (#108)
|
||||||
default silently discards forwarded headers and every client rate-limits as
|
- 2026-08-12 Warn when `TRUSTED_PROXIES` is empty in production, where
|
||||||
the proxy's address (#149)
|
the safe default silently discards forwarded headers and every client
|
||||||
- 2026-08-12 Bound the receiver rate limit per client IP across the whole
|
rate-limits as the proxy's address (#149)
|
||||||
`/webhook/*` route. The existing limiter keyed on the request path and
|
- 2026-08-12 Bound the receiver rate limit per client IP across the
|
||||||
`/webhook/{uuid}` matches any single segment, so a client that invented a
|
whole `/webhook/*` route. The existing limiter keyed on the request
|
||||||
fresh path per request minted a fresh bucket per request: the limit on the
|
path and `/webhook/{uuid}` matches any single segment, so a client
|
||||||
only unauthenticated endpoint bounded nothing in aggregate, and every request
|
that invented a fresh path per request minted a fresh bucket per
|
||||||
still cost an entrypoint lookup before it 404ed. An outer limiter keyed on the
|
request: the limit on the only unauthenticated endpoint bounded
|
||||||
client address alone now bounds that, chained in front of the unchanged
|
nothing in aggregate, and every request still cost an entrypoint
|
||||||
|
lookup before it 404ed. An outer limiter keyed on the client address
|
||||||
|
alone now bounds that, chained in front of the unchanged
|
||||||
per-entrypoint limiter (#139)
|
per-entrypoint limiter (#139)
|
||||||
- 2026-08-12 Correct release-blocking documentation inaccuracies: the README
|
- 2026-08-12 Correct release-blocking documentation inaccuracies: the
|
||||||
promised manual redelivery in the present tense in three places when nothing
|
README promised manual redelivery in the present tense in three
|
||||||
implements it (the same false claim also sat in the doc comment that was its
|
places when nothing implements it (the same false claim also sat in
|
||||||
source text), the env table omitted `RETENTION_SWEEP_INTERVAL`, and `TODO.md`
|
the doc comment that was its source text), the env table omitted
|
||||||
itself omitted five landed units (#141)
|
`RETENTION_SWEEP_INTERVAL`, and `TODO.md` itself omitted five landed
|
||||||
- 2026-08-12 Make the CI gate execute the checks it reports on. The workflow now
|
units (#141)
|
||||||
writes a build-context fingerprint before calling `script/cibuild`, so a code
|
- 2026-08-12 Make the CI gate execute the checks it reports on. The
|
||||||
commit invalidates the `COPY` layer of the lint and builder stages while a
|
workflow now writes a build-context fingerprint before calling
|
||||||
docs-only commit still replays from cache; a superseding run also rewrites the
|
`script/cibuild`, so a code commit invalidates the `COPY` layer of
|
||||||
`failure` status Gitea leaves on commits it cancelled and never tested.
|
the lint and builder stages while a docs-only commit still replays
|
||||||
Verified by pushing a deliberately broken test and watching CI go red (#119)
|
from cache; a superseding run also rewrites the `failure` status
|
||||||
- 2026-08-12 Require a positive `RETENTION_SWEEP_INTERVAL`: a non-positive value
|
Gitea leaves on commits it cancelled and never tested. Verified by
|
||||||
reached `time.NewTicker` in both the retention reaper and the archive sweeper,
|
pushing a deliberately broken test and watching CI go red (#119)
|
||||||
panicking two goroutines with no recover after startup had already reported
|
- 2026-08-12 Require a positive `RETENTION_SWEEP_INTERVAL`: a
|
||||||
success (#140)
|
non-positive value reached `time.NewTicker` in both the retention
|
||||||
- 2026-08-12 Bound the `X-Forwarded-For` scan's allocation to the hop cap: the
|
reaper and the archive sweeper, panicking two goroutines with no
|
||||||
reverse walk cuts entries with `strings.LastIndexByte` instead of joining and
|
recover after startup had already reported success (#140)
|
||||||
splitting, so a 1 MB header allocates 16 bytes rather than 1.6 MB per request
|
- 2026-08-12 Bound the `X-Forwarded-For` scan's allocation to the hop
|
||||||
on the unauthenticated receiver. Semantics proven unchanged by differential
|
cap: the reverse walk cuts entries with `strings.LastIndexByte`
|
||||||
testing against the previous implementation (#133)
|
instead of joining and splitting, so a 1 MB header allocates 16 bytes
|
||||||
|
rather than 1.6 MB per request on the unauthenticated receiver.
|
||||||
|
Semantics proven unchanged by differential testing against the
|
||||||
|
previous implementation (#133)
|
||||||
- 2026-08-12 Cap the `X-Forwarded-For` hop walk at 64 entries, so an
|
- 2026-08-12 Cap the `X-Forwarded-For` hop walk at 64 entries, so an
|
||||||
attacker-supplied chain cannot burn unbounded CPU in the rate-limit key
|
attacker-supplied chain cannot burn unbounded CPU in the rate-limit
|
||||||
function; running off the end falls back to the peer address (#124)
|
key function; running off the end falls back to the peer address
|
||||||
- 2026-08-12 Gate forwarded-header trust behind a `TRUSTED_PROXIES` CIDR list:
|
(#124)
|
||||||
all three rate limiters key on the connection's own address unless the direct
|
- 2026-08-12 Gate forwarded-header trust behind a `TRUSTED_PROXIES` CIDR
|
||||||
peer is a configured proxy, in which case `X-Forwarded-For` is walked right to
|
list: all three rate limiters key on the connection's own address
|
||||||
left for the first non-proxy hop. Default trusts nothing, and a
|
unless the direct peer is a configured proxy, in which case
|
||||||
set-but-unparseable value aborts startup. Before this, any client could mint a
|
`X-Forwarded-For` is walked right to left for the first non-proxy hop.
|
||||||
fresh bucket or drain another's by rotating a spoofed header (#88)
|
Default trusts nothing, and a set-but-unparseable value aborts
|
||||||
- 2026-08-11 Web UI cleanup: nav terminology unified on Webhooks, the Profile
|
startup. Before this, any client could mint a fresh bucket or drain
|
||||||
settings placeholder removed, a progressive-enhancement copy button for the
|
another's by rotating a spoofed header (#88)
|
||||||
entrypoint URL, and retention form copy that states the actual policy
|
- 2026-08-11 Web UI cleanup: nav terminology unified on Webhooks, the
|
||||||
(deletion by the reaper, 0 retains forever) (#57)
|
Profile settings placeholder removed, a progressive-enhancement copy
|
||||||
- 2026-08-11 Mask the webhook credential in delivery errors and logs: Go embeds
|
button for the entrypoint URL, and retention form copy that states the
|
||||||
the request URL in `*url.Error`, so every transport failure persisted the full
|
actual policy (deletion by the reaper, 0 retains forever) (#57)
|
||||||
Slack webhook URL into the per-webhook event database via
|
- 2026-08-11 Mask the webhook credential in delivery errors and logs:
|
||||||
`DeliveryResult.Error`, a field a future REST API would have served.
|
Go embeds the request URL in `*url.Error`, so every transport failure
|
||||||
`maskURLError` drops path, query and userinfo while preserving the wrapped
|
persisted the full Slack webhook URL into the per-webhook event
|
||||||
cause, so `errors.Is`/`As` and `Timeout()` still work and DNS, TLS and timeout
|
database via `DeliveryResult.Error`, a field a future REST API would
|
||||||
failures still read differently (#118)
|
have served. `maskURLError` drops path, query and userinfo while
|
||||||
|
preserving the wrapped cause, so `errors.Is`/`As` and `Timeout()`
|
||||||
|
still work and DNS, TLS and timeout failures still read differently
|
||||||
|
(#118)
|
||||||
- 2026-08-11 Rate-limit the public webhook receiver endpoint
|
- 2026-08-11 Rate-limit the public webhook receiver endpoint
|
||||||
(`RECEIVER_RATE_LIMIT`, default 120/min), keyed on client IP plus entrypoint
|
(`RECEIVER_RATE_LIMIT`, default 120/min), keyed on client IP plus
|
||||||
path so one entrypoint cannot exhaust another's budget; over-limit requests
|
entrypoint path so one entrypoint cannot exhaust another's budget;
|
||||||
get 429 with `Retry-After`. It was the one unauthenticated, internet-facing
|
over-limit requests get 429 with `Retry-After`. It was the one
|
||||||
endpoint with no limit at all (#64)
|
unauthenticated, internet-facing endpoint with no limit at all (#64)
|
||||||
- 2026-08-11 Enforce the body size limit before CSRF parses the form:
|
- 2026-08-11 Enforce the body size limit before CSRF parses the form:
|
||||||
`MaxBodySize` is now first in all four form-parsing route groups, so an
|
`MaxBodySize` is now first in all four form-parsing route groups, so
|
||||||
oversized request is rejected with 413 instead of being read in full by the
|
an oversized request is rejected with 413 instead of being read in
|
||||||
CSRF middleware before any cap applied (#90)
|
full by the CSRF middleware before any cap applied (#90)
|
||||||
- 2026-08-11 Mask target config on the source detail page, which rendered the
|
- 2026-08-11 Mask target config on the source detail page, which
|
||||||
stored blob verbatim and so exposed the Slack incoming-webhook URL — a bearer
|
rendered the stored blob verbatim and so exposed the Slack
|
||||||
credential that cannot be revoked per-holder. Config reaches the template only
|
incoming-webhook URL — a bearer credential that cannot be revoked
|
||||||
as a `TargetView` of labelled fields, and header values are rendered as a
|
per-holder. Config reaches the template only as a `TargetView` of
|
||||||
count (#113)
|
labelled fields, and header values are rendered as a count (#113)
|
||||||
- 2026-08-11 Allow `retention_days` of 0 to mean retain forever, via a sentinel
|
- 2026-08-11 Allow `retention_days` of 0 to mean retain forever, via a
|
||||||
written in `BeforeSave` so the GORM column default cannot win the race. Also
|
sentinel written in `BeforeSave` so the GORM column default cannot
|
||||||
bounds the reaper's cutoff arithmetic: day counts above 106751 overflowed
|
win the race. Also bounds the reaper's cutoff arithmetic: day counts
|
||||||
`time.Duration` and wrapped the cutoff into the future, where every row
|
above 106751 overflowed `time.Duration` and wrapped the cutoff into
|
||||||
matched and the sweep deleted everything (#79)
|
the future, where every row matched and the sweep deleted everything
|
||||||
|
(#79)
|
||||||
- 2026-08-09 Inactivity-based session timeout: sliding idle expiry
|
- 2026-08-09 Inactivity-based session timeout: sliding idle expiry
|
||||||
(`SESSION_IDLE_TIMEOUT`, default `24h`) refreshed on authenticated requests,
|
(`SESSION_IDLE_TIMEOUT`, default `24h`) refreshed on authenticated
|
||||||
with the 7-day absolute cap kept as an independent backstop that activity
|
requests, with the 7-day absolute cap kept as an independent
|
||||||
never extends (#66)
|
backstop that activity never extends (#66)
|
||||||
- 2026-08-09 Restart recovery and the 60s retry sweep terminally fail an
|
- 2026-08-09 Restart recovery and the 60s retry sweep terminally fail an
|
||||||
orphaned `retrying` delivery whose target type no longer supports retries,
|
orphaned `retrying` delivery whose target type no longer supports
|
||||||
recording a `DeliveryResult` with the reason instead of leaving the delivery
|
retries, recording a `DeliveryResult` with the reason instead of
|
||||||
stuck forever (#82)
|
leaving the delivery stuck forever (#82)
|
||||||
- 2026-08-09 Root the delivery engine's worker pool and the retention reaper's
|
- 2026-08-09 Root the delivery engine's worker pool and the retention
|
||||||
sweep loop at `context.Background()` rather than the fx `OnStart` hook context
|
reaper's sweep loop at `context.Background()` rather than the fx
|
||||||
(#97), which carries fx's 15s start timeout and killed both roughly fifteen
|
`OnStart` hook context (#97), which carries fx's 15s start timeout and
|
||||||
seconds after boot: the proxy silently stopped delivering webhooks entirely,
|
killed both roughly fifteen seconds after boot: the proxy silently
|
||||||
and the reaper never ran a single sweep under its default one-hour interval
|
stopped delivering webhooks entirely, and the reaper never ran a
|
||||||
- 2026-08-09 Archive writer lifecycle (#89): deleting a webhook (or its last
|
single sweep under its default one-hour interval
|
||||||
`database` target) evicts the cached archive writer and closes its handle
|
- 2026-08-09 Archive writer lifecycle (#89): deleting a webhook (or its
|
||||||
while deliberately leaving `archive-{webhookID}.db` on disk, and a new
|
last `database` target) evicts the cached archive writer and closes
|
||||||
`ArchiveSweeper` prunes idle archives on the existing
|
its handle while deliberately leaving `archive-{webhookID}.db` on
|
||||||
|
disk, and a new `ArchiveSweeper` prunes idle archives on the existing
|
||||||
`RETENTION_SWEEP_INTERVAL` without ever creating an archive file
|
`RETENTION_SWEEP_INTERVAL` without ever creating an archive file
|
||||||
- 2026-08-09 Configuration parsing fails loudly on set-but-unparseable
|
- 2026-08-09 Configuration parsing fails loudly on set-but-unparseable
|
||||||
environment values: `envInt` removed in favour of `envPositiveInt` plus a
|
environment values: `envInt` removed in favour of `envPositiveInt`
|
||||||
`PORT` range check, `envBool` now parses with `strconv.ParseBool`, and
|
plus a `PORT` range check, `envBool` now parses with
|
||||||
defaults apply only to unset variables (#80)
|
`strconv.ParseBool`, and defaults apply only to unset variables (#80)
|
||||||
- 2026-08-07 Automatic event retention cleanup based on `retention_days`,
|
- 2026-08-07 Automatic event retention cleanup based on
|
||||||
deleting expired events, deliveries, and delivery results from each
|
`retention_days`, deleting expired events, deliveries, and delivery
|
||||||
per-webhook event database (#63)
|
results from each per-webhook event database (#63)
|
||||||
- 2026-08-07 Update golangci-lint to v2.12.2 (Docker image digest in
|
- 2026-08-07 Update golangci-lint to v2.12.2 (Docker image digest in
|
||||||
`Dockerfile`, release-archive sha256 pins in `script/bootstrap`), adopt the
|
`Dockerfile`, release-archive sha256 pins in `script/bootstrap`),
|
||||||
canonical `.golangci.yml` (v2 `linters.settings` layout so
|
adopt the canonical `.golangci.yml` (v2 `linters.settings` layout so
|
||||||
`lll`/`funlen`/`cyclop`/`dupl` thresholds actually apply), and fix all newly
|
`lll`/`funlen`/`cyclop`/`dupl` thresholds actually apply), and fix
|
||||||
surfaced lint findings
|
all newly surfaced lint findings
|
||||||
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints, Makefile
|
- 2026-07-07 Adopted scripts-to-rule-them-all: `script/` entrypoints,
|
||||||
shims, README Entrypoints section
|
Makefile shims, README Entrypoints section
|
||||||
- 2026-03-25 pin golangci-lint Docker image for linting (#55)
|
- 2026-03-25 pin golangci-lint Docker image for linting (#55)
|
||||||
- 2026-03-18 CSRF middleware detects TLS per-request, fixing login over plain
|
- 2026-03-18 CSRF middleware detects TLS per-request, fixing login over
|
||||||
HTTP and behind reverse proxies (#54)
|
plain HTTP and behind reverse proxies (#54)
|
||||||
- 2026-03-17 root path redirects based on auth state (#52)
|
- 2026-03-17 root path redirects based on auth state (#52)
|
||||||
- 2026-03-17 CSRF protection, SSRF prevention for HTTP delivery targets with DNS
|
- 2026-03-17 CSRF protection, SSRF prevention for HTTP delivery targets
|
||||||
rebinding defense, and per-IP login rate limiting (#42)
|
with DNS rebinding defense, and per-IP login rate limiting (#42)
|
||||||
- 2026-03-17 Slack target type for incoming webhook notifications (#47)
|
- 2026-03-17 Slack target type for incoming webhook notifications (#47)
|
||||||
- 2026-03-17 Dockerfile absolute paths and static linking (#49); absolute dev
|
- 2026-03-17 Dockerfile absolute paths and static linking (#49);
|
||||||
DATA_DIR default and clarified env docs (#46)
|
absolute dev DATA_DIR default and clarified env docs (#46)
|
||||||
- 2026-03-05 security headers middleware, session regeneration on login, request
|
- 2026-03-05 security headers middleware, session regeneration on
|
||||||
body size limits (#41)
|
login, request body size limits (#41)
|
||||||
- 2026-03-04 tests for delivery, middleware, and session packages (#32); removed
|
- 2026-03-04 tests for delivery, middleware, and session packages
|
||||||
the build-architecture global (#31)
|
(#32); removed the build-architecture global (#31)
|
||||||
- 2026-03-04 1.0 MVP merge: Webhook/Entrypoint/Target rename, core delivery
|
- 2026-03-04 1.0 MVP merge: Webhook/Entrypoint/Target rename, core
|
||||||
engine with bounded worker pool and circuit breaker, parallel fan-out,
|
delivery engine with bounded worker pool and circuit breaker,
|
||||||
per-webhook event databases, management UI (#16)
|
parallel fan-out, per-webhook event databases, management UI (#16)
|
||||||
- 2026-03-01 repo brought to REPO_POLICIES standards; TODO.md folded into README
|
- 2026-03-01 repo brought to REPO_POLICIES standards; TODO.md folded
|
||||||
(#6)
|
into README (#6)
|
||||||
|
|
||||||
# Future Steps
|
# Future Steps
|
||||||
|
|
||||||
- Delivery status and retry management UI. Replay of a terminally failed
|
- Delivery status and retry management UI. Replay of a terminally
|
||||||
delivery and per-attempt detail already landed
|
failed delivery and per-attempt detail already landed
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/203,
|
(https://git.eeqj.de/sneak/webhooker/issues/203,
|
||||||
https://git.eeqj.de/sneak/webhooker/issues/202)
|
https://git.eeqj.de/sneak/webhooker/issues/202)
|
||||||
- Per-webhook rate limiting in the receiver handler (per-webhook config plus
|
- Per-webhook rate limiting in the receiver handler (per-webhook config
|
||||||
handler enforcement; global limits must not apply to receiver endpoints)
|
plus handler enforcement; global limits must not apply to receiver
|
||||||
- API key authentication for programmatic access (APIKey model exists; Bearer
|
endpoints)
|
||||||
token middleware does not)
|
- API key authentication for programmatic access (APIKey model exists;
|
||||||
|
Bearer token middleware does not)
|
||||||
- REST API v1
|
- REST API v1
|
||||||
- CRUD for webhooks, entrypoints, targets
|
- CRUD for webhooks, entrypoints, targets
|
||||||
- event viewing and filtering endpoints
|
- event viewing and filtering endpoints
|
||||||
@@ -370,9 +406,9 @@ Clear the rest of the open 1.0.0 milestone
|
|||||||
- OpenAPI specification
|
- OpenAPI specification
|
||||||
- Analytics dashboard: success rates, response times, volume
|
- Analytics dashboard: success rates, response times, volume
|
||||||
- A remember-me option at login
|
- A remember-me option at login
|
||||||
- Password reset flow for a forgotten password over the web. The authenticated
|
- Password reset flow for a forgotten password over the web. The
|
||||||
password _change_ flow already landed, and a lost password is recoverable from
|
authenticated password *change* flow already landed, and a lost
|
||||||
the console with `webhooker resetpw`
|
password is recoverable from the console with `webhooker resetpw`
|
||||||
(https://git.eeqj.de/sneak/webhooker/issues/208)
|
(https://git.eeqj.de/sneak/webhooker/issues/208)
|
||||||
- Later, nice to have
|
- Later, nice to have
|
||||||
- email delivery target type
|
- email delivery target type
|
||||||
|
|||||||
@@ -149,62 +149,6 @@ func TestEventTierQueriesUseTheirIndexes(t *testing.T) {
|
|||||||
Delete(&database.Event{}), "sqlite_autoindex_events_1 (id=?)")
|
Delete(&database.Event{}), "sqlite_autoindex_events_1 (id=?)")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestEventLogFiltersUseTheStatusIndex does the same for the event log's
|
|
||||||
// Failed and Pending lists, of the newest events with a delivery in
|
|
||||||
// given statuses, and for their counts (eventsWithStatus and
|
|
||||||
// countEventsWithStatus in the handlers). The lists must also reach
|
|
||||||
// the events table only by ID: from the matching deliveries, then from
|
|
||||||
// the newest of those events.
|
|
||||||
func TestEventLogFiltersUseTheStatusIndex(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
mgr, lc := setupTestWebhookDBManager(t)
|
|
||||||
ctx := context.Background()
|
|
||||||
require.NoError(t, lc.Start(ctx))
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
|
||||||
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
|
|
||||||
db, err := mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
dry := db.Session(&gorm.Session{DryRun: true})
|
|
||||||
byStatus := "idx_deliveries_status (status=? AND deleted_at=?)"
|
|
||||||
pending := []database.DeliveryStatus{
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
database.DeliveryStatusRetrying,
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
rows []struct{ ID string }
|
|
||||||
count int64
|
|
||||||
)
|
|
||||||
|
|
||||||
matching := dry.Model(&database.Delivery{}).
|
|
||||||
Distinct("event_id").Where("status IN ?", pending)
|
|
||||||
newest := dry.Table("(?) AS matching", matching).
|
|
||||||
Joins("CROSS JOIN events ON events.id = matching.event_id").
|
|
||||||
Where(
|
|
||||||
"events.webhook_id = ? AND events.deleted_at IS NULL",
|
|
||||||
webhookID,
|
|
||||||
).
|
|
||||||
Order("events.created_at DESC").Limit(50).
|
|
||||||
Select("events.id AS event_id")
|
|
||||||
|
|
||||||
// Each step of the plan is printed in braces, so these name the
|
|
||||||
// lookup that follows each scan.
|
|
||||||
byID := "{SEARCH events USING INDEX sqlite_autoindex_events_1 (id=?)}"
|
|
||||||
|
|
||||||
assertPlanUses(t, db, dry.Table("(?) AS newest", newest).
|
|
||||||
Joins("CROSS JOIN events ON events.id = newest.event_id").
|
|
||||||
Select("id").Order("created_at DESC").Limit(50).Find(&rows),
|
|
||||||
byStatus, "{SCAN matching} "+byID, "{SCAN newest} "+byID)
|
|
||||||
assertPlanUses(t, db, dry.Model(&database.Delivery{}).
|
|
||||||
Distinct("event_id").Where("status IN ?", pending).Count(&count),
|
|
||||||
byStatus)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStatisticsQueriesUseTheirIndexes does the same for the webhook
|
// TestStatisticsQueriesUseTheirIndexes does the same for the webhook
|
||||||
// page's statistics (readEventStats in the handlers): deliveries in
|
// page's statistics (readEventStats in the handlers): deliveries in
|
||||||
// progress, each target's deliveries finished since a time, which must
|
// progress, each target's deliveries finished since a time, which must
|
||||||
|
|||||||
@@ -163,17 +163,6 @@ func (env *archiveEnv) seedArchiveRows(
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
path := env.archivePath(tgt)
|
path := env.archivePath(tgt)
|
||||||
seedArchiveFile(t, path, tgt.WebhookID, archivedAt...)
|
|
||||||
|
|
||||||
return path
|
|
||||||
}
|
|
||||||
|
|
||||||
// seedArchiveFile creates the archive file at path and inserts one row
|
|
||||||
// per supplied archived-at timestamp, as seedArchiveRows does.
|
|
||||||
func seedArchiveFile(
|
|
||||||
t *testing.T, path, webhookID string, archivedAt ...time.Time,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
sqlDB, err := sql.Open(
|
sqlDB, err := sql.Open(
|
||||||
"sqlite", fmt.Sprintf("file:%s?mode=rwc", path),
|
"sqlite", fmt.Sprintf("file:%s?mode=rwc", path),
|
||||||
@@ -193,7 +182,7 @@ func seedArchiveFile(
|
|||||||
for i, at := range archivedAt {
|
for i, at := range archivedAt {
|
||||||
row := delivery.ExportArchivedEvent{
|
row := delivery.ExportArchivedEvent{
|
||||||
EventID: fmt.Sprintf("ev-%d", i),
|
EventID: fmt.Sprintf("ev-%d", i),
|
||||||
WebhookID: webhookID,
|
WebhookID: tgt.WebhookID,
|
||||||
Method: http.MethodPost,
|
Method: http.MethodPost,
|
||||||
Body: `{"seeded":true}`,
|
Body: `{"seeded":true}`,
|
||||||
ArchivedAt: at,
|
ArchivedAt: at,
|
||||||
@@ -202,6 +191,8 @@ func seedArchiveFile(
|
|||||||
}
|
}
|
||||||
|
|
||||||
require.NoError(t, sqlDB.Close())
|
require.NoError(t, sqlDB.Close())
|
||||||
|
|
||||||
|
return path
|
||||||
}
|
}
|
||||||
|
|
||||||
// archivedEventIDs returns the event ids currently stored in an
|
// archivedEventIDs returns the event ids currently stored in an
|
||||||
|
|||||||
@@ -295,13 +295,12 @@ func (e *Engine) EvictTarget(targetID string) {
|
|||||||
e.dbTarget.evict(targetID)
|
e.dbTarget.evict(targetID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Rename implements Archives. It renames every one of a database
|
// Rename implements Archives. It renames a database target's
|
||||||
// target's archive files to ArchiveFileName(webhookName, targetName,
|
// archive file to ArchiveFileName(webhookName, targetName,
|
||||||
// targetID), each keeping the period in its name, under the lock the
|
// targetID), under the lock the target's archive writes and the
|
||||||
// target's archive writes and the idle sweep take. It never replaces
|
// idle sweep take. It never replaces a file: if one already has the
|
||||||
// a file: if one already has a new name, the error is
|
// new name, the error is ErrArchiveNameTaken. The caller renames
|
||||||
// ErrArchiveNameTaken. The caller renames before it saves the new
|
// before it saves the new name: see databaseTarget.rename.
|
||||||
// name: see databaseTarget.rename.
|
|
||||||
func (e *Engine) Rename(
|
func (e *Engine) Rename(
|
||||||
targetID, webhookName, targetName string,
|
targetID, webhookName, targetName string,
|
||||||
) error {
|
) error {
|
||||||
|
|||||||
@@ -501,32 +501,23 @@ func NewExportArchiveWriter(
|
|||||||
return &ExportArchiveWriter{w: w}
|
return &ExportArchiveWriter{w: w}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write archives a row through the writer, into the file named
|
// Write archives a row through the writer.
|
||||||
// without a period.
|
|
||||||
func (e *ExportArchiveWriter) Write(
|
func (e *ExportArchiveWriter) Write(
|
||||||
row ExportArchivedEvent, expiry time.Duration,
|
row ExportArchivedEvent, expiry time.Duration,
|
||||||
) error {
|
) error {
|
||||||
return e.w.write(row, expiry, "")
|
return e.w.write(row, expiry)
|
||||||
}
|
|
||||||
|
|
||||||
// WritePeriod archives a row through the writer, into the file for
|
|
||||||
// period.
|
|
||||||
func (e *ExportArchiveWriter) WritePeriod(
|
|
||||||
row ExportArchivedEvent, expiry time.Duration, period string,
|
|
||||||
) error {
|
|
||||||
return e.w.write(row, expiry, period)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Open opens the archive file, pruning when expiry is positive.
|
// Open opens the archive file, pruning when expiry is positive.
|
||||||
func (e *ExportArchiveWriter) Open(expiry time.Duration) error {
|
func (e *ExportArchiveWriter) Open(expiry time.Duration) error {
|
||||||
return e.w.open(e.w.path, expiry)
|
return e.w.open(expiry)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Reopen closes and reopens the archive file.
|
// Reopen closes and reopens the archive file.
|
||||||
func (e *ExportArchiveWriter) Reopen(
|
func (e *ExportArchiveWriter) Reopen(
|
||||||
expiry time.Duration,
|
expiry time.Duration,
|
||||||
) error {
|
) error {
|
||||||
return e.w.reopen(e.w.path, expiry)
|
return e.w.reopen(expiry)
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetNow replaces the clock the writer measures its reopen
|
// SetNow replaces the clock the writer measures its reopen
|
||||||
@@ -556,7 +547,7 @@ func (e *ExportArchiveWriter) Path() string {
|
|||||||
func (e *ExportArchiveWriter) OpenExisting(
|
func (e *ExportArchiveWriter) OpenExisting(
|
||||||
expiry time.Duration,
|
expiry time.Duration,
|
||||||
) error {
|
) error {
|
||||||
return e.w.openMode(e.w.path, archiveModeExisting, expiry)
|
return e.w.openMode(archiveModeExisting, expiry)
|
||||||
}
|
}
|
||||||
|
|
||||||
// SweepExpired runs an idle sweep of the archive.
|
// SweepExpired runs an idle sweep of the archive.
|
||||||
|
|||||||
@@ -41,8 +41,6 @@ type TargetConfigForm struct {
|
|||||||
Timeout string
|
Timeout string
|
||||||
// Expiry is the database (archive) target's row expiry.
|
// Expiry is the database (archive) target's row expiry.
|
||||||
Expiry string
|
Expiry string
|
||||||
// Rotation is the database (archive) target's rotation.
|
|
||||||
Rotation string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewTargetConfigForm parses a target's stored configuration into
|
// NewTargetConfigForm parses a target's stored configuration into
|
||||||
@@ -87,13 +85,11 @@ func NewTargetConfigForm(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// databaseConfigForm parses an archive target's optional expiry and
|
// databaseConfigForm parses an archive target's optional expiry.
|
||||||
// rotation. An absent, empty or never expiry yields an empty expiry,
|
// An absent, empty or never expiry yields an empty expiry, on which
|
||||||
// on which the edit form starts at never; saving it unchanged stores
|
// the edit form starts at never; saving it unchanged stores never,
|
||||||
// never, which means the same as an empty expiry. An absent rotation
|
// which means the same as an empty expiry. An expiry that is set
|
||||||
// is empty too, and the form starts at none. An expiry that is set
|
// but not a valid duration is an error, not a blank field.
|
||||||
// but not a valid duration, or a rotation that is not one of the
|
|
||||||
// four, is an error, not a blank field.
|
|
||||||
func databaseConfigForm(
|
func databaseConfigForm(
|
||||||
configJSON string,
|
configJSON string,
|
||||||
) (TargetConfigForm, error) {
|
) (TargetConfigForm, error) {
|
||||||
@@ -110,15 +106,8 @@ func databaseConfigForm(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = ValidateArchiveRotation(cfg.Rotation)
|
|
||||||
if err != nil {
|
|
||||||
return TargetConfigForm{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
form := TargetConfigForm{Rotation: cfg.Rotation}
|
|
||||||
|
|
||||||
if cfg.Expiry == "" || cfg.Expiry == archiveExpiryNever {
|
if cfg.Expiry == "" || cfg.Expiry == archiveExpiryNever {
|
||||||
return form, nil
|
return TargetConfigForm{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
err = ValidateArchiveExpiry(cfg.Expiry)
|
err = ValidateArchiveExpiry(cfg.Expiry)
|
||||||
@@ -126,7 +115,5 @@ func databaseConfigForm(
|
|||||||
return TargetConfigForm{}, err
|
return TargetConfigForm{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
form.Expiry = cfg.Expiry
|
return TargetConfigForm{Expiry: cfg.Expiry}, nil
|
||||||
|
|
||||||
return form, nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -177,25 +177,23 @@ func httpConfigFields(t *database.Target) []ConfigField {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// maxRetriesField describes a target's retry count, which lives
|
// maxRetriesField describes a target's retry count, which lives
|
||||||
// on the target row rather than in its configuration blob. A
|
// on the target row rather than in its configuration blob.
|
||||||
// stored 0 makes a single attempt, so it is shown as 1.
|
|
||||||
func maxRetriesField(t *database.Target) ConfigField {
|
func maxRetriesField(t *database.Target) ConfigField {
|
||||||
attempts := strconv.Itoa(t.MaxRetries)
|
retries := strconv.Itoa(t.MaxRetries)
|
||||||
if t.MaxRetries == 0 {
|
if t.MaxRetries == 0 {
|
||||||
attempts = "1 (fire-and-forget: no retries, no circuit breaker)"
|
retries += " (fire-and-forget)"
|
||||||
}
|
}
|
||||||
|
|
||||||
return ConfigField{
|
return ConfigField{
|
||||||
Label: "Delivery attempts",
|
Label: "Max Retries",
|
||||||
Value: attempts,
|
Value: retries,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// databaseConfigFields describes an archive target by its
|
// databaseConfigFields describes an archive target by its
|
||||||
// expiry in plain units, such as "30 days", or "never" when
|
// expiry in plain units, such as "30 days", or "never" when
|
||||||
// the archive is kept forever, and by its rotation. An expiry
|
// the archive is kept forever. An expiry that is set but not
|
||||||
// that is set but not a valid duration, or a rotation that is
|
// a valid duration is reported as unavailable rather than
|
||||||
// not one of the four, is reported as unavailable rather than
|
|
||||||
// echoed back.
|
// echoed back.
|
||||||
func databaseConfigFields(configJSON string) []ConfigField {
|
func databaseConfigFields(configJSON string) []ConfigField {
|
||||||
expiry, err := parseArchiveExpiry(configJSON)
|
expiry, err := parseArchiveExpiry(configJSON)
|
||||||
@@ -203,22 +201,14 @@ func databaseConfigFields(configJSON string) []ConfigField {
|
|||||||
return unavailableConfigFields()
|
return unavailableConfigFields()
|
||||||
}
|
}
|
||||||
|
|
||||||
rotation, err := parseArchiveRotation(configJSON)
|
|
||||||
if err != nil {
|
|
||||||
return unavailableConfigFields()
|
|
||||||
}
|
|
||||||
|
|
||||||
value := archiveExpiryNever
|
value := archiveExpiryNever
|
||||||
if expiry > 0 {
|
if expiry > 0 {
|
||||||
value = plainDuration(expiry)
|
value = plainDuration(expiry)
|
||||||
}
|
}
|
||||||
|
|
||||||
return []ConfigField{{
|
return []ConfigField{{
|
||||||
Label: "Archive expiry",
|
Label: "Archive Expiry",
|
||||||
Value: value,
|
Value: value,
|
||||||
}, {
|
|
||||||
Label: "Archive rotation",
|
|
||||||
Value: rotation,
|
|
||||||
}}
|
}}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ const (
|
|||||||
viewMaskedOrigin = viewExampleOrigin + "/..."
|
viewMaskedOrigin = viewExampleOrigin + "/..."
|
||||||
viewUnavailable = "(unavailable)"
|
viewUnavailable = "(unavailable)"
|
||||||
viewExpiryNever = "never"
|
viewExpiryNever = "never"
|
||||||
viewMaxRetries = "Delivery attempts"
|
viewMaxRetries = "Max Retries"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMaskedWebhookURL(t *testing.T) {
|
func TestMaskedWebhookURL(t *testing.T) {
|
||||||
@@ -190,7 +190,7 @@ func TestNewTargetViews_Slack(t *testing.T) {
|
|||||||
t,
|
t,
|
||||||
map[string]string{
|
map[string]string{
|
||||||
"Webhook URL": slackMaskedURL,
|
"Webhook URL": slackMaskedURL,
|
||||||
viewMaxRetries: "1 (fire-and-forget: no retries, no circuit breaker)",
|
viewMaxRetries: "0 (fire-and-forget)",
|
||||||
},
|
},
|
||||||
fieldMap(view.Config),
|
fieldMap(view.Config),
|
||||||
)
|
)
|
||||||
@@ -258,7 +258,7 @@ func TestNewTargetViews_HTTPFireAndForget(t *testing.T) {
|
|||||||
t,
|
t,
|
||||||
map[string]string{
|
map[string]string{
|
||||||
"Destination URL": viewMaskedOrigin,
|
"Destination URL": viewMaskedOrigin,
|
||||||
viewMaxRetries: "1 (fire-and-forget: no retries, no circuit breaker)",
|
viewMaxRetries: "0 (fire-and-forget)",
|
||||||
},
|
},
|
||||||
fieldMap(view.Config),
|
fieldMap(view.Config),
|
||||||
)
|
)
|
||||||
@@ -328,49 +328,13 @@ func TestNewTargetViews_Database(t *testing.T) {
|
|||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
map[string]string{
|
map[string]string{"Archive Expiry": tc.want},
|
||||||
"Archive expiry": tc.want,
|
|
||||||
"Archive rotation": rotationNone,
|
|
||||||
},
|
|
||||||
fieldMap(view.Config),
|
fieldMap(view.Config),
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestNewTargetViews_DatabaseRotation proves the target list shows a
|
|
||||||
// database target's rotation, none when it has none stored.
|
|
||||||
func TestNewTargetViews_DatabaseRotation(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Each stored config, and the rotation the list shows for it.
|
|
||||||
tests := map[string]string{
|
|
||||||
"": rotationNone,
|
|
||||||
`{"rotation":""}`: rotationNone,
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, rotation := range []string{
|
|
||||||
rotationNone, rotationMonthly, rotationDaily, rotationHourly,
|
|
||||||
} {
|
|
||||||
tests[`{"rotation":"`+rotation+`"}`] = rotation
|
|
||||||
}
|
|
||||||
|
|
||||||
for config, want := range tests {
|
|
||||||
t.Run(config, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
view := viewFor(t, database.Target{
|
|
||||||
Type: database.TargetTypeDatabase,
|
|
||||||
Config: config,
|
|
||||||
})
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t, want, fieldMap(view.Config)["Archive rotation"],
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNewTargetViews_Log(t *testing.T) {
|
func TestNewTargetViews_Log(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -418,10 +382,6 @@ func TestNewTargetViews_Unpresentable(t *testing.T) {
|
|||||||
Type: database.TargetTypeDatabase,
|
Type: database.TargetTypeDatabase,
|
||||||
Config: `{"expiry":"a fortnight"}`,
|
Config: `{"expiry":"a fortnight"}`,
|
||||||
},
|
},
|
||||||
"invalid archive rotation": {
|
|
||||||
Type: database.TargetTypeDatabase,
|
|
||||||
Config: weeklyConfig,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for name, target := range tests {
|
for name, target := range tests {
|
||||||
|
|||||||
@@ -20,8 +20,7 @@ const archiveNameMaxLen = 40
|
|||||||
// from the per-webhook event database. The event is already
|
// from the per-webhook event database. The event is already
|
||||||
// persisted in the per-webhook event DB by the time delivery runs;
|
// persisted in the per-webhook event DB by the time delivery runs;
|
||||||
// the database target additionally writes a durable long-term copy
|
// the database target additionally writes a durable long-term copy
|
||||||
// into the file ArchiveFileName names, with a period added when the
|
// into the file ArchiveFileName names and then records a single
|
||||||
// target rotates (see archivePeriodPath), and then records a single
|
|
||||||
// attempt whose outcome reflects whether the archive write
|
// attempt whose outcome reflects whether the archive write
|
||||||
// succeeded. See archiveWriter for the close/reopen, auto-recreate,
|
// succeeded. See archiveWriter for the close/reopen, auto-recreate,
|
||||||
// and expiry semantics.
|
// and expiry semantics.
|
||||||
@@ -147,9 +146,8 @@ func (t *databaseTarget) Deliver(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// archive writes the full event as a row into the target's
|
// archive writes the full event as a row into the target's
|
||||||
// archive database, honouring the optional per-target expiry and
|
// archive database, honouring the optional per-target expiry
|
||||||
// rotation parsed from the target config JSON. With rotation, the
|
// parsed from the target config JSON.
|
||||||
// event goes to the file for the period of its receive time.
|
|
||||||
func (t *databaseTarget) archive(d *database.Delivery) error {
|
func (t *databaseTarget) archive(d *database.Delivery) error {
|
||||||
webhookID := d.Event.WebhookID
|
webhookID := d.Event.WebhookID
|
||||||
if webhookID == "" {
|
if webhookID == "" {
|
||||||
@@ -161,19 +159,6 @@ func (t *databaseTarget) archive(d *database.Delivery) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
rotation, err := parseArchiveRotation(d.Target.Config)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// An event whose stored row was gone before its delivery ran has
|
|
||||||
// no receive time (see Engine.hydrateEvent), and goes to the
|
|
||||||
// file for now.
|
|
||||||
receivedAt := d.Event.CreatedAt
|
|
||||||
if receivedAt.IsZero() {
|
|
||||||
receivedAt = time.Now()
|
|
||||||
}
|
|
||||||
|
|
||||||
w, err := t.writerFor(d.TargetID)
|
w, err := t.writerFor(d.TargetID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -189,7 +174,7 @@ func (t *databaseTarget) archive(d *database.Delivery) error {
|
|||||||
ContentType: d.Event.ContentType,
|
ContentType: d.Event.ContentType,
|
||||||
}
|
}
|
||||||
|
|
||||||
return w.write(row, expiry, archivePeriod(rotation, receivedAt))
|
return w.write(row, expiry)
|
||||||
}
|
}
|
||||||
|
|
||||||
// writerFor returns the archive writer for a database target,
|
// writerFor returns the archive writer for a database target,
|
||||||
@@ -290,10 +275,10 @@ func (t *databaseTarget) releaseSweepWriter(
|
|||||||
delete(t.writers, targetID)
|
delete(t.writers, targetID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// newWriter builds the writer for a database target's archive. Its
|
// newWriter builds the writer for a database target's archive. The
|
||||||
// path is the one ArchivePath gives for the webhook and the target as
|
// file is the one ArchivePath gives for the webhook and the target as
|
||||||
// the main database names them now; from then on only rename changes
|
// the main database names them now; from then on only rename changes
|
||||||
// the name the writer uses. It does not touch the archive files.
|
// the name the writer uses. It does not touch the archive file.
|
||||||
func (t *databaseTarget) newWriter(
|
func (t *databaseTarget) newWriter(
|
||||||
targetID string,
|
targetID string,
|
||||||
) (*archiveWriter, error) {
|
) (*archiveWriter, error) {
|
||||||
@@ -321,10 +306,10 @@ func (t *databaseTarget) newWriter(
|
|||||||
return w, nil
|
return w, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// rename moves every one of a database target's archive files to the
|
// rename moves a database target's archive file to the name for
|
||||||
// name for webhookName and targetName. It goes through the target's
|
// webhookName and targetName. It goes through the target's writer,
|
||||||
// writer, so the move holds the lock that writes and the idle sweep
|
// so the move holds the lock that writes and the idle sweep take,
|
||||||
// take, and later writes use the new name.
|
// and later writes use the new name.
|
||||||
//
|
//
|
||||||
// The writer is created if there is none, and it stays cached. The
|
// The writer is created if there is none, and it stays cached. The
|
||||||
// handlers rename before they save the new name, so until the save
|
// handlers rename before they save the new name, so until the save
|
||||||
|
|||||||
@@ -85,10 +85,6 @@ type databaseTargetConfig struct {
|
|||||||
// archived rows are pruned, or "never" (the default) to
|
// archived rows are pruned, or "never" (the default) to
|
||||||
// keep them forever.
|
// keep them forever.
|
||||||
Expiry string `json:"expiry"`
|
Expiry string `json:"expiry"`
|
||||||
|
|
||||||
// Rotation is none (the default), monthly, daily or hourly: see
|
|
||||||
// archivePeriod.
|
|
||||||
Rotation string `json:"rotation"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// archivedEvent is one fully captured webhook event stored in a
|
// archivedEvent is one fully captured webhook event stored in a
|
||||||
@@ -182,7 +178,7 @@ func ValidateArchiveExpiry(expiry string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// archiveWriter owns one database target's archive SQLite files.
|
// archiveWriter owns one database target's archive SQLite file.
|
||||||
// It serialises writes, and after each write closes and reopens
|
// It serialises writes, and after each write closes and reopens
|
||||||
// the file (debounced to at most once per debounce window) so
|
// the file (debounced to at most once per debounce window) so
|
||||||
// an operator can move the file away for offline archiving. The
|
// an operator can move the file away for offline archiving. The
|
||||||
@@ -190,16 +186,8 @@ func ValidateArchiveExpiry(expiry string) error {
|
|||||||
// file is opened create-if-missing and its schema is migrated
|
// file is opened create-if-missing and its schema is migrated
|
||||||
// on every open.
|
// on every open.
|
||||||
type archiveWriter struct {
|
type archiveWriter struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
|
path string
|
||||||
// path is the target's archive file as ArchivePath names it. A
|
|
||||||
// target that rotates writes to the files archivePeriodPath names
|
|
||||||
// for path and a period instead.
|
|
||||||
path string
|
|
||||||
|
|
||||||
// current is the file db is open on.
|
|
||||||
current string
|
|
||||||
|
|
||||||
log *slog.Logger
|
log *slog.Logger
|
||||||
debounce time.Duration
|
debounce time.Duration
|
||||||
db *gorm.DB
|
db *gorm.DB
|
||||||
@@ -248,14 +236,12 @@ func newArchiveWriter(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// write appends the event as a row to the archive file for period
|
// write appends the event as a row, then applies the debounced
|
||||||
// (see archivePeriodPath), then applies the debounced close/reopen.
|
// close/reopen. It recreates the archive file if it was moved
|
||||||
// When period names a different file from the one open, the open one
|
// or removed since the last open. A positive expiry prunes rows
|
||||||
// is closed first. It recreates the archive file if it was moved or
|
// older than it on each (re)open.
|
||||||
// removed since the last open. A positive expiry prunes rows older
|
|
||||||
// than it on each (re)open.
|
|
||||||
func (w *archiveWriter) write(
|
func (w *archiveWriter) write(
|
||||||
row archivedEvent, expiry time.Duration, period string,
|
row archivedEvent, expiry time.Duration,
|
||||||
) error {
|
) error {
|
||||||
w.mu.Lock()
|
w.mu.Lock()
|
||||||
defer w.mu.Unlock()
|
defer w.mu.Unlock()
|
||||||
@@ -266,10 +252,8 @@ func (w *archiveWriter) write(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
file := archivePeriodPath(w.path, period)
|
if w.db == nil || !fileExists(w.path) {
|
||||||
|
err := w.reopen(expiry)
|
||||||
if w.db == nil || w.current != file || !fileExists(file) {
|
|
||||||
err := w.reopen(file, expiry)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -280,41 +264,41 @@ func (w *archiveWriter) write(
|
|||||||
err := w.db.Create(&row).Error
|
err := w.db.Create(&row).Error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"archiving event to %s: %w", file, err,
|
"archiving event to %s: %w", w.path, err,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if w.now().Sub(w.lastReopen) >= w.debounce {
|
if w.now().Sub(w.lastReopen) >= w.debounce {
|
||||||
return w.reopen(file, expiry)
|
return w.reopen(expiry)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// open opens (creating if missing) an archive file, migrates
|
// open opens (creating if missing) the archive file, migrates
|
||||||
// its schema, records the reopen time, and prunes expired rows
|
// its schema, records the reopen time, and prunes expired rows
|
||||||
// when expiry is positive.
|
// when expiry is positive.
|
||||||
func (w *archiveWriter) open(file string, expiry time.Duration) error {
|
func (w *archiveWriter) open(expiry time.Duration) error {
|
||||||
return w.openMode(file, archiveModeCreate, expiry)
|
return w.openMode(archiveModeCreate, expiry)
|
||||||
}
|
}
|
||||||
|
|
||||||
// openMode opens an archive file with the given SQLite URI
|
// openMode opens the archive file with the given SQLite URI
|
||||||
// mode, migrates its schema, records the reopen time, and
|
// mode, migrates its schema, records the reopen time, and
|
||||||
// prunes expired rows when expiry is positive. The write path
|
// prunes expired rows when expiry is positive. The write path
|
||||||
// passes archiveModeCreate so a missing file is recreated; the
|
// passes archiveModeCreate so a missing file is recreated; the
|
||||||
// idle sweep passes archiveModeExisting so a missing file is an
|
// idle sweep passes archiveModeExisting so a missing file is an
|
||||||
// error rather than a newly conjured empty archive.
|
// error rather than a newly conjured empty archive.
|
||||||
func (w *archiveWriter) openMode(
|
func (w *archiveWriter) openMode(
|
||||||
file, mode string, expiry time.Duration,
|
mode string, expiry time.Duration,
|
||||||
) error {
|
) error {
|
||||||
// Opened through database.OpenSQLite so an archive file carries
|
// Opened through database.OpenSQLite so an archive file carries
|
||||||
// the same WAL journaling, busy timeout, immediate-transaction
|
// the same WAL journaling, busy timeout, immediate-transaction
|
||||||
// locking, and pool bounds as every other database file. See
|
// locking, and pool bounds as every other database file. See
|
||||||
// internal/database/sqlite_open.go.
|
// internal/database/sqlite_open.go.
|
||||||
sqlDB, err := database.OpenSQLite(file, mode)
|
sqlDB, err := database.OpenSQLite(w.path, mode)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"opening archive database %s: %w", file, err,
|
"opening archive database %s: %w", w.path, err,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -330,7 +314,7 @@ func (w *archiveWriter) openMode(
|
|||||||
|
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"connecting to archive database %s: %w",
|
"connecting to archive database %s: %w",
|
||||||
file, err,
|
w.path, err,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -339,12 +323,11 @@ func (w *archiveWriter) openMode(
|
|||||||
_ = sqlDB.Close()
|
_ = sqlDB.Close()
|
||||||
|
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"migrating archive database %s: %w", file, err,
|
"migrating archive database %s: %w", w.path, err,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
w.db = gdb
|
w.db = gdb
|
||||||
w.current = file
|
|
||||||
w.lastReopen = w.now()
|
w.lastReopen = w.now()
|
||||||
w.reopens++
|
w.reopens++
|
||||||
|
|
||||||
@@ -355,12 +338,12 @@ func (w *archiveWriter) openMode(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// reopen closes any open handle and opens file afresh. The
|
// reopen closes any open handle and opens the file afresh. The
|
||||||
// fresh open recreates the file if it was moved away.
|
// fresh open recreates the file if it was moved away.
|
||||||
func (w *archiveWriter) reopen(file string, expiry time.Duration) error {
|
func (w *archiveWriter) reopen(expiry time.Duration) error {
|
||||||
w.close()
|
w.close()
|
||||||
|
|
||||||
return w.open(file, expiry)
|
return w.open(expiry)
|
||||||
}
|
}
|
||||||
|
|
||||||
// close closes the underlying handle, if any.
|
// close closes the underlying handle, if any.
|
||||||
@@ -377,56 +360,22 @@ func (w *archiveWriter) close() {
|
|||||||
w.db = nil
|
w.db = nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// sweepExpired prunes the target's archive files, which may have
|
// sweepExpired prunes an archive that may have gone idle, with
|
||||||
// gone idle, with no write to trigger the usual on-reopen prune. It
|
// no write to trigger the usual on-reopen prune. It takes the
|
||||||
// lists the files under the writer's own mutex, then takes the mutex
|
// writer's own mutex for the whole operation, so a sweep is
|
||||||
// again for one file at a time, so a write waits for at most one
|
// ordered against concurrent writes rather than reaching around
|
||||||
// file's prune, and each prune is ordered against concurrent writes
|
// them to the file.
|
||||||
// rather than reaching around them to the file.
|
|
||||||
//
|
//
|
||||||
// It never creates an archive file: it prunes only the files
|
// It never creates the archive file: a missing file is skipped,
|
||||||
// archiveFiles lists, skips one that is gone by the time it is
|
// and the reopen uses archiveModeExisting so SQLite itself
|
||||||
// reached (moved away, or renamed since the listing), and opens each
|
// refuses to create one if the file disappears between the
|
||||||
// with archiveModeExisting so SQLite itself refuses to create one if
|
// check and the open.
|
||||||
// the file disappears between the check and the open. A file named
|
|
||||||
// for a period that the prune leaves empty is deleted.
|
|
||||||
//
|
//
|
||||||
// The archive is left CLOSED afterwards. An idle archive holding
|
// The archive is left CLOSED afterwards. An idle archive holding
|
||||||
// no handle is what keeps the operator's move-the-file-away
|
// no handle is what keeps the operator's move-the-file-away
|
||||||
// workflow working; the next write reopens (and recreates) the
|
// workflow working; the next write reopens (and recreates) the
|
||||||
// file as it always has.
|
// file as it always has.
|
||||||
func (w *archiveWriter) sweepExpired(expiry time.Duration) error {
|
func (w *archiveWriter) sweepExpired(expiry time.Duration) error {
|
||||||
w.mu.Lock()
|
|
||||||
files, err := archiveFiles(w.path)
|
|
||||||
w.mu.Unlock()
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
var errs []error
|
|
||||||
|
|
||||||
for _, file := range files {
|
|
||||||
err = w.sweepFile(file, expiry)
|
|
||||||
if errors.Is(err, errArchiveWriterEvicted) {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
errs = append(errs, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return errors.Join(errs...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// sweepFile prunes one of the target's archive files for sweepExpired,
|
|
||||||
// holding w.mu while it does. It skips a file that is gone, and deletes
|
|
||||||
// the file, with its -wal and -shm, when it is named for a period and
|
|
||||||
// the prune leaves it empty.
|
|
||||||
func (w *archiveWriter) sweepFile(
|
|
||||||
file archiveFile, expiry time.Duration,
|
|
||||||
) error {
|
|
||||||
w.mu.Lock()
|
w.mu.Lock()
|
||||||
defer w.mu.Unlock()
|
defer w.mu.Unlock()
|
||||||
|
|
||||||
@@ -436,7 +385,7 @@ func (w *archiveWriter) sweepFile(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if !fileExists(file.path) {
|
if !fileExists(w.path) {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -444,56 +393,26 @@ func (w *archiveWriter) sweepFile(
|
|||||||
// freshly opened file, matching the write path's semantics.
|
// freshly opened file, matching the write path's semantics.
|
||||||
w.close()
|
w.close()
|
||||||
|
|
||||||
err := w.openMode(file.path, archiveModeExisting, expiry)
|
err := w.openMode(archiveModeExisting, expiry)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if file.period == "" {
|
|
||||||
w.close()
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var rows int64
|
|
||||||
|
|
||||||
err = w.db.Model(&archivedEvent{}).Count(&rows).Error
|
|
||||||
|
|
||||||
w.close()
|
w.close()
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"counting rows in archive %s: %w", file.path, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if rows > 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, suffix := range []string{"", "-wal", "-shm"} {
|
|
||||||
err = os.Remove(file.path + suffix)
|
|
||||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
|
||||||
return fmt.Errorf("deleting empty archive file: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
w.log.Info("deleted empty archive file", "path", file.path)
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// rename gives every one of the target's archive files the new
|
// rename gives the archive file a new name in the same directory,
|
||||||
// name, keeping the period in the name of each (see
|
// and the writer uses the file under that name from now on. The
|
||||||
// archivePeriodPath), and the writer uses the files under that name
|
// handle is closed first, which folds the -wal into the .db; any
|
||||||
// from now on. The handle is closed first, which folds the -wal into
|
// -wal or -shm still beside the file (left by a crash) is moved with
|
||||||
// the .db; any -wal or -shm still beside a file (left by a crash) is
|
// it, because SQLite finds them by name. A missing file is not an
|
||||||
// moved with it, because SQLite finds them by name. A target with no
|
// error: the operator may have moved it away, and the next write
|
||||||
// files is not an error: the operator may have moved them away, and
|
// creates it under the new name.
|
||||||
// the next write creates its file under the new name.
|
|
||||||
//
|
//
|
||||||
// If a file already has one of the new names, nothing is moved and
|
// If a file already has the new name, nothing is moved and the
|
||||||
// the error is ErrArchiveNameTaken. If one file fails to move, those
|
// error is ErrArchiveNameTaken. If one file fails to move, those
|
||||||
// already moved are moved back before the error is returned, so the
|
// already moved are moved back before the error is returned, so the
|
||||||
// archive is never split across two names.
|
// archive is never split across two names.
|
||||||
func (w *archiveWriter) rename(name string) error {
|
func (w *archiveWriter) rename(name string) error {
|
||||||
@@ -511,53 +430,38 @@ func (w *archiveWriter) rename(name string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
files, err := archiveFiles(w.path)
|
suffixes := []string{"", "-wal", "-shm"}
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// from[i] moves to to[i].
|
for _, suffix := range suffixes {
|
||||||
var from, to []string
|
if fileExists(path + suffix) {
|
||||||
|
|
||||||
for _, file := range files {
|
|
||||||
renamed := archivePeriodPath(path, file.period)
|
|
||||||
|
|
||||||
for _, suffix := range []string{"", "-wal", "-shm"} {
|
|
||||||
from = append(from, file.path+suffix)
|
|
||||||
to = append(to, renamed+suffix)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, taken := range to {
|
|
||||||
if fileExists(taken) {
|
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"%w: %s", ErrArchiveNameTaken, filepath.Base(taken),
|
"%w: %s", ErrArchiveNameTaken, name+suffix,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
w.close()
|
w.close()
|
||||||
|
|
||||||
for i := range from {
|
for i, suffix := range suffixes {
|
||||||
err = os.Rename(from[i], to[i])
|
err := os.Rename(w.path+suffix, path+suffix)
|
||||||
if err == nil || errors.Is(err, fs.ErrNotExist) {
|
if err == nil || errors.Is(err, fs.ErrNotExist) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
for j := range i {
|
for _, moved := range suffixes[:i] {
|
||||||
backErr := os.Rename(to[j], from[j])
|
backErr := os.Rename(path+moved, w.path+moved)
|
||||||
if backErr != nil && !errors.Is(backErr, fs.ErrNotExist) {
|
if backErr != nil && !errors.Is(backErr, fs.ErrNotExist) {
|
||||||
w.log.Error(
|
w.log.Error(
|
||||||
"failed to move archive file back",
|
"failed to move archive file back",
|
||||||
"from", to[j],
|
"from", path+moved,
|
||||||
"to", from[j],
|
"to", w.path+moved,
|
||||||
"error", backErr,
|
"error", backErr,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"renaming archive %s to %s: %w", from[i], to[i], err,
|
"renaming archive %s to %s: %w", w.path+suffix, path+suffix, err,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -595,7 +499,7 @@ func (w *archiveWriter) prune(expiry time.Duration) {
|
|||||||
if res.Error != nil {
|
if res.Error != nil {
|
||||||
w.log.Error(
|
w.log.Error(
|
||||||
"failed to prune expired archive rows",
|
"failed to prune expired archive rows",
|
||||||
"path", w.current,
|
"path", w.path,
|
||||||
"error", res.Error,
|
"error", res.Error,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -605,76 +509,48 @@ func (w *archiveWriter) prune(expiry time.Duration) {
|
|||||||
if res.RowsAffected > 0 {
|
if res.RowsAffected > 0 {
|
||||||
w.log.Info(
|
w.log.Info(
|
||||||
"pruned expired archive rows",
|
"pruned expired archive rows",
|
||||||
"path", w.current,
|
"path", w.path,
|
||||||
"rows_deleted", res.RowsAffected,
|
"rows_deleted", res.RowsAffected,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArchiveFileInfo is what the metadata of a database target's archive
|
// ArchiveFileInfo is what the metadata of a database target's archive
|
||||||
// files says about them.
|
// file says about it.
|
||||||
type ArchiveFileInfo struct {
|
type ArchiveFileInfo struct {
|
||||||
// Files counts the files.
|
// Size is the bytes on disk of the file and its -wal together.
|
||||||
Files int
|
|
||||||
|
|
||||||
// Size is the bytes on disk of the files and their -wal together.
|
|
||||||
Size int64
|
Size int64
|
||||||
|
|
||||||
// Written is when a file or a -wal was last modified, whichever is
|
// Written is when the file or its -wal was last modified, whichever
|
||||||
// latest: a write lands in the -wal first.
|
// is later: a write lands in the -wal first.
|
||||||
Written time.Time
|
Written time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// StatArchive reads the metadata of a database target's archive
|
// StatArchive reads the metadata of the archive file at path and of
|
||||||
// files, given the path ArchivePath gives it (see archiveFiles), and
|
// its -wal, without opening the archive. With no file at path, which is
|
||||||
// of their -wal, without opening them. With no files, which is so
|
// so before the first write and after the operator moved it away, the
|
||||||
// before the first write and after the operator moved them away, the
|
|
||||||
// error wraps fs.ErrNotExist.
|
// error wraps fs.ErrNotExist.
|
||||||
func StatArchive(path string) (ArchiveFileInfo, error) {
|
func StatArchive(path string) (ArchiveFileInfo, error) {
|
||||||
files, err := archiveFiles(path)
|
file, err := os.Stat(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ArchiveFileInfo{}, err
|
return ArchiveFileInfo{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
var info ArchiveFileInfo
|
info := ArchiveFileInfo{Size: file.Size(), Written: file.ModTime()}
|
||||||
|
|
||||||
for _, file := range files {
|
wal, err := os.Stat(path + "-wal")
|
||||||
db, err := os.Stat(file.path)
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
return info, nil
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return ArchiveFileInfo{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
info.Files++
|
|
||||||
info.Size += db.Size()
|
|
||||||
|
|
||||||
if db.ModTime().After(info.Written) {
|
|
||||||
info.Written = db.ModTime()
|
|
||||||
}
|
|
||||||
|
|
||||||
wal, err := os.Stat(file.path + "-wal")
|
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return ArchiveFileInfo{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
info.Size += wal.Size()
|
|
||||||
|
|
||||||
if wal.ModTime().After(info.Written) {
|
|
||||||
info.Written = wal.ModTime()
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if info.Files == 0 {
|
if err != nil {
|
||||||
return ArchiveFileInfo{}, fmt.Errorf(
|
return ArchiveFileInfo{}, err
|
||||||
"no archive file for %s: %w", path, fs.ErrNotExist,
|
}
|
||||||
)
|
|
||||||
|
info.Size += wal.Size()
|
||||||
|
|
||||||
|
if wal.ModTime().After(info.Written) {
|
||||||
|
info.Written = wal.ModTime()
|
||||||
}
|
}
|
||||||
|
|
||||||
return info, nil
|
return info, nil
|
||||||
|
|||||||
@@ -6,14 +6,10 @@ import (
|
|||||||
"database/sql"
|
"database/sql"
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"io/fs"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
@@ -54,46 +50,21 @@ func ArchiveExportFileName(
|
|||||||
at.UTC().Format("20060102T150405Z") + ".json.gz"
|
at.UTC().Format("20060102T150405Z") + ".json.gz"
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArchiveExport is a database target's archive listed for download. It
|
// ArchiveExport is a database target's archive opened for download.
|
||||||
// opens one of the target's files at a time, only when its rows are
|
// It reads the file on its own connection, inside one read-only
|
||||||
// about to be written out, and closes it before it opens the next, so
|
// transaction, so it writes out the archive as it stood when
|
||||||
// an export holds at most one file open however many the target has.
|
// OpenArchiveExport returned.
|
||||||
//
|
//
|
||||||
// Each file is read on its own connection inside one read-only
|
// Archives are in WAL mode, where a reader works from a snapshot and
|
||||||
// transaction, so its rows are written out as the file stood when it
|
// never blocks a writer: archive writes go on while an export is open,
|
||||||
// was opened. Archives are in WAL mode, where a reader works from a
|
// and the export does not see them. SQLite cannot checkpoint the -wal
|
||||||
// snapshot and never blocks a writer: archive writes go on while a file
|
// past an open snapshot, so the -wal grows until the export is closed.
|
||||||
// is open, and the export does not see them. SQLite cannot checkpoint
|
|
||||||
// a -wal past an open snapshot, so the open file's -wal grows until the
|
|
||||||
// export has written that file out.
|
|
||||||
type ArchiveExport struct {
|
type ArchiveExport struct {
|
||||||
// periods are the periods of the target's files when the export
|
|
||||||
// was listed, "" for the file without one, in the order
|
|
||||||
// archiveFiles lists them.
|
|
||||||
periods []string
|
|
||||||
|
|
||||||
// lock is held while currentPath is called and a file is opened,
|
|
||||||
// so that a rename, which holds it too, cannot move the file in
|
|
||||||
// between.
|
|
||||||
lock sync.Locker
|
|
||||||
|
|
||||||
// currentPath returns the path ArchivePath gives the target under
|
|
||||||
// the names stored for it now, which a rename may have changed since
|
|
||||||
// the export was listed.
|
|
||||||
currentPath func() (string, error)
|
|
||||||
|
|
||||||
log *slog.Logger
|
|
||||||
}
|
|
||||||
|
|
||||||
// exportFile is one archive file opened for an export.
|
|
||||||
type exportFile struct {
|
|
||||||
db *sql.DB
|
db *sql.DB
|
||||||
tx *gorm.DB
|
tx *gorm.DB
|
||||||
|
|
||||||
// period is the period in the file's name, "" for none.
|
// empty is true when there is nothing to read: no file, or a file
|
||||||
period string
|
// without the archive's table yet.
|
||||||
|
|
||||||
// empty is true for a file without the archive's table yet.
|
|
||||||
empty bool
|
empty bool
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -103,46 +74,26 @@ type exportedName struct {
|
|||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewArchiveExport lists a database target's archive files for export,
|
// OpenArchiveExport opens the archive file at path for export and
|
||||||
// given the path ArchivePath gives it (see archiveFiles). It opens none
|
// takes the snapshot the export reads. It never creates the file: with
|
||||||
// of them. Its caller holds lock, which every rename of the target's
|
// no file at path, the export has no rows.
|
||||||
// files runs under, from reading the names path is made of until it
|
|
||||||
// returns, so the files it lists are the ones those names give.
|
|
||||||
//
|
//
|
||||||
// WriteGzipJSON, called without lock held, finds each file again by its
|
// Once it has returned, the file is open, so a rename or a move of it
|
||||||
// period under the path currentPath gives, holding lock while it does
|
// does not affect the export, which reads the same file under its new
|
||||||
// and while it opens the file, so a rename during the export loses no
|
// name.
|
||||||
// file. A file that is gone by then, emptied by the sweep or moved
|
//
|
||||||
// away, is skipped. The export never creates a file: with no files, it
|
// The transaction lasts as long as ctx does, so ctx must last for the
|
||||||
// has no rows.
|
// whole export.
|
||||||
func NewArchiveExport(
|
func OpenArchiveExport(
|
||||||
path string,
|
ctx context.Context, path string, log *slog.Logger,
|
||||||
lock sync.Locker,
|
|
||||||
currentPath func() (string, error),
|
|
||||||
log *slog.Logger,
|
|
||||||
) (*ArchiveExport, error) {
|
) (*ArchiveExport, error) {
|
||||||
files, err := archiveFiles(path)
|
if !fileExists(path) {
|
||||||
if err != nil {
|
return &ArchiveExport{empty: true}, nil
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
x := &ArchiveExport{lock: lock, currentPath: currentPath, log: log}
|
db, err := database.OpenSQLite(path, archiveModeExisting)
|
||||||
|
|
||||||
for _, file := range files {
|
|
||||||
x.periods = append(x.periods, file.period)
|
|
||||||
}
|
|
||||||
|
|
||||||
return x, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// openExportFile opens one archive file for an export and takes its
|
|
||||||
// snapshot. The transaction lasts as long as ctx does.
|
|
||||||
func openExportFile(
|
|
||||||
ctx context.Context, file archiveFile, log *slog.Logger,
|
|
||||||
) (*exportFile, error) {
|
|
||||||
db, err := database.OpenSQLite(file.path, archiveModeExisting)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("opening archive %s: %w", file.path, err)
|
return nil, fmt.Errorf("opening archive %s: %w", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
gdb, err := gorm.Open(
|
gdb, err := gorm.Open(
|
||||||
@@ -155,7 +106,7 @@ func openExportFile(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
_ = db.Close()
|
_ = db.Close()
|
||||||
|
|
||||||
return nil, fmt.Errorf("opening archive %s: %w", file.path, err)
|
return nil, fmt.Errorf("opening archive %s: %w", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReadOnly makes the driver begin a deferred transaction in place
|
// ReadOnly makes the driver begin a deferred transaction in place
|
||||||
@@ -165,9 +116,7 @@ func openExportFile(
|
|||||||
if tx.Error != nil {
|
if tx.Error != nil {
|
||||||
_ = db.Close()
|
_ = db.Close()
|
||||||
|
|
||||||
return nil, fmt.Errorf(
|
return nil, fmt.Errorf("reading archive %s: %w", path, tx.Error)
|
||||||
"reading archive %s: %w", file.path, tx.Error,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// The transaction's first read is what takes the snapshot.
|
// The transaction's first read is what takes the snapshot.
|
||||||
@@ -178,27 +127,22 @@ func openExportFile(
|
|||||||
_ = tx.Rollback()
|
_ = tx.Rollback()
|
||||||
_ = db.Close()
|
_ = db.Close()
|
||||||
|
|
||||||
return nil, fmt.Errorf("reading archive %s: %w", file.path, err)
|
return nil, fmt.Errorf("reading archive %s: %w", path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &exportFile{
|
return &ArchiveExport{db: db, tx: tx, empty: tables == 0}, nil
|
||||||
db: db, tx: tx, period: file.period, empty: tables == 0,
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteGzipJSON writes the export to w as one gzipped JSON object:
|
// WriteGzipJSON writes the export to w as one gzipped JSON object:
|
||||||
// webhook and target, each an id and a name; exported_at; and
|
// webhook and target, each an id and a name; exported_at; and
|
||||||
// archived_events, one object per archived row, keyed by column name,
|
// archived_events, one object per archived row, keyed by column name.
|
||||||
// the files in the order archiveFiles lists them. A row from a file
|
// A body that is not valid UTF-8 cannot be a JSON string, so it is
|
||||||
// named for a period has "period" beside its columns. A body that is
|
// written in base64, with "body_encoding": "base64" beside it.
|
||||||
// not valid UTF-8 cannot be a JSON string, so it is written in base64,
|
|
||||||
// with "body_encoding": "base64" beside it.
|
|
||||||
//
|
//
|
||||||
// Each row is written out before the next is read, so neither the
|
// Each row is written out before the next is read, so neither the
|
||||||
// archive nor its JSON is ever held in memory whole, and each file is
|
// archive nor its JSON is ever held in memory whole. After an error
|
||||||
// closed once its rows are written, before the next is opened. When it
|
// the gzip stream is left unfinished, so what was written does not
|
||||||
// returns, no file is open. After an error the gzip stream is left
|
// decompress as a whole file.
|
||||||
// unfinished, so what was written does not decompress as a whole file.
|
|
||||||
func (x *ArchiveExport) WriteGzipJSON(
|
func (x *ArchiveExport) WriteGzipJSON(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
w io.Writer,
|
w io.Writer,
|
||||||
@@ -225,35 +169,15 @@ func (x *ArchiveExport) WriteGzipJSON(
|
|||||||
return zw.Close()
|
return zw.Close()
|
||||||
}
|
}
|
||||||
|
|
||||||
// openFile finds the target's archive file for period under the path
|
// Close ends the export's transaction and closes its connection.
|
||||||
// currentPath gives now, and opens it for the export, holding x.lock
|
func (x *ArchiveExport) Close() error {
|
||||||
// for both. For a file that is gone, the error wraps fs.ErrNotExist.
|
if x.db == nil {
|
||||||
func (x *ArchiveExport) openFile(
|
return nil
|
||||||
ctx context.Context, period string,
|
|
||||||
) (*exportFile, error) {
|
|
||||||
x.lock.Lock()
|
|
||||||
defer x.lock.Unlock()
|
|
||||||
|
|
||||||
path, err := x.currentPath()
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("finding archive file: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
file := archiveFile{path: archivePeriodPath(path, period), period: period}
|
_ = x.tx.Rollback()
|
||||||
|
|
||||||
_, err = os.Stat(file.path)
|
return x.db.Close()
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return openExportFile(ctx, file, x.log)
|
|
||||||
}
|
|
||||||
|
|
||||||
// close ends the file's transaction and closes its connection.
|
|
||||||
func (f *exportFile) close() error {
|
|
||||||
_ = f.tx.Rollback()
|
|
||||||
|
|
||||||
return f.db.Close()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeJSON writes head with archived_events added as its last key,
|
// writeJSON writes head with archived_events added as its last key,
|
||||||
@@ -283,77 +207,46 @@ func (x *ArchiveExport) writeJSON(
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeRows writes the archived rows of each file to w, one per line,
|
// writeRows writes each archived row to w, oldest first, one per line,
|
||||||
// separated by commas, opening each file in turn and closing it once
|
// separated by commas.
|
||||||
// its rows are written.
|
|
||||||
func (x *ArchiveExport) writeRows(ctx context.Context, w io.Writer) error {
|
func (x *ArchiveExport) writeRows(ctx context.Context, w io.Writer) error {
|
||||||
sep := "\n"
|
if x.empty {
|
||||||
|
return nil
|
||||||
for _, period := range x.periods {
|
|
||||||
f, err := x.openFile(ctx, period)
|
|
||||||
if errors.Is(err, fs.ErrNotExist) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
sep, err = f.writeRows(ctx, w, sep)
|
|
||||||
|
|
||||||
err = errors.Join(err, f.close())
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
rows, err := x.tx.WithContext(ctx).
|
||||||
}
|
|
||||||
|
|
||||||
// writeRows writes the file's archived rows to w, oldest first, the
|
|
||||||
// first after sep and each other after ",\n". It returns what goes
|
|
||||||
// before the next row: sep again when the file had no rows.
|
|
||||||
func (f *exportFile) writeRows(
|
|
||||||
ctx context.Context, w io.Writer, sep string,
|
|
||||||
) (string, error) {
|
|
||||||
if f.empty {
|
|
||||||
return sep, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
rows, err := f.tx.WithContext(ctx).
|
|
||||||
Model(&archivedEvent{}).Order("id").Rows()
|
Model(&archivedEvent{}).Order("id").Rows()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func() { _ = rows.Close() }()
|
defer func() { _ = rows.Close() }()
|
||||||
|
|
||||||
for ; rows.Next(); sep = ",\n" {
|
for sep := "\n"; rows.Next(); sep = ",\n" {
|
||||||
var ev archivedEvent
|
var ev archivedEvent
|
||||||
|
|
||||||
err = f.tx.ScanRows(rows, &ev)
|
err = x.tx.ScanRows(rows, &ev)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = io.WriteString(w, sep)
|
_, err = io.WriteString(w, sep)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = writeRow(w, &ev, f.period)
|
err = writeRow(w, &ev)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return sep, rows.Err()
|
return rows.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeRow writes an archived row to w as a JSON object keyed by
|
// writeRow writes an archived row to w as a JSON object keyed by
|
||||||
// column name, its body in base64 when it is not valid UTF-8, with
|
// column name, its body in base64 when it is not valid UTF-8.
|
||||||
// the period of its file beside them unless that is "".
|
func writeRow(w io.Writer, ev *archivedEvent) error {
|
||||||
func writeRow(w io.Writer, ev *archivedEvent, period string) error {
|
|
||||||
row := map[string]any{
|
row := map[string]any{
|
||||||
"id": ev.ID,
|
"id": ev.ID,
|
||||||
"event_id": ev.EventID,
|
"event_id": ev.EventID,
|
||||||
@@ -371,10 +264,6 @@ func writeRow(w io.Writer, ev *archivedEvent, period string) error {
|
|||||||
row["body_encoding"] = "base64"
|
row["body_encoding"] = "base64"
|
||||||
}
|
}
|
||||||
|
|
||||||
if period != "" {
|
|
||||||
row["period"] = period
|
|
||||||
}
|
|
||||||
|
|
||||||
line, err := json.Marshal(row)
|
line, err := json.Marshal(row)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -12,8 +12,6 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -31,8 +29,14 @@ const (
|
|||||||
exportTargetName = "Long-term archive"
|
exportTargetName = "Long-term archive"
|
||||||
)
|
)
|
||||||
|
|
||||||
// binaryBody is a body that is not valid UTF-8.
|
const (
|
||||||
const binaryBody = "\xff\xfe\x00\x01binary\x80"
|
// binaryBody is a body that is not valid UTF-8.
|
||||||
|
binaryBody = "\xff\xfe\x00\x01binary\x80"
|
||||||
|
|
||||||
|
// openedEventID is the event the snapshot tests archive before
|
||||||
|
// they open the export.
|
||||||
|
openedEventID = "opened"
|
||||||
|
)
|
||||||
|
|
||||||
// writeExportTo writes export to w as the archive of the export tests'
|
// writeExportTo writes export to w as the archive of the export tests'
|
||||||
// webhook and target, exported at 2026-10-02T12:03:04Z.
|
// webhook and target, exported at 2026-10-02T12:03:04Z.
|
||||||
@@ -55,41 +59,19 @@ func writeExportTo(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// listExport lists the archive at path for export, as the archive of a
|
|
||||||
// target whose names do not change.
|
|
||||||
func listExport(t *testing.T, path string) *delivery.ArchiveExport {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return newExport(t, path, &sync.Mutex{}, func() (string, error) {
|
|
||||||
return path, nil
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// newExport lists the archive at path for export, to find each file
|
|
||||||
// again under the path currentPath gives, holding lock while it does.
|
|
||||||
// Nothing else takes lock while it lists, so it does not hold lock.
|
|
||||||
func newExport(
|
|
||||||
t *testing.T,
|
|
||||||
path string,
|
|
||||||
lock sync.Locker,
|
|
||||||
currentPath func() (string, error),
|
|
||||||
) *delivery.ArchiveExport {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
export, err := delivery.NewArchiveExport(
|
|
||||||
path, lock, currentPath, archiveTestLogger(),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
return export
|
|
||||||
}
|
|
||||||
|
|
||||||
// exportArchive runs a whole export of the archive at path and returns
|
// exportArchive runs a whole export of the archive at path and returns
|
||||||
// its JSON, decompressed and parsed.
|
// its JSON, decompressed and parsed.
|
||||||
func exportArchive(t *testing.T, path string) map[string]any {
|
func exportArchive(t *testing.T, path string) map[string]any {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
return writeExport(t, listExport(t, path))
|
export, err := delivery.OpenArchiveExport(
|
||||||
|
t.Context(), path, archiveTestLogger(),
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
defer func() { require.NoError(t, export.Close()) }()
|
||||||
|
|
||||||
|
return writeExport(t, export)
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeExport writes an opened export and returns its JSON,
|
// writeExport writes an opened export and returns its JSON,
|
||||||
@@ -259,202 +241,64 @@ func TestArchiveExport_Empty(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// unlockHook is a sync.Locker that runs fn each time it is unlocked. An
|
// TestArchiveExport_ReadsOneSnapshot proves an export writes the
|
||||||
// export unlocks its lock right after it opens a file.
|
// archive as it was when it was opened, and holds up no archive
|
||||||
type unlockHook struct {
|
// write: a row written while the export is open is stored, and is not
|
||||||
sync.Mutex
|
// in the export. A write held up for the whole busy timeout would
|
||||||
|
// fail.
|
||||||
fn func()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (u *unlockHook) Unlock() {
|
|
||||||
u.Mutex.Unlock()
|
|
||||||
u.fn()
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveExport_ReadsOneSnapshot proves an export writes a file
|
|
||||||
// out as it was when the export opened it, and holds up no archive
|
|
||||||
// write: a row written after the export was listed but before the file
|
|
||||||
// was opened is in the export, and one written while the file is open
|
|
||||||
// is stored, and is not. A write held up for the whole busy timeout
|
|
||||||
// would fail.
|
|
||||||
func TestArchiveExport_ReadsOneSnapshot(t *testing.T) {
|
func TestArchiveExport_ReadsOneSnapshot(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive.db")
|
path := filepath.Join(t.TempDir(), "archive.db")
|
||||||
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
||||||
|
|
||||||
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "listed"}, 0))
|
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: openedEventID}, 0))
|
||||||
|
|
||||||
opened := &unlockHook{fn: func() {
|
export, err := delivery.OpenArchiveExport(
|
||||||
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "during"}, 0))
|
t.Context(), path, archiveTestLogger(),
|
||||||
}}
|
)
|
||||||
export := newExport(t, path, opened, func() (string, error) {
|
require.NoError(t, err)
|
||||||
return path, nil
|
|
||||||
})
|
|
||||||
|
|
||||||
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "before-open"}, 0))
|
defer func() { require.NoError(t, export.Close()) }()
|
||||||
|
|
||||||
|
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "during"}, 0))
|
||||||
|
|
||||||
assert.Equal(t,
|
assert.Equal(t,
|
||||||
[]string{"listed", "before-open"},
|
[]string{openedEventID}, exportedEventIDs(t, writeExport(t, export)),
|
||||||
exportedEventIDs(t, writeExport(t, export)),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var stored int64
|
var stored int64
|
||||||
|
|
||||||
require.NoError(t, openArchiveDBForRead(t, path).
|
require.NoError(t, openArchiveDBForRead(t, path).
|
||||||
Model(&delivery.ExportArchivedEvent{}).Count(&stored).Error)
|
Model(&delivery.ExportArchivedEvent{}).Count(&stored).Error)
|
||||||
assert.Equal(t, int64(3), stored)
|
assert.Equal(t, int64(2), stored)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveExport_FindsFilesAfterRename proves that renaming the
|
// TestArchiveExport_SurvivesRename proves that renaming the archive
|
||||||
// archive after an export has listed it, as renaming its webhook or
|
// while an export of it is open, as renaming its webhook or target
|
||||||
// target does, loses no file: the export finds each file again by its
|
// does, leaves the export reading the same file.
|
||||||
// period under the new name. A file moved away by then is skipped.
|
func TestArchiveExport_SurvivesRename(t *testing.T) {
|
||||||
func TestArchiveExport_FindsFilesAfterRename(t *testing.T) {
|
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
dir := t.TempDir()
|
path := filepath.Join(t.TempDir(), "archive-old.db")
|
||||||
path := filepath.Join(dir, "archive-old.db")
|
|
||||||
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
||||||
|
|
||||||
for _, period := range []string{"", dayPeriod, hourPeriod} {
|
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: openedEventID}, 0))
|
||||||
require.NoError(t, w.WritePeriod(
|
|
||||||
delivery.ExportArchivedEvent{EventID: "in-" + period}, 0, period,
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
current := path
|
export, err := delivery.OpenArchiveExport(
|
||||||
export := newExport(t, path, &sync.Mutex{}, func() (string, error) {
|
t.Context(), path, archiveTestLogger(),
|
||||||
return current, nil
|
|
||||||
})
|
|
||||||
|
|
||||||
require.NoError(t, w.Rename("archive-new.db"))
|
|
||||||
|
|
||||||
current = filepath.Join(dir, "archive-new.db")
|
|
||||||
|
|
||||||
removeArchiveFiles(t, periodPath(current, dayPeriod))
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
[]string{"in-", "in-" + hourPeriod},
|
|
||||||
exportedEventIDs(t, writeExport(t, export)),
|
|
||||||
)
|
)
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveExport_EveryFileOldestFirst writes a row to a target's
|
|
||||||
// file without a period and to its files for a month, an hour and a
|
|
||||||
// day, and proves the export holds every row: the file without a
|
|
||||||
// period first, then the others oldest period first, each row from a
|
|
||||||
// file named for a period carrying that period. A file made after the
|
|
||||||
// export was listed is not in it.
|
|
||||||
func TestArchiveExport_EveryFileOldestFirst(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
|
||||||
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
|
||||||
|
|
||||||
// Written in an order that is not the export's.
|
|
||||||
for _, period := range []string{nextDayPeriod, "", hourPeriod, "2026-03"} {
|
|
||||||
require.NoError(t, w.WritePeriod(
|
|
||||||
delivery.ExportArchivedEvent{EventID: "in-" + period}, 0, period,
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
export := listExport(t, path)
|
|
||||||
|
|
||||||
require.NoError(t, w.WritePeriod(
|
|
||||||
delivery.ExportArchivedEvent{EventID: "later"}, 0, "2026-03-06",
|
|
||||||
))
|
|
||||||
|
|
||||||
events := exportedEvents(t, writeExport(t, export))
|
|
||||||
ids := make([]string, 0, len(events))
|
|
||||||
periods := make([]any, 0, len(events))
|
|
||||||
|
|
||||||
for _, ev := range events {
|
|
||||||
ids = append(ids, fmt.Sprint(ev["event_id"]))
|
|
||||||
periods = append(periods, ev["period"])
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
[]string{"in-", "in-2026-03", "in-" + hourPeriod, "in-" + nextDayPeriod},
|
|
||||||
ids,
|
|
||||||
)
|
|
||||||
assert.Equal(t,
|
|
||||||
[]any{nil, "2026-03", hourPeriod, nextDayPeriod}, periods,
|
|
||||||
)
|
|
||||||
assert.NotContains(t, events[0], "period",
|
|
||||||
"a row from the file without a period has no period")
|
|
||||||
}
|
|
||||||
|
|
||||||
// openFilesPeak is an io.Writer that discards what it is given and
|
|
||||||
// records the most archive files in dir the process had open at any
|
|
||||||
// write, as /proc/self/fd lists the files a process has open.
|
|
||||||
type openFilesPeak struct {
|
|
||||||
dir string
|
|
||||||
max int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *openFilesPeak) Write(b []byte) (int, error) {
|
|
||||||
fds, err := os.ReadDir("/proc/self/fd")
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
open := map[string]bool{}
|
|
||||||
|
|
||||||
for _, fd := range fds {
|
|
||||||
file, err := os.Readlink(filepath.Join("/proc/self/fd", fd.Name()))
|
|
||||||
if err == nil && filepath.Dir(file) == p.dir &&
|
|
||||||
strings.HasSuffix(file, ".db") {
|
|
||||||
open[file] = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
p.max = max(p.max, len(open))
|
|
||||||
|
|
||||||
return len(b), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveExport_OneFileOpenAtATime exports a target with a file for
|
|
||||||
// each of 24 hours and proves the export never had more than one of
|
|
||||||
// them open, and had one open while it wrote. Each file holds a row of
|
|
||||||
// 48 KiB of random base64, which gzip shrinks little, so the export
|
|
||||||
// writes output while it reads each file.
|
|
||||||
func TestArchiveExport_OneFileOpenAtATime(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
if runtime.GOOS != "linux" {
|
|
||||||
t.Skip("only Linux lists a process's open files in /proc/self/fd")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Readlink gives each open file's path with no symbolic link in it.
|
|
||||||
dir, err := filepath.EvalSymlinks(t.TempDir())
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
path := filepath.Join(dir, "archive-wh.db")
|
defer func() { require.NoError(t, export.Close()) }()
|
||||||
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
|
||||||
random := make([]byte, 36<<10)
|
|
||||||
|
|
||||||
for hour := range 24 {
|
require.NoError(t, w.Rename("archive-new.db"))
|
||||||
_, _ = rand.Read(random)
|
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "after"}, 0))
|
||||||
|
require.NoFileExists(t, path)
|
||||||
|
|
||||||
require.NoError(t, w.WritePeriod(delivery.ExportArchivedEvent{
|
assert.Equal(t,
|
||||||
Body: base64.StdEncoding.EncodeToString(random),
|
[]string{openedEventID}, exportedEventIDs(t, writeExport(t, export)),
|
||||||
}, 0, fmt.Sprintf("2026-10-01-%02d", hour)))
|
)
|
||||||
}
|
|
||||||
|
|
||||||
// The writer's own handle on the last file is not the export's.
|
|
||||||
w.Evict()
|
|
||||||
|
|
||||||
// Through a buffer, the open files are listed once per 8 KiB of
|
|
||||||
// output, a few times for each file, rather than at each of gzip's
|
|
||||||
// small writes, which takes far longer.
|
|
||||||
peak := &openFilesPeak{dir: dir}
|
|
||||||
buffered := bufio.NewWriterSize(peak, 8<<10)
|
|
||||||
|
|
||||||
require.NoError(t, writeExportTo(t, listExport(t, path), buffered))
|
|
||||||
require.NoError(t, buffered.Flush())
|
|
||||||
assert.Equal(t, 1, peak.max)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// heapPeak is an io.Writer that discards what it is given and records
|
// heapPeak is an io.Writer that discards what it is given and records
|
||||||
@@ -494,7 +338,12 @@ func exportHeapGrowth(t *testing.T, rows, bodySize int) uint64 {
|
|||||||
}, 0))
|
}, 0))
|
||||||
}
|
}
|
||||||
|
|
||||||
export := listExport(t, path)
|
export, err := delivery.OpenArchiveExport(
|
||||||
|
t.Context(), path, archiveTestLogger(),
|
||||||
|
)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
defer func() { require.NoError(t, export.Close()) }()
|
||||||
|
|
||||||
runtime.GC()
|
runtime.GC()
|
||||||
|
|
||||||
|
|||||||
@@ -1,198 +0,0 @@
|
|||||||
package delivery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The archive rotations: how often a database target starts a new
|
|
||||||
// archive file. Every rotation but none puts the period of an event's
|
|
||||||
// receive time, in UTC, in the name of the file the event goes to,
|
|
||||||
// written in the layout beside it.
|
|
||||||
const (
|
|
||||||
archiveRotationNone = "none"
|
|
||||||
archiveRotationMonthly = "monthly"
|
|
||||||
archiveRotationDaily = "daily"
|
|
||||||
archiveRotationHourly = "hourly"
|
|
||||||
|
|
||||||
archiveMonthLayout = "2006-01"
|
|
||||||
archiveDayLayout = "2006-01-02"
|
|
||||||
archiveHourLayout = "2006-01-02-15"
|
|
||||||
)
|
|
||||||
|
|
||||||
// errArchiveRotationUnknown is returned for a rotation that is not one
|
|
||||||
// of the four.
|
|
||||||
var errArchiveRotationUnknown = errors.New(
|
|
||||||
"rotation must be none, monthly, daily or hourly",
|
|
||||||
)
|
|
||||||
|
|
||||||
// archiveFile is one of a database target's archive files, and the
|
|
||||||
// period in its name: "" for the file named without one.
|
|
||||||
type archiveFile struct {
|
|
||||||
path string
|
|
||||||
period string
|
|
||||||
}
|
|
||||||
|
|
||||||
// ValidateArchiveRotation checks a user-supplied archive rotation for
|
|
||||||
// a database target: empty or none (both meaning one file), monthly,
|
|
||||||
// daily or hourly.
|
|
||||||
func ValidateArchiveRotation(rotation string) error {
|
|
||||||
switch rotation {
|
|
||||||
case "", archiveRotationNone, archiveRotationMonthly,
|
|
||||||
archiveRotationDaily, archiveRotationHourly:
|
|
||||||
return nil
|
|
||||||
default:
|
|
||||||
return fmt.Errorf("%w: %q", errArchiveRotationUnknown, rotation)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseArchiveRotation reads the rotation from a database target's
|
|
||||||
// config JSON. An empty config or an empty rotation is none.
|
|
||||||
func parseArchiveRotation(configJSON string) (string, error) {
|
|
||||||
if configJSON == "" {
|
|
||||||
return archiveRotationNone, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var cfg databaseTargetConfig
|
|
||||||
|
|
||||||
err := json.Unmarshal([]byte(configJSON), &cfg)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("parsing database target config: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = ValidateArchiveRotation(cfg.Rotation)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
if cfg.Rotation == "" {
|
|
||||||
return archiveRotationNone, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return cfg.Rotation, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// archivePeriod returns the period, in UTC, that a rotation puts an
|
|
||||||
// event received at receivedAt in: "2026-10" for monthly,
|
|
||||||
// "2026-10-01" for daily, "2026-10-01-19" for hourly, and "" for none.
|
|
||||||
func archivePeriod(rotation string, receivedAt time.Time) string {
|
|
||||||
switch rotation {
|
|
||||||
case archiveRotationMonthly:
|
|
||||||
return receivedAt.UTC().Format(archiveMonthLayout)
|
|
||||||
case archiveRotationDaily:
|
|
||||||
return receivedAt.UTC().Format(archiveDayLayout)
|
|
||||||
case archiveRotationHourly:
|
|
||||||
return receivedAt.UTC().Format(archiveHourLayout)
|
|
||||||
default:
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// archivePeriodPath returns the path of a database target's archive
|
|
||||||
// file for a period: path, as ArchivePath gives it, with "-" and the
|
|
||||||
// period put before its ".db". The period "" gives path itself.
|
|
||||||
func archivePeriodPath(path, period string) string {
|
|
||||||
if period == "" {
|
|
||||||
return path
|
|
||||||
}
|
|
||||||
|
|
||||||
return strings.TrimSuffix(path, ".db") + "-" + period + ".db"
|
|
||||||
}
|
|
||||||
|
|
||||||
// ArchivePathAt returns the archive file a database target writes an
|
|
||||||
// event received at receivedAt to: ArchivePath's file, with the
|
|
||||||
// period in its name when the target rotates.
|
|
||||||
func ArchivePathAt(
|
|
||||||
dbMgr *database.WebhookDBManager,
|
|
||||||
webhook *database.Webhook,
|
|
||||||
target *database.Target,
|
|
||||||
receivedAt time.Time,
|
|
||||||
) (string, error) {
|
|
||||||
rotation, err := parseArchiveRotation(target.Config)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
return archivePeriodPath(
|
|
||||||
ArchivePath(dbMgr, webhook, target),
|
|
||||||
archivePeriod(rotation, receivedAt),
|
|
||||||
), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// archiveFiles lists the database target's archive files that exist,
|
|
||||||
// given the path ArchivePath gives it: the file at path, then each
|
|
||||||
// file archivePeriodPath names for path and a period, oldest period
|
|
||||||
// first. Which rotation wrote a file does not matter, so the files of
|
|
||||||
// an earlier rotation setting are listed too.
|
|
||||||
func archiveFiles(path string) ([]archiveFile, error) {
|
|
||||||
dir := filepath.Dir(path)
|
|
||||||
|
|
||||||
entries, err := os.ReadDir(dir)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("listing archive files: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
stem := strings.TrimSuffix(filepath.Base(path), ".db")
|
|
||||||
|
|
||||||
var files []archiveFile
|
|
||||||
|
|
||||||
for _, entry := range entries {
|
|
||||||
period, ok := archiveFilePeriod(stem, entry.Name())
|
|
||||||
if ok {
|
|
||||||
files = append(files, archiveFile{
|
|
||||||
path: filepath.Join(dir, entry.Name()),
|
|
||||||
period: period,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A month sorts before the days and hours in it.
|
|
||||||
slices.SortFunc(files, func(a, b archiveFile) int {
|
|
||||||
return strings.Compare(a.period, b.period)
|
|
||||||
})
|
|
||||||
|
|
||||||
return files, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// archiveFilePeriod reports whether name is the name of an archive
|
|
||||||
// file of the target whose file name without a period is stem+".db",
|
|
||||||
// and the period in it.
|
|
||||||
func archiveFilePeriod(stem, name string) (string, bool) {
|
|
||||||
rest, ok := strings.CutPrefix(name, stem)
|
|
||||||
if !ok {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
rest, ok = strings.CutSuffix(rest, ".db")
|
|
||||||
if !ok {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
if rest == "" {
|
|
||||||
return "", true
|
|
||||||
}
|
|
||||||
|
|
||||||
period, ok := strings.CutPrefix(rest, "-")
|
|
||||||
if !ok {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, layout := range []string{
|
|
||||||
archiveMonthLayout, archiveDayLayout, archiveHourLayout,
|
|
||||||
} {
|
|
||||||
_, err := time.Parse(layout, period)
|
|
||||||
if err == nil {
|
|
||||||
return period, true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
@@ -1,389 +0,0 @@
|
|||||||
package delivery_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The archive rotations, and the configs of a daily target and of one
|
|
||||||
// whose rotation is not one of the four.
|
|
||||||
const (
|
|
||||||
rotationNone = "none"
|
|
||||||
rotationMonthly = "monthly"
|
|
||||||
rotationDaily = "daily"
|
|
||||||
rotationHourly = "hourly"
|
|
||||||
|
|
||||||
dailyConfig = `{"rotation":"daily"}`
|
|
||||||
weeklyConfig = `{"rotation":"weekly"}`
|
|
||||||
)
|
|
||||||
|
|
||||||
// The periods the tests archive into most: two days, and an hour of
|
|
||||||
// the first.
|
|
||||||
const (
|
|
||||||
dayPeriod = "2026-03-04"
|
|
||||||
nextDayPeriod = "2026-03-05"
|
|
||||||
hourPeriod = "2026-03-04-05"
|
|
||||||
)
|
|
||||||
|
|
||||||
// periodPath returns the archive file for a period of the target whose
|
|
||||||
// file without a period is path.
|
|
||||||
func periodPath(path, period string) string {
|
|
||||||
return strings.TrimSuffix(path, ".db") + "-" + period + ".db"
|
|
||||||
}
|
|
||||||
|
|
||||||
// deliverReceivedAt delivers to a database target an event whose
|
|
||||||
// receive time is receivedAt, and returns the event's id. The receive
|
|
||||||
// time is what decides a rotated archive's file, so setting it is how
|
|
||||||
// these tests move the clock across a period boundary.
|
|
||||||
func (env *archiveEnv) deliverReceivedAt(
|
|
||||||
t *testing.T, tgt *database.Target, receivedAt time.Time,
|
|
||||||
) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
|
||||||
event := seedEvent(t, webhookDB, `{"n":1}`)
|
|
||||||
event.CreatedAt = receivedAt
|
|
||||||
|
|
||||||
env.eng.ExportDeliverDatabase(
|
|
||||||
webhookDB, seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
|
|
||||||
)
|
|
||||||
|
|
||||||
return event.ID
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDeliverDatabase_RotatesAtEachPeriodBoundary delivers, for each
|
|
||||||
// rotation, an event received in the last second of a period and one
|
|
||||||
// received in the first second of the next, and checks each lands in
|
|
||||||
// the file named for its own period, in UTC. Rotation none keeps both
|
|
||||||
// in the one file.
|
|
||||||
func TestDeliverDatabase_RotatesAtEachPeriodBoundary(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
berlin := time.FixedZone("CEST", 2*60*60)
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
rotation string
|
|
||||||
before, after time.Time
|
|
||||||
// periods are the periods of before and after.
|
|
||||||
periods [2]string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
rotationMonthly, rotationMonthly,
|
|
||||||
time.Date(2026, 1, 31, 23, 59, 59, 0, time.UTC),
|
|
||||||
time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC),
|
|
||||||
[2]string{"2026-01", "2026-02"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
rotationDaily, rotationDaily,
|
|
||||||
time.Date(2026, 3, 4, 23, 59, 59, 0, time.UTC),
|
|
||||||
time.Date(2026, 3, 5, 0, 0, 0, 0, time.UTC),
|
|
||||||
[2]string{dayPeriod, nextDayPeriod},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
// The same instants, received in a zone two hours ahead
|
|
||||||
// of UTC, where they fall on 5 March: the period is UTC's.
|
|
||||||
"daily in another zone", rotationDaily,
|
|
||||||
time.Date(2026, 3, 5, 1, 59, 59, 0, berlin),
|
|
||||||
time.Date(2026, 3, 5, 2, 0, 0, 0, berlin),
|
|
||||||
[2]string{dayPeriod, nextDayPeriod},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
rotationHourly, rotationHourly,
|
|
||||||
time.Date(2026, 3, 4, 5, 59, 59, 0, time.UTC),
|
|
||||||
time.Date(2026, 3, 4, 6, 0, 0, 0, time.UTC),
|
|
||||||
[2]string{hourPeriod, "2026-03-04-06"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"rotation":"`+tc.rotation+`"}`)
|
|
||||||
path := env.archivePath(tgt)
|
|
||||||
|
|
||||||
first := env.deliverReceivedAt(t, tgt, tc.before)
|
|
||||||
second := env.deliverReceivedAt(t, tgt, tc.after)
|
|
||||||
|
|
||||||
assert.Equal(t, []string{first},
|
|
||||||
archivedEventIDs(t, periodPath(path, tc.periods[0])))
|
|
||||||
assert.Equal(t, []string{second},
|
|
||||||
archivedEventIDs(t, periodPath(path, tc.periods[1])))
|
|
||||||
assert.NoFileExists(t, path,
|
|
||||||
"a rotated target never writes the file without a period")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run(rotationNone, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"rotation":"`+rotationNone+`"}`)
|
|
||||||
|
|
||||||
first := env.deliverReceivedAt(t, tgt, cases[0].before)
|
|
||||||
second := env.deliverReceivedAt(t, tgt, cases[0].after)
|
|
||||||
|
|
||||||
assert.ElementsMatch(t, []string{first, second},
|
|
||||||
archivedEventIDs(t, env.archivePath(tgt)))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDeliverDatabase_EventWithoutReceiveTime proves an event whose
|
|
||||||
// receive time is not known goes to the file for the time it is
|
|
||||||
// archived, rather than to one for the year 1.
|
|
||||||
func TestDeliverDatabase_EventWithoutReceiveTime(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"rotation":"`+rotationMonthly+`"}`)
|
|
||||||
path := env.archivePath(tgt)
|
|
||||||
|
|
||||||
before := time.Now().UTC().Format("2006-01")
|
|
||||||
id := env.deliverReceivedAt(t, tgt, time.Time{})
|
|
||||||
|
|
||||||
file := periodPath(path, time.Now().UTC().Format("2006-01"))
|
|
||||||
|
|
||||||
_, err := os.Stat(file)
|
|
||||||
if err != nil {
|
|
||||||
// The month turned during the delivery.
|
|
||||||
file = periodPath(path, before)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, []string{id}, archivedEventIDs(t, file))
|
|
||||||
assert.NoFileExists(t, periodPath(path, "0001-01"))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDeliverDatabase_RotationChangeKeepsOldFiles changes a target's
|
|
||||||
// rotation from none to daily between two events, and checks the
|
|
||||||
// second goes to the daily file while the first stays where it was.
|
|
||||||
func TestDeliverDatabase_RotationChangeKeepsOldFiles(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, "")
|
|
||||||
path := env.archivePath(tgt)
|
|
||||||
at := time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
first := env.deliverReceivedAt(t, tgt, at)
|
|
||||||
|
|
||||||
tgt.Config = dailyConfig
|
|
||||||
second := env.deliverReceivedAt(t, tgt, at)
|
|
||||||
|
|
||||||
assert.Equal(t, []string{first}, archivedEventIDs(t, path))
|
|
||||||
assert.Equal(t, []string{second},
|
|
||||||
archivedEventIDs(t, periodPath(path, dayPeriod)))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveSweep_PrunesEveryFile gives a daily target three files:
|
|
||||||
// the file without a period, left from before it rotated, and two
|
|
||||||
// daily files. Each holds a row older than the expiry, and one daily
|
|
||||||
// file also a newer row. The sweep prunes the old row from every file,
|
|
||||||
// deletes the daily file it leaves empty, and keeps the file without a
|
|
||||||
// period although it is empty too.
|
|
||||||
func TestArchiveSweep_PrunesEveryFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(
|
|
||||||
t, `{"expiry":"1h","rotation":"`+rotationDaily+`"}`,
|
|
||||||
)
|
|
||||||
path := env.archivePath(tgt)
|
|
||||||
emptied := periodPath(path, dayPeriod)
|
|
||||||
kept := periodPath(path, nextDayPeriod)
|
|
||||||
|
|
||||||
now := time.Now()
|
|
||||||
old := now.Add(-48 * time.Hour)
|
|
||||||
|
|
||||||
seedArchiveFile(t, path, tgt.WebhookID, old)
|
|
||||||
seedArchiveFile(t, emptied, tgt.WebhookID, old)
|
|
||||||
seedArchiveFile(t, kept, tgt.WebhookID, old, now.Add(-time.Minute))
|
|
||||||
|
|
||||||
env.sweeper.ExportSweep(t.Context())
|
|
||||||
|
|
||||||
assert.Empty(t, archivedEventIDs(t, path))
|
|
||||||
assert.Equal(t, []string{sweepRowNew}, archivedEventIDs(t, kept))
|
|
||||||
|
|
||||||
for _, suffix := range archiveFileSuffixes() {
|
|
||||||
assert.NoFileExists(t, emptied+suffix)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRename_MovesEveryFile renames a daily target that also has a
|
|
||||||
// file without a period, and checks every file moves to the new name
|
|
||||||
// with its period, rows and all, and that a later write uses the new
|
|
||||||
// name.
|
|
||||||
func TestRename_MovesEveryFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, "")
|
|
||||||
oldPath := env.archivePath(tgt)
|
|
||||||
day := time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
unrotated := env.deliverReceivedAt(t, tgt, day)
|
|
||||||
|
|
||||||
tgt.Config = dailyConfig
|
|
||||||
first := env.deliverReceivedAt(t, tgt, day)
|
|
||||||
second := env.deliverReceivedAt(t, tgt, day.Add(24*time.Hour))
|
|
||||||
|
|
||||||
require.NoError(t, env.eng.Rename(tgt.ID, "Orders", "Long Term"))
|
|
||||||
|
|
||||||
newPath := filepath.Join(
|
|
||||||
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
|
||||||
)
|
|
||||||
|
|
||||||
for _, old := range []string{
|
|
||||||
oldPath,
|
|
||||||
periodPath(oldPath, dayPeriod),
|
|
||||||
periodPath(oldPath, nextDayPeriod),
|
|
||||||
} {
|
|
||||||
assert.NoFileExists(t, old)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, []string{unrotated}, archivedEventIDs(t, newPath))
|
|
||||||
assert.Equal(t, []string{first},
|
|
||||||
archivedEventIDs(t, periodPath(newPath, dayPeriod)))
|
|
||||||
assert.Equal(t, []string{second},
|
|
||||||
archivedEventIDs(t, periodPath(newPath, nextDayPeriod)))
|
|
||||||
|
|
||||||
third := env.deliverReceivedAt(t, tgt, day.Add(48*time.Hour))
|
|
||||||
assert.Equal(t, []string{third},
|
|
||||||
archivedEventIDs(t, periodPath(newPath, "2026-03-06")))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRename_NeverReplacesARotatedFile plants a file at the new name
|
|
||||||
// of a target's daily file, and proves the rename is refused and moves
|
|
||||||
// none of the target's files.
|
|
||||||
func TestRename_NeverReplacesARotatedFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, dailyConfig)
|
|
||||||
oldPath := env.archivePath(tgt)
|
|
||||||
day := time.Date(2026, 3, 4, 12, 0, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
env.deliverReceivedAt(t, tgt, day)
|
|
||||||
env.deliverReceivedAt(t, tgt, day.Add(24*time.Hour))
|
|
||||||
|
|
||||||
newPath := filepath.Join(
|
|
||||||
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
|
||||||
)
|
|
||||||
planted := periodPath(newPath, nextDayPeriod)
|
|
||||||
require.NoError(t, os.WriteFile(planted, []byte("planted"), 0o600))
|
|
||||||
|
|
||||||
require.ErrorIs(
|
|
||||||
t, env.eng.Rename(tgt.ID, "Orders", "Long Term"),
|
|
||||||
delivery.ErrArchiveNameTaken,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.FileExists(t, periodPath(oldPath, dayPeriod))
|
|
||||||
assert.FileExists(t, periodPath(oldPath, nextDayPeriod))
|
|
||||||
assert.NoFileExists(t, periodPath(newPath, dayPeriod))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestStatArchive_EveryFile proves StatArchive counts and adds up every
|
|
||||||
// one of a target's files, takes the latest write of any of them, and
|
|
||||||
// leaves out files whose names only look like the target's.
|
|
||||||
func TestStatArchive_EveryFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
path := filepath.Join(dir, "archive-wh.db")
|
|
||||||
files := []string{
|
|
||||||
path, periodPath(path, "2026-03"), periodPath(path, hourPeriod),
|
|
||||||
}
|
|
||||||
written := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)
|
|
||||||
|
|
||||||
var size int64
|
|
||||||
|
|
||||||
for i, file := range files {
|
|
||||||
require.NoError(t, os.WriteFile(file, make([]byte, 100*(i+1)), 0o600))
|
|
||||||
|
|
||||||
size += int64(100 * (i + 1))
|
|
||||||
at := written.Add(-time.Duration(i) * time.Hour)
|
|
||||||
require.NoError(t, os.Chtimes(file, at, at))
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, other := range []string{
|
|
||||||
"archive-wh-2026-13.db", "archive-wh-2026-3.db",
|
|
||||||
"archive-wh-other.db", "archive-wh-2026-03.json",
|
|
||||||
"archive-whx.db",
|
|
||||||
} {
|
|
||||||
require.NoError(t,
|
|
||||||
os.WriteFile(filepath.Join(dir, other), []byte("x"), 0o600))
|
|
||||||
}
|
|
||||||
|
|
||||||
got, err := delivery.StatArchive(path)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, len(files), got.Files)
|
|
||||||
assert.Equal(t, size, got.Size)
|
|
||||||
assert.True(t, written.Equal(got.Written), got.Written)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchivePathAt names the file each rotation writes an event to.
|
|
||||||
func TestArchivePathAt(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dataDir := t.TempDir()
|
|
||||||
dbMgr := database.NewTestWebhookDBManager(dataDir)
|
|
||||||
webhook := &database.Webhook{
|
|
||||||
BaseModel: database.BaseModel{ID: "wh-id"}, Name: "Orders",
|
|
||||||
}
|
|
||||||
at := time.Date(2026, 10, 1, 19, 30, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
cases := map[string]string{
|
|
||||||
"": "",
|
|
||||||
`{"rotation":"` + rotationNone + `"}`: "",
|
|
||||||
`{"rotation":"` + rotationMonthly + `"}`: "-2026-10",
|
|
||||||
dailyConfig: "-2026-10-01",
|
|
||||||
`{"rotation":"` + rotationHourly + `"}`: "-2026-10-01-19",
|
|
||||||
}
|
|
||||||
|
|
||||||
for config, period := range cases {
|
|
||||||
target := &database.Target{
|
|
||||||
BaseModel: database.BaseModel{ID: "tgt-id"},
|
|
||||||
Name: "Archive",
|
|
||||||
Config: config,
|
|
||||||
}
|
|
||||||
|
|
||||||
got, err := delivery.ArchivePathAt(dbMgr, webhook, target, at)
|
|
||||||
require.NoError(t, err, config)
|
|
||||||
assert.Equal(t,
|
|
||||||
filepath.Join(
|
|
||||||
dataDir, "archive-orders-archive-tgt-id"+period+".db",
|
|
||||||
),
|
|
||||||
got, config,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := delivery.ArchivePathAt(dbMgr, webhook, &database.Target{
|
|
||||||
Config: weeklyConfig,
|
|
||||||
}, at)
|
|
||||||
require.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestValidateArchiveRotation accepts the four rotations, and empty,
|
|
||||||
// and refuses anything else.
|
|
||||||
func TestValidateArchiveRotation(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, ok := range []string{
|
|
||||||
"", rotationNone, rotationMonthly, rotationDaily, rotationHourly,
|
|
||||||
} {
|
|
||||||
require.NoError(t, delivery.ValidateArchiveRotation(ok), ok)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, bad := range []string{"weekly", "Daily", "hourly "} {
|
|
||||||
require.Error(t, delivery.ValidateArchiveRotation(bad), bad)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -218,7 +218,6 @@ func TestStatArchive(t *testing.T) {
|
|||||||
|
|
||||||
got, err := delivery.StatArchive(path)
|
got, err := delivery.StatArchive(path)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, 1, got.Files)
|
|
||||||
assert.Equal(t, file.Size()+wal.Size(), got.Size)
|
assert.Equal(t, file.Size()+wal.Size(), got.Size)
|
||||||
assert.True(t, written.Equal(got.Written), got.Written)
|
assert.True(t, written.Equal(got.Written), got.Written)
|
||||||
|
|
||||||
@@ -721,41 +720,3 @@ func TestArchiveWriter_RenameMovesBackOnFailure(t *testing.T) {
|
|||||||
assert.NoFileExists(t, filepath.Join(dir, newName))
|
assert.NoFileExists(t, filepath.Join(dir, newName))
|
||||||
assert.Equal(t, oldPath, w.Path())
|
assert.Equal(t, oldPath, w.Path())
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveWriter_RenameMovesBackEveryFile renames a target with a
|
|
||||||
// file without a period and a file for a month, and proves that when
|
|
||||||
// the month's file fails to move, the file already moved is moved back:
|
|
||||||
// both files are under the old name with their rows, and nothing is
|
|
||||||
// under the new name. The new name is 251 bytes, so the file without a
|
|
||||||
// period, with its -wal and -shm, can take it, but the month's file,
|
|
||||||
// eight bytes longer, cannot.
|
|
||||||
func TestArchiveWriter_RenameMovesBackEveryFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
oldPath := filepath.Join(dir, "archive-old.db")
|
|
||||||
monthPath := filepath.Join(dir, "archive-old-2026-03.db")
|
|
||||||
w := delivery.NewExportArchiveWriter(oldPath, archiveTestLogger(), 0)
|
|
||||||
|
|
||||||
require.NoError(t, w.WritePeriod(
|
|
||||||
delivery.ExportArchivedEvent{EventID: "in-none"}, 0, "",
|
|
||||||
))
|
|
||||||
require.NoError(t, w.WritePeriod(
|
|
||||||
delivery.ExportArchivedEvent{EventID: "in-month"}, 0, "2026-03",
|
|
||||||
))
|
|
||||||
|
|
||||||
require.Error(t, w.Rename(strings.Repeat("a", 248)+".db"))
|
|
||||||
|
|
||||||
assert.Equal(t, []string{"in-none"}, archivedEventIDs(t, oldPath))
|
|
||||||
assert.Equal(t, []string{"in-month"}, archivedEventIDs(t, monthPath))
|
|
||||||
|
|
||||||
entries, err := os.ReadDir(dir)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
for _, entry := range entries {
|
|
||||||
assert.True(t, strings.HasPrefix(entry.Name(), "archive-old"),
|
|
||||||
"%s is not under the old name", entry.Name())
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, oldPath, w.Path())
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -204,11 +204,10 @@ func TestNewTargetConfigForm(t *testing.T) {
|
|||||||
|
|
||||||
form, err = delivery.NewTargetConfigForm(&database.Target{
|
form, err = delivery.NewTargetConfigForm(&database.Target{
|
||||||
Type: database.TargetTypeDatabase,
|
Type: database.TargetTypeDatabase,
|
||||||
Config: `{"expiry":"720h","rotation":"daily"}`,
|
Config: `{"expiry":"720h"}`,
|
||||||
})
|
})
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, "720h", form.Expiry)
|
assert.Equal(t, "720h", form.Expiry)
|
||||||
assert.Equal(t, rotationDaily, form.Rotation)
|
|
||||||
|
|
||||||
form, err = delivery.NewTargetConfigForm(&database.Target{
|
form, err = delivery.NewTargetConfigForm(&database.Target{
|
||||||
Type: database.TargetTypeLog,
|
Type: database.TargetTypeLog,
|
||||||
@@ -249,10 +248,6 @@ func TestNewTargetConfigForm_UnreadableConfigErrors(t *testing.T) {
|
|||||||
Type: database.TargetTypeDatabase,
|
Type: database.TargetTypeDatabase,
|
||||||
Config: `{"expiry":"soon"}`,
|
Config: `{"expiry":"soon"}`,
|
||||||
},
|
},
|
||||||
{
|
|
||||||
Type: database.TargetTypeDatabase,
|
|
||||||
Config: weeklyConfig,
|
|
||||||
},
|
|
||||||
{Type: database.TargetType("nope")},
|
{Type: database.TargetType("nope")},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -10,10 +10,10 @@ const (
|
|||||||
tmplKeyArchiveExpiryChoices = "ArchiveExpiryChoices"
|
tmplKeyArchiveExpiryChoices = "ArchiveExpiryChoices"
|
||||||
)
|
)
|
||||||
|
|
||||||
// archiveChoice is one entry of a database target's archive expiry
|
// archiveExpiryChoice is one entry of a database target's archive
|
||||||
// or archive rotation select: the value stored, the label shown, and
|
// expiry select: the expiry stored, the label shown, and whether the
|
||||||
// whether the select starts on it.
|
// select starts on it.
|
||||||
type archiveChoice struct {
|
type archiveExpiryChoice struct {
|
||||||
Value string
|
Value string
|
||||||
Label string
|
Label string
|
||||||
Selected bool
|
Selected bool
|
||||||
@@ -21,8 +21,8 @@ type archiveChoice struct {
|
|||||||
|
|
||||||
// archiveExpiryChoices lists the archive expiries offered by the new
|
// archiveExpiryChoices lists the archive expiries offered by the new
|
||||||
// webhook page, the add target form and the target edit form.
|
// webhook page, the add target form and the target edit form.
|
||||||
func archiveExpiryChoices() []archiveChoice {
|
func archiveExpiryChoices() []archiveExpiryChoice {
|
||||||
return []archiveChoice{
|
return []archiveExpiryChoice{
|
||||||
{Value: archiveExpiryNever, Label: archiveExpiryNever},
|
{Value: archiveExpiryNever, Label: archiveExpiryNever},
|
||||||
{Value: "1h", Label: "1h"},
|
{Value: "1h", Label: "1h"},
|
||||||
{Value: "12h", Label: "12h"},
|
{Value: "12h", Label: "12h"},
|
||||||
@@ -37,7 +37,7 @@ func archiveExpiryChoices() []archiveChoice {
|
|||||||
// empty expiry selects never. An expiry that is not one of the
|
// empty expiry selects never. An expiry that is not one of the
|
||||||
// choices comes first as its own selected entry, so saving the form
|
// choices comes first as its own selected entry, so saving the form
|
||||||
// unchanged keeps it.
|
// unchanged keeps it.
|
||||||
func archiveExpiryOptions(expiry string) []archiveChoice {
|
func archiveExpiryOptions(expiry string) []archiveExpiryChoice {
|
||||||
if expiry == "" {
|
if expiry == "" {
|
||||||
expiry = archiveExpiryNever
|
expiry = archiveExpiryNever
|
||||||
}
|
}
|
||||||
@@ -52,7 +52,7 @@ func archiveExpiryOptions(expiry string) []archiveChoice {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
own := archiveChoice{Value: expiry, Label: expiry, Selected: true}
|
own := archiveExpiryChoice{Value: expiry, Label: expiry, Selected: true}
|
||||||
|
|
||||||
return append([]archiveChoice{own}, options...)
|
return append([]archiveExpiryChoice{own}, options...)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"net/url"
|
"net/url"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
@@ -45,44 +44,26 @@ func expiryShown(
|
|||||||
require.Equal(t, http.StatusOK, w.Code)
|
require.Equal(t, http.StatusOK, w.Code)
|
||||||
|
|
||||||
return matched(
|
return matched(
|
||||||
`Archive expiry:</span>\s*<span>([^<]*)</span>`, w.Body.String(),
|
`Archive Expiry:</span>\s*<span>([^<]*)</span>`, w.Body.String(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// expirySelected returns the target edit page and the expiries its
|
// expirySelected returns the target edit page and the expiries its
|
||||||
// expiry select starts on.
|
// select starts on.
|
||||||
func expirySelected(
|
func expirySelected(
|
||||||
t *testing.T, env *sourceTestEnv, webhookID, targetID string,
|
t *testing.T, env *sourceTestEnv, webhookID, targetID string,
|
||||||
) (string, []string) {
|
) (string, []string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
page := targetEditPage(t, env, webhookID, targetID)
|
|
||||||
|
|
||||||
return page, selectedIn(page, "expiry")
|
|
||||||
}
|
|
||||||
|
|
||||||
// targetEditPage returns a target's edit page.
|
|
||||||
func targetEditPage(
|
|
||||||
t *testing.T, env *sourceTestEnv, webhookID, targetID string,
|
|
||||||
) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
w := serveTarget(
|
w := serveTarget(
|
||||||
env, http.MethodGet,
|
env, http.MethodGet,
|
||||||
"/hook/"+webhookID+"/targets/"+targetID+"/edit", nil,
|
"/hook/"+webhookID+"/targets/"+targetID+"/edit", nil,
|
||||||
)
|
)
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
require.Equal(t, http.StatusOK, w.Code)
|
||||||
|
|
||||||
return w.Body.String()
|
page := w.Body.String()
|
||||||
}
|
|
||||||
|
|
||||||
// selectedIn returns the values the select named name on page starts
|
return page, matched(`<option value="([^"]*)" selected>`, page)
|
||||||
// on.
|
|
||||||
func selectedIn(page, name string) []string {
|
|
||||||
_, rest, _ := strings.Cut(page, `<select id="`+name+`" name="`+name+`"`)
|
|
||||||
options, _, _ := strings.Cut(rest, "</select>")
|
|
||||||
|
|
||||||
return matched(`<option value="([^"]*)" selected>`, options)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveExpiryChoices adds a database target with each archive
|
// TestArchiveExpiryChoices adds a database target with each archive
|
||||||
|
|||||||
@@ -1,42 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
const (
|
|
||||||
// archiveRotationNone is the archive rotation that keeps a
|
|
||||||
// database target's archive in one file. A stored empty rotation
|
|
||||||
// means the same.
|
|
||||||
archiveRotationNone = "none"
|
|
||||||
|
|
||||||
// tmplKeyArchiveRotationChoices is the template data key for the
|
|
||||||
// entries of a page's archive rotation select.
|
|
||||||
tmplKeyArchiveRotationChoices = "ArchiveRotationChoices"
|
|
||||||
)
|
|
||||||
|
|
||||||
// archiveRotationChoices lists the archive rotations offered by the
|
|
||||||
// new webhook page, the add target form and the target edit form.
|
|
||||||
func archiveRotationChoices() []archiveChoice {
|
|
||||||
return []archiveChoice{
|
|
||||||
{Value: archiveRotationNone, Label: archiveRotationNone},
|
|
||||||
{Value: "monthly", Label: "monthly"},
|
|
||||||
{Value: "daily", Label: "daily"},
|
|
||||||
{Value: "hourly", Label: "hourly"},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// archiveRotationOptions returns the choices with rotation selected;
|
|
||||||
// an empty rotation, or one that is not a choice, selects none. A
|
|
||||||
// stored rotation is always a choice: the forms refuse any other.
|
|
||||||
func archiveRotationOptions(rotation string) []archiveChoice {
|
|
||||||
options := archiveRotationChoices()
|
|
||||||
|
|
||||||
for i := range options {
|
|
||||||
if options[i].Value == rotation {
|
|
||||||
options[i].Selected = true
|
|
||||||
|
|
||||||
return options
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
options[0].Selected = true
|
|
||||||
|
|
||||||
return options
|
|
||||||
}
|
|
||||||
@@ -1,229 +0,0 @@
|
|||||||
package handlers_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// rotationNone is the archive rotation that keeps one file.
|
|
||||||
const rotationNone = "none"
|
|
||||||
|
|
||||||
// rotationShown returns the archive rotations the webhook page's
|
|
||||||
// target list shows.
|
|
||||||
func rotationShown(
|
|
||||||
t *testing.T, env *sourceTestEnv, webhookID string,
|
|
||||||
) []string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return matched(
|
|
||||||
`Archive rotation:</span>\s*<span>([^<]*)</span>`,
|
|
||||||
renderedPage(t, env, webhookID),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveRotationChoices adds a database target with each archive
|
|
||||||
// rotation the forms offer, and checks that it is stored as chosen,
|
|
||||||
// shown in the target list, and that the target edit form starts on
|
|
||||||
// it. It then edits the target to hourly, keeping its expiry.
|
|
||||||
func TestArchiveRotationChoices(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
|
|
||||||
for _, rotation := range []string{rotationNone, "monthly", "daily", "hourly"} {
|
|
||||||
t.Run(rotation, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
webhook := seedWebhookWithRetention(t, env.db, 30)
|
|
||||||
|
|
||||||
form := url.Values{}
|
|
||||||
form.Set("name", "archive")
|
|
||||||
form.Set("type", string(database.TargetTypeDatabase))
|
|
||||||
form.Set("expiry", "720h")
|
|
||||||
form.Set("rotation", rotation)
|
|
||||||
|
|
||||||
w := serveTarget(
|
|
||||||
env, http.MethodPost, "/hook/"+webhook.ID+"/targets", form,
|
|
||||||
)
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
|
||||||
|
|
||||||
targets := targetsForWebhook(t, env.db, webhook.ID)
|
|
||||||
require.Len(t, targets, 1)
|
|
||||||
assert.JSONEq(t,
|
|
||||||
`{"expiry":"720h","rotation":"`+rotation+`"}`,
|
|
||||||
targets[0].Config,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
[]string{rotation}, rotationShown(t, env, webhook.ID))
|
|
||||||
|
|
||||||
page := targetEditPage(t, env, webhook.ID, targets[0].ID)
|
|
||||||
assert.Equal(t, []string{rotation}, selectedIn(page, "rotation"))
|
|
||||||
|
|
||||||
form = url.Values{}
|
|
||||||
form.Set("name", "archive")
|
|
||||||
form.Set("expiry", "720h")
|
|
||||||
form.Set("rotation", "hourly")
|
|
||||||
|
|
||||||
w = submitTargetEdit(env, webhook.ID, targets[0].ID, form)
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
|
||||||
assert.JSONEq(t,
|
|
||||||
`{"expiry":"720h","rotation":"hourly"}`,
|
|
||||||
storedTarget(t, env, targets[0].ID).Config,
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveRotationEditStartsOnNone checks the edit form of a
|
|
||||||
// database target with no rotation stored starts on none.
|
|
||||||
func TestArchiveRotationEditStartsOnNone(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
webhook := seedWebhookWithRetention(t, env.db, 30)
|
|
||||||
target := seedConfiguredTarget(
|
|
||||||
t, env.db, webhook.ID, database.TargetTypeDatabase, "",
|
|
||||||
)
|
|
||||||
|
|
||||||
page := targetEditPage(t, env, webhook.ID, target.ID)
|
|
||||||
assert.Equal(t, []string{rotationNone}, selectedIn(page, "rotation"))
|
|
||||||
assert.Equal(t, []string{rotationNone}, rotationShown(t, env, webhook.ID))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveRotationRefused proves a rotation that is not one of the
|
|
||||||
// four is refused on the add target form and the target edit form, and
|
|
||||||
// that nothing is stored.
|
|
||||||
func TestArchiveRotationRefused(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
webhook := seedWebhookWithRetention(t, env.db, 30)
|
|
||||||
|
|
||||||
form := url.Values{}
|
|
||||||
form.Set("name", "archive")
|
|
||||||
form.Set("type", string(database.TargetTypeDatabase))
|
|
||||||
form.Set("rotation", "weekly")
|
|
||||||
|
|
||||||
w := serveTarget(
|
|
||||||
env, http.MethodPost, "/hook/"+webhook.ID+"/targets", form,
|
|
||||||
)
|
|
||||||
assert.Equal(t, http.StatusBadRequest, w.Code)
|
|
||||||
assert.Contains(t, w.Body.String(), "Invalid archive rotation")
|
|
||||||
assert.Empty(t, targetsForWebhook(t, env.db, webhook.ID))
|
|
||||||
|
|
||||||
target := seedConfiguredTarget(
|
|
||||||
t, env.db, webhook.ID, database.TargetTypeDatabase,
|
|
||||||
`{"rotation":"daily"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
form.Del("type")
|
|
||||||
|
|
||||||
w = submitTargetEdit(env, webhook.ID, target.ID, form)
|
|
||||||
assert.Equal(t, http.StatusBadRequest, w.Code)
|
|
||||||
assert.Contains(t, w.Body.String(), "Invalid archive rotation")
|
|
||||||
assert.JSONEq(t,
|
|
||||||
`{"rotation":"daily"}`, storedTarget(t, env, target.ID).Config,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceCreateSubmit_ArchiveRotation proves the new webhook
|
|
||||||
// page's archive rotation is stored on the archive target it creates.
|
|
||||||
func TestHandleSourceCreateSubmit_ArchiveRotation(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
|
|
||||||
form := url.Values{}
|
|
||||||
form.Set("name", "rotated")
|
|
||||||
form.Set("archive", "on")
|
|
||||||
form.Set("archive_expiry", "720h")
|
|
||||||
form.Set("archive_rotation", "daily")
|
|
||||||
|
|
||||||
w := submitCreateForm(env, form)
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
|
||||||
|
|
||||||
var webhook database.Webhook
|
|
||||||
|
|
||||||
require.NoError(t, env.db.DB().
|
|
||||||
Where("name = ?", "rotated").First(&webhook).Error)
|
|
||||||
|
|
||||||
targets := targetsForWebhook(t, env.db, webhook.ID)
|
|
||||||
require.Len(t, targets, 1)
|
|
||||||
assert.JSONEq(t,
|
|
||||||
`{"expiry":"720h","rotation":"daily"}`, targets[0].Config,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveFileView_Rotated describes a daily target's archive files
|
|
||||||
// at two times. On a day that has a file, the view names that file;
|
|
||||||
// on the next, before any event, it names the file the next event
|
|
||||||
// will go to, not created yet. Both times the size is of every file
|
|
||||||
// together and the last write the latest of them.
|
|
||||||
func TestArchiveFileView_Rotated(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
webhook := seedWebhookWithRetention(t, env.db, 30)
|
|
||||||
target := seedConfiguredTarget(
|
|
||||||
t, env.db, webhook.ID, database.TargetTypeDatabase,
|
|
||||||
`{"rotation":"daily"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
path := delivery.ArchivePath(env.dbMgr, &webhook, target)
|
|
||||||
stem := strings.TrimSuffix(path, ".db")
|
|
||||||
written := time.Date(2026, 10, 2, 9, 0, 0, 0, time.UTC)
|
|
||||||
|
|
||||||
for i, day := range []string{"2026-10-01", "2026-10-02"} {
|
|
||||||
file := stem + "-" + day + ".db"
|
|
||||||
require.NoError(t, os.WriteFile(file, make([]byte, 1000), 0o600))
|
|
||||||
|
|
||||||
at := written.Add(time.Duration(i-1) * 24 * time.Hour)
|
|
||||||
require.NoError(t, os.Chtimes(file, at, at))
|
|
||||||
}
|
|
||||||
|
|
||||||
view := env.handlers.ArchiveFileViewForTest(
|
|
||||||
&webhook, target, time.Date(2026, 10, 2, 23, 0, 0, 0, time.UTC),
|
|
||||||
)
|
|
||||||
assert.Equal(t, filepath.Base(stem)+"-2026-10-02.db", view.Name)
|
|
||||||
assert.Empty(t, view.Note)
|
|
||||||
assert.Equal(t, 2, view.Files)
|
|
||||||
assert.Equal(t, "2.0 kB", view.Size)
|
|
||||||
assert.Equal(t, "2026-10-02 09:00:00 UTC", view.WrittenUTC)
|
|
||||||
|
|
||||||
view = env.handlers.ArchiveFileViewForTest(
|
|
||||||
&webhook, target, time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC),
|
|
||||||
)
|
|
||||||
assert.Equal(t, filepath.Base(stem)+"-2026-10-03.db", view.Name)
|
|
||||||
assert.Equal(t, "not created yet", view.Note)
|
|
||||||
assert.Equal(t, 2, view.Files)
|
|
||||||
assert.Equal(t, "2.0 kB", view.Size)
|
|
||||||
|
|
||||||
page := targetList(t, renderedPage(t, env, webhook.ID))
|
|
||||||
assert.Contains(t, page, "Archive size: 2.0 kB in 2 files")
|
|
||||||
}
|
|
||||||
|
|
||||||
// renderedPage returns the webhook page.
|
|
||||||
func renderedPage(t *testing.T, env *sourceTestEnv, webhookID string) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
env.handlers.HandleSourceDetail().ServeHTTP(w, getRequest(
|
|
||||||
t, "/hook/"+webhookID, env.cookies,
|
|
||||||
map[string]string{sourceIDParam: webhookID},
|
|
||||||
))
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
|
|
||||||
return w.Body.String()
|
|
||||||
}
|
|
||||||
@@ -268,7 +268,7 @@ func (h *Handlers) rejectLogin(
|
|||||||
)))
|
)))
|
||||||
h.renderLoginError(
|
h.renderLoginError(
|
||||||
w, r,
|
w, r,
|
||||||
"Too many failed sign-in attempts. Please try again later.",
|
"Too many failed login attempts. Please try again later.",
|
||||||
http.StatusTooManyRequests,
|
http.StatusTooManyRequests,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -329,25 +329,15 @@ func replayBody(body string) *string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// redirectToEventLog redirects a replay or resubmit back to the event
|
// redirectToEventLog redirects a replay or resubmit back to the event
|
||||||
// log it was triggered from, carrying the outcome as its notice. A
|
// log it was triggered from, carrying the outcome as its notice.
|
||||||
// Replay form carries the list it was pressed in as show, so a replay
|
|
||||||
// returns to the Failed or Pending list; a Resubmit form carries none,
|
|
||||||
// so a resubmit returns to the full log, where its new event is the
|
|
||||||
// newest.
|
|
||||||
func redirectToEventLog(
|
func redirectToEventLog(
|
||||||
w http.ResponseWriter,
|
w http.ResponseWriter,
|
||||||
r *http.Request,
|
r *http.Request,
|
||||||
webhook database.Webhook,
|
webhook database.Webhook,
|
||||||
code noticeCode,
|
code noticeCode,
|
||||||
) {
|
) {
|
||||||
location := withNotice("/hook/"+webhook.ID+"/events", code)
|
http.Redirect(
|
||||||
|
w, r, withNotice("/hook/"+webhook.ID+"/events", code),
|
||||||
show := r.PostFormValue(showParam)
|
http.StatusSeeOther,
|
||||||
if eventLogStatuses(show) != nil {
|
|
||||||
location += "&" + showParam + "=" + show
|
|
||||||
}
|
|
||||||
|
|
||||||
http.Redirect( //nolint:gosec // show is checked by eventLogStatuses
|
|
||||||
w, r, location, http.StatusSeeOther,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package handlers_test
|
package handlers_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"io"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -472,66 +471,6 @@ func TestHandleDeliveryReplay_RefusesWhileEarlierReplayInFlight(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleDeliveryReplay_ReturnsToTheListItWasPressedIn proves a
|
|
||||||
// Replay pressed in the Failed list carries that list in its form and
|
|
||||||
// returns to it, and that a show value the event log does not know
|
|
||||||
// returns to the full log.
|
|
||||||
func TestHandleDeliveryReplay_ReturnsToTheListItWasPressedIn(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
tgt := seedConfiguredTarget(
|
|
||||||
t, db, wh.ID, database.TargetTypeHTTP,
|
|
||||||
`{"url":"`+replayTargetURL+`"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
_, original := seedFailedDelivery(t, dbMgr, wh.ID, tgt.ID)
|
|
||||||
|
|
||||||
assert.Contains(t, renderSourceLogsPageWithQuery(
|
|
||||||
t, h, sess, wh.ID, "?show=failed",
|
|
||||||
), `name="show" value="failed"`)
|
|
||||||
|
|
||||||
// The second replay is refused, as the first is still queued.
|
|
||||||
for _, tc := range []struct{ show, location string }{
|
|
||||||
{"failed", "/hook/" + wh.ID +
|
|
||||||
"/events?notice=replay-queued&show=failed"},
|
|
||||||
{"made-up", "/hook/" + wh.ID + "/events?notice=replay-in-flight"},
|
|
||||||
} {
|
|
||||||
req := postRequest(
|
|
||||||
"/hook/"+wh.ID+"/deliveries/"+original.ID+"/replay",
|
|
||||||
authenticatedCookies(
|
|
||||||
t, sess, deleteTestUserID, deleteTestUsername,
|
|
||||||
),
|
|
||||||
map[string]string{
|
|
||||||
paramSourceID: wh.ID,
|
|
||||||
paramDeliveryID: original.ID,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
req.Body = io.NopCloser(strings.NewReader("show=" + tc.show))
|
|
||||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
h.HandleDeliveryReplay().ServeHTTP(w, req)
|
|
||||||
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code, tc.show)
|
|
||||||
assert.Equal(t, tc.location, w.Header().Get("Location"), tc.show)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceLogs_RendersReplayControlAndBanner proves the action
|
// TestHandleSourceLogs_RendersReplayControlAndBanner proves the action
|
||||||
// reaches the page it belongs on: a finished delivery renders a POST
|
// reaches the page it belongs on: a finished delivery renders a POST
|
||||||
// form carrying a CSRF token, and the outcome code a refusal redirects
|
// form carrying a CSRF token, and the outcome code a refusal redirects
|
||||||
|
|||||||
@@ -1,169 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// HandleEntrypointCreate handles adding a new entrypoint.
|
|
||||||
func (h *Handlers) HandleEntrypointCreate() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
userID, ok := h.getUserID(r)
|
|
||||||
if !ok {
|
|
||||||
http.Redirect(
|
|
||||||
w, r, "/pages/login", http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
sourceID := chi.URLParam(r, "sourceID")
|
|
||||||
|
|
||||||
var webhook database.Webhook
|
|
||||||
|
|
||||||
err := h.db.DB().Where(
|
|
||||||
"id = ? AND user_id = ?", sourceID, userID,
|
|
||||||
).First(&webhook).Error
|
|
||||||
if err != nil {
|
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// The body size cap is enforced by the MaxBodySize
|
|
||||||
// middleware, which runs before CSRF parses the form.
|
|
||||||
err = r.ParseForm()
|
|
||||||
if err != nil {
|
|
||||||
h.renderError(w, r, http.StatusBadRequest)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
description := r.PostFormValue("description")
|
|
||||||
|
|
||||||
entrypoint := &database.Entrypoint{
|
|
||||||
WebhookID: webhook.ID,
|
|
||||||
Path: uuid.New().String(),
|
|
||||||
Description: description,
|
|
||||||
Active: true,
|
|
||||||
}
|
|
||||||
|
|
||||||
err = h.db.DB().Create(entrypoint).Error
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, r, "failed to create entrypoint", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
http.Redirect(
|
|
||||||
w, r, withNotice("/hook/"+webhook.ID, entrypointAdded),
|
|
||||||
http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleEntrypointEdit handles changing an entrypoint's description.
|
|
||||||
// It writes only the description column, so the entrypoint keeps its
|
|
||||||
// URL, and an activate or deactivate saved since the page was shown
|
|
||||||
// is not undone.
|
|
||||||
func (h *Handlers) HandleEntrypointEdit() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
userID, ok := h.getUserID(r)
|
|
||||||
if !ok {
|
|
||||||
http.Redirect(
|
|
||||||
w, r, "/pages/login", http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
sourceID := chi.URLParam(r, "sourceID")
|
|
||||||
entrypointID := chi.URLParam(r, "entrypointID")
|
|
||||||
|
|
||||||
var webhook database.Webhook
|
|
||||||
|
|
||||||
err := h.db.DB().Where(
|
|
||||||
"id = ? AND user_id = ?", sourceID, userID,
|
|
||||||
).First(&webhook).Error
|
|
||||||
if err != nil {
|
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// The body size cap is enforced by the MaxBodySize
|
|
||||||
// middleware, which runs before CSRF parses the form.
|
|
||||||
err = r.ParseForm()
|
|
||||||
if err != nil {
|
|
||||||
h.renderError(w, r, http.StatusBadRequest)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
result := h.db.DB().Model(&database.Entrypoint{}).Where(
|
|
||||||
"id = ? AND webhook_id = ?", entrypointID, webhook.ID,
|
|
||||||
).Update("description", r.PostFormValue("description"))
|
|
||||||
if result.Error != nil {
|
|
||||||
h.serverError(
|
|
||||||
w, r, "failed to edit entrypoint", result.Error,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// The id came from the URL and may name another webhook's
|
|
||||||
// entrypoint, which this webhook does not have.
|
|
||||||
if result.RowsAffected == 0 {
|
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
http.Redirect(
|
|
||||||
w, r, withNotice("/hook/"+webhook.ID, entrypointSaved),
|
|
||||||
http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleEntrypointDelete handles deleting an entrypoint.
|
|
||||||
func (h *Handlers) HandleEntrypointDelete() http.HandlerFunc {
|
|
||||||
return h.deleteChildResource(
|
|
||||||
"entrypointID", &database.Entrypoint{},
|
|
||||||
"failed to delete entrypoint",
|
|
||||||
nil,
|
|
||||||
entrypointDeleted,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleEntrypointToggle handles toggling an entrypoint's
|
|
||||||
// active state.
|
|
||||||
func (h *Handlers) HandleEntrypointToggle() http.HandlerFunc {
|
|
||||||
return h.toggleChildResource(
|
|
||||||
"entrypointID",
|
|
||||||
func(webhookID, childID string) (bool, error) {
|
|
||||||
var ep database.Entrypoint
|
|
||||||
|
|
||||||
err := h.db.DB().Where(
|
|
||||||
"id = ? AND webhook_id = ?",
|
|
||||||
childID, webhookID,
|
|
||||||
).First(&ep).Error
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Only the active column: saving the whole row would
|
|
||||||
// write back the description read above over an edit
|
|
||||||
// saved since.
|
|
||||||
active := !ep.Active
|
|
||||||
|
|
||||||
return active, h.db.DB().Model(&ep).
|
|
||||||
Update("active", active).Error
|
|
||||||
},
|
|
||||||
"failed to toggle entrypoint",
|
|
||||||
entrypointActivated, entrypointDeactivated,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
package handlers
|
package handlers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"math"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
"github.com/go-chi/chi"
|
||||||
@@ -60,9 +59,8 @@ func (h *Handlers) HandleEventDetail() http.HandlerFunc {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// The page shows every request header.
|
|
||||||
views, ok := h.eventLogViews(
|
views, ok := h.eventLogViews(
|
||||||
w, r, webhookDB, webhook.ID, rows, targets, math.MaxInt,
|
w, r, webhookDB, webhook.ID, rows, targets,
|
||||||
)
|
)
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -1,620 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"slices"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// DeliveryView is the display-safe projection of a delivery
|
|
||||||
// for the event log page. Its target is a TargetView, so the
|
|
||||||
// stored configuration blob — which holds the target's
|
|
||||||
// credential — has no path to the template.
|
|
||||||
type DeliveryView struct {
|
|
||||||
ID string
|
|
||||||
Status database.DeliveryStatus
|
|
||||||
Target delivery.TargetView
|
|
||||||
|
|
||||||
// Replay is set on a delivery the Replay action created.
|
|
||||||
Replay bool
|
|
||||||
|
|
||||||
// Created is how long ago the delivery was created, and
|
|
||||||
// CreatedUTC the full timestamp the page shows on hover.
|
|
||||||
Created string
|
|
||||||
CreatedUTC string
|
|
||||||
|
|
||||||
// Results is this delivery's attempts in attempt order,
|
|
||||||
// bounded by maxRenderedAttempts. Without them a failure
|
|
||||||
// renders as the status word alone and says nothing about
|
|
||||||
// why.
|
|
||||||
Results []DeliveryResultView
|
|
||||||
|
|
||||||
// AttemptCount is how many attempts were recorded, which
|
|
||||||
// is more than len(Results) once the middle was dropped.
|
|
||||||
AttemptCount int
|
|
||||||
|
|
||||||
// AttemptsOmitted is how many attempts were dropped from
|
|
||||||
// the middle of Results. The page must show it, or the
|
|
||||||
// bound would hide history rather than fold it.
|
|
||||||
AttemptsOmitted int
|
|
||||||
|
|
||||||
// Paused is set while the delivery is retrying and its
|
|
||||||
// target's circuit breaker is open, and nil otherwise.
|
|
||||||
Paused *PausedView
|
|
||||||
}
|
|
||||||
|
|
||||||
// eventLogTarget is what the event log needs to know about
|
|
||||||
// one target: the display-safe view its template renders, and
|
|
||||||
// the redactor that keeps that target's own credential out of
|
|
||||||
// the text its remote peer chose. The two are kept together
|
|
||||||
// so a caller cannot pick up one without the other, and apart
|
|
||||||
// from TargetView so the secrets never reach a template.
|
|
||||||
type eventLogTarget struct {
|
|
||||||
View delivery.TargetView
|
|
||||||
Redactor delivery.Redactor
|
|
||||||
}
|
|
||||||
|
|
||||||
// The event log's show query parameter and its two values: the events
|
|
||||||
// with a failed delivery, and those with a delivery still pending or
|
|
||||||
// retrying.
|
|
||||||
const (
|
|
||||||
showParam = "show"
|
|
||||||
showFailed = "failed"
|
|
||||||
showPending = "pending"
|
|
||||||
)
|
|
||||||
|
|
||||||
// HandleSourceLogs shows the request/response logs for a
|
|
||||||
// webhook.
|
|
||||||
func (h *Handlers) HandleSourceLogs() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
webhook, ok := h.ownedWebhook(w, r)
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
targets, err := h.loadTargetMap(webhook.ID)
|
|
||||||
if err != nil {
|
|
||||||
// Without the map every delivery renders through a
|
|
||||||
// zero redactor, so failing the page is the only
|
|
||||||
// safe answer.
|
|
||||||
h.serverError(w, r, "failed to load targets", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Any other value of show lists every event, as no value
|
|
||||||
// does.
|
|
||||||
show := r.URL.Query().Get(showParam)
|
|
||||||
|
|
||||||
statuses := eventLogStatuses(show)
|
|
||||||
if statuses == nil {
|
|
||||||
show = ""
|
|
||||||
}
|
|
||||||
|
|
||||||
evts, total, ok := h.loadEventsWithDeliveries(
|
|
||||||
w, r, webhook, targets, statuses,
|
|
||||||
)
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
failed, pending, err := h.countFailedAndPendingEvents(webhook.ID)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, r, "failed to count events", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
data := map[string]any{
|
|
||||||
tmplKeyWebhook: &webhook,
|
|
||||||
"Events": evts,
|
|
||||||
"TotalEvents": total,
|
|
||||||
"Show": show,
|
|
||||||
"FailedEvents": failed,
|
|
||||||
"PendingEvents": pending,
|
|
||||||
}
|
|
||||||
|
|
||||||
h.renderTemplate(w, r, "source_logs.html", data)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadTargetMap loads targets into a map of display-safe
|
|
||||||
// views keyed by target ID, each paired with its redactor.
|
|
||||||
// The projection happens here so that no caller can hand a
|
|
||||||
// raw target, configuration blob and all, to a template: the
|
|
||||||
// raw rows do not leave this function.
|
|
||||||
//
|
|
||||||
// The load is Unscoped because deleting a target only soft
|
|
||||||
// deletes the row while its deliveries survive in the
|
|
||||||
// per-webhook database. Both halves of the map need those rows:
|
|
||||||
// a scoped load leaves an old delivery with a zero redactor,
|
|
||||||
// which renders its response bodies unredacted, and with a zero
|
|
||||||
// view, which renders its target as a blank name.
|
|
||||||
//
|
|
||||||
// This map is historical display only. It is built for the event
|
|
||||||
// log and an event's own page, and reaches nothing but
|
|
||||||
// DeliveryView.Target: the target list on the source detail page,
|
|
||||||
// the edit form and the replay path each resolve targets
|
|
||||||
// themselves, and a deleted row is refused there as before.
|
|
||||||
func (h *Handlers) loadTargetMap(
|
|
||||||
webhookID string,
|
|
||||||
) (map[string]eventLogTarget, error) {
|
|
||||||
var targets []database.Target
|
|
||||||
|
|
||||||
err := h.db.DB().Unscoped().Where(
|
|
||||||
"webhook_id = ?", webhookID,
|
|
||||||
).Find(&targets).Error
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
targetMap := make(
|
|
||||||
map[string]eventLogTarget, len(targets),
|
|
||||||
)
|
|
||||||
|
|
||||||
for i := range targets {
|
|
||||||
targetMap[targets[i].ID] = eventLogTarget{
|
|
||||||
Redactor: delivery.NewRedactor(&targets[i]),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The views come from NewTargetViews rather than being
|
|
||||||
// rebuilt here, so the masking rules stay in one place and a
|
|
||||||
// deleted target's configuration is masked by the same code
|
|
||||||
// that masks a live one's.
|
|
||||||
for _, v := range delivery.NewTargetViews(targets) {
|
|
||||||
entry := targetMap[v.ID]
|
|
||||||
entry.View = v
|
|
||||||
targetMap[v.ID] = entry
|
|
||||||
}
|
|
||||||
|
|
||||||
return targetMap, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadEventsWithDeliveries loads the recentEventLimit newest events
|
|
||||||
// and their deliveries from the per-webhook database, and the total
|
|
||||||
// number of events stored. Given delivery statuses, both cover only
|
|
||||||
// the events with a delivery in one of them. Events come back as
|
|
||||||
// capped projections rather than database.Event rows: see
|
|
||||||
// eventLogColumns for why the cut happens in SQL.
|
|
||||||
//
|
|
||||||
// The bool reports whether the load succeeded. It is false
|
|
||||||
// once this has answered the request with an error, and the
|
|
||||||
// caller must then render nothing further.
|
|
||||||
func (h *Handlers) loadEventsWithDeliveries(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
r *http.Request,
|
|
||||||
webhook database.Webhook,
|
|
||||||
targetMap map[string]eventLogTarget,
|
|
||||||
statuses []database.DeliveryStatus,
|
|
||||||
) ([]EventLogView, int64, bool) {
|
|
||||||
if !h.dbMgr.DBExists(webhook.ID) {
|
|
||||||
return nil, 0, true
|
|
||||||
}
|
|
||||||
|
|
||||||
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(
|
|
||||||
w, r, "failed to get webhook database", err,
|
|
||||||
)
|
|
||||||
|
|
||||||
return nil, 0, false
|
|
||||||
}
|
|
||||||
|
|
||||||
rows, totalEvents, err := loadEventLogRows(
|
|
||||||
webhookDB, webhook.ID, statuses,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, r, "failed to load events", err)
|
|
||||||
|
|
||||||
return nil, 0, false
|
|
||||||
}
|
|
||||||
|
|
||||||
result, ok := h.eventLogViews(
|
|
||||||
w, r, webhookDB, webhook.ID, rows, targetMap,
|
|
||||||
maxRenderedBodyBytes,
|
|
||||||
)
|
|
||||||
|
|
||||||
return result, totalEvents, ok
|
|
||||||
}
|
|
||||||
|
|
||||||
// eventLogViews projects loaded events for rendering, each with
|
|
||||||
// its deliveries, how many times it has been resubmitted and the
|
|
||||||
// entrypoint it arrived at (for a resubmitted copy, the one the
|
|
||||||
// request it copies arrived at), and with its request headers only
|
|
||||||
// when their text holds at most maxHeaderBytes. Like
|
|
||||||
// loadEventsWithDeliveries, it reports false once it has answered
|
|
||||||
// the request with an error.
|
|
||||||
func (h *Handlers) eventLogViews(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
r *http.Request,
|
|
||||||
webhookDB *gorm.DB,
|
|
||||||
webhookID string,
|
|
||||||
rows []eventLogRow,
|
|
||||||
targetMap map[string]eventLogTarget,
|
|
||||||
maxHeaderBytes int,
|
|
||||||
) ([]EventLogView, bool) {
|
|
||||||
result := make([]EventLogView, len(rows))
|
|
||||||
eventDeliveries := make([][]database.Delivery, len(rows))
|
|
||||||
|
|
||||||
var deliveryIDs []string
|
|
||||||
|
|
||||||
eventIDs := make([]string, len(rows))
|
|
||||||
|
|
||||||
for i := range rows {
|
|
||||||
result[i] = rows[i].view(webhookID, maxHeaderBytes)
|
|
||||||
eventIDs[i] = rows[i].ID
|
|
||||||
|
|
||||||
webhookDB.Where(
|
|
||||||
"event_id = ?", rows[i].ID,
|
|
||||||
).Find(&eventDeliveries[i])
|
|
||||||
|
|
||||||
for j := range eventDeliveries[i] {
|
|
||||||
deliveryIDs = append(
|
|
||||||
deliveryIDs, eventDeliveries[i][j].ID,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
attempts, err := h.loadDeliveryResults(
|
|
||||||
webhookDB, deliveryIDs,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(
|
|
||||||
w, r, "failed to load delivery attempts", err,
|
|
||||||
)
|
|
||||||
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
resubmits, err := resubmitCounts(webhookDB, eventIDs)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(
|
|
||||||
w, r, "failed to count event resubmissions", err,
|
|
||||||
)
|
|
||||||
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
entrypoints, err := h.entrypointNames(webhookID)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, r, "failed to load entrypoints", err)
|
|
||||||
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := range rows {
|
|
||||||
result[i].Deliveries = h.newDeliveryViews(
|
|
||||||
eventDeliveries[i], targetMap, attempts,
|
|
||||||
)
|
|
||||||
result[i].ResubmitCount = resubmits[rows[i].ID]
|
|
||||||
|
|
||||||
name, ok := entrypoints[rows[i].EntrypointID]
|
|
||||||
if !ok {
|
|
||||||
name = "deleted entrypoint"
|
|
||||||
}
|
|
||||||
|
|
||||||
result[i].Entrypoint = name
|
|
||||||
}
|
|
||||||
|
|
||||||
return result, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadEventLogRows reads the event log projection of the
|
|
||||||
// recentEventLimit newest events, newest first, and the total number
|
|
||||||
// of events stored, both narrowed by statuses as eventsWithStatus
|
|
||||||
// narrows them.
|
|
||||||
func loadEventLogRows(
|
|
||||||
webhookDB *gorm.DB,
|
|
||||||
webhookID string,
|
|
||||||
statuses []database.DeliveryStatus,
|
|
||||||
) ([]eventLogRow, int64, error) {
|
|
||||||
totalEvents, err := countEventsWithStatus(
|
|
||||||
webhookDB, webhookID, statuses,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return nil, 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
var rows []eventLogRow
|
|
||||||
|
|
||||||
err = eventsWithStatus(webhookDB, webhookID, statuses).Select(
|
|
||||||
eventLogColumns, maxRenderedBodyBytes, maxRenderedBodyBytes,
|
|
||||||
).Order("created_at DESC").Limit(recentEventLimit).Find(&rows).Error
|
|
||||||
|
|
||||||
return rows, totalEvents, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// eventLogStatuses returns the delivery statuses the event log's show
|
|
||||||
// value lists events by, or nil for one that lists every event.
|
|
||||||
func eventLogStatuses(show string) []database.DeliveryStatus {
|
|
||||||
switch show {
|
|
||||||
case showFailed:
|
|
||||||
return []database.DeliveryStatus{database.DeliveryStatusFailed}
|
|
||||||
case showPending:
|
|
||||||
return []database.DeliveryStatus{
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
database.DeliveryStatusRetrying,
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// eventsWithStatus selects the webhook's events, or, given statuses,
|
|
||||||
// the recentEventLimit newest of those with at least one delivery in
|
|
||||||
// one of them.
|
|
||||||
//
|
|
||||||
// Given statuses, its cost follows the matching deliveries. SQLite
|
|
||||||
// never reorders a CROSS JOIN, so it reads each join's left side
|
|
||||||
// first: the distinct event IDs of the matching deliveries, through
|
|
||||||
// idx_deliveries_status; then each of those events by ID, sorted to
|
|
||||||
// keep the newest; then the rows of only the events kept, so no other
|
|
||||||
// event's body is read. With a plain "id IN (matching deliveries)"
|
|
||||||
// condition instead, SQLite, which keeps no statistics on these
|
|
||||||
// tables, walks every event newest first.
|
|
||||||
func eventsWithStatus(
|
|
||||||
webhookDB *gorm.DB,
|
|
||||||
webhookID string,
|
|
||||||
statuses []database.DeliveryStatus,
|
|
||||||
) *gorm.DB {
|
|
||||||
if statuses == nil {
|
|
||||||
return webhookDB.Model(&database.Event{}).Where(
|
|
||||||
"webhook_id = ?", webhookID,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
matching := webhookDB.Model(&database.Delivery{}).
|
|
||||||
Distinct("event_id").Where("status IN ?", statuses)
|
|
||||||
|
|
||||||
newest := webhookDB.Table("(?) AS matching", matching).
|
|
||||||
Joins("CROSS JOIN events ON events.id = matching.event_id").
|
|
||||||
Where(
|
|
||||||
"events.webhook_id = ? AND events.deleted_at IS NULL",
|
|
||||||
webhookID,
|
|
||||||
).
|
|
||||||
Order("events.created_at DESC").Limit(recentEventLimit).
|
|
||||||
Select("events.id AS event_id")
|
|
||||||
|
|
||||||
return webhookDB.Table("(?) AS newest", newest).
|
|
||||||
Joins("CROSS JOIN events ON events.id = newest.event_id")
|
|
||||||
}
|
|
||||||
|
|
||||||
// countEventsWithStatus counts the webhook's events with at least one
|
|
||||||
// delivery in one of the statuses, or every event when statuses is
|
|
||||||
// nil. Given statuses, it counts the distinct events of the matching
|
|
||||||
// deliveries and reads nothing but those deliveries, through
|
|
||||||
// idx_deliveries_status, where counting the events would read every
|
|
||||||
// event row. That is the same number, because retention deletes an
|
|
||||||
// event's deliveries with it.
|
|
||||||
func countEventsWithStatus(
|
|
||||||
webhookDB *gorm.DB,
|
|
||||||
webhookID string,
|
|
||||||
statuses []database.DeliveryStatus,
|
|
||||||
) (int64, error) {
|
|
||||||
var count int64
|
|
||||||
|
|
||||||
if statuses == nil {
|
|
||||||
err := webhookDB.Model(&database.Event{}).Where(
|
|
||||||
"webhook_id = ?", webhookID,
|
|
||||||
).Count(&count).Error
|
|
||||||
|
|
||||||
return count, err
|
|
||||||
}
|
|
||||||
|
|
||||||
err := webhookDB.Model(&database.Delivery{}).Distinct("event_id").
|
|
||||||
Where("status IN ?", statuses).Count(&count).Error
|
|
||||||
|
|
||||||
return count, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// countFailedAndPendingEvents returns how many of the webhook's events
|
|
||||||
// the event log lists when it shows only those with a failed delivery,
|
|
||||||
// and when it shows only those with a delivery pending or retrying.
|
|
||||||
func (h *Handlers) countFailedAndPendingEvents(
|
|
||||||
webhookID string,
|
|
||||||
) (int64, int64, error) {
|
|
||||||
if !h.dbMgr.DBExists(webhookID) {
|
|
||||||
return 0, 0, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
webhookDB, err := h.dbMgr.GetDB(webhookID)
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
failed, err := countEventsWithStatus(
|
|
||||||
webhookDB, webhookID, eventLogStatuses(showFailed),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
pending, err := countEventsWithStatus(
|
|
||||||
webhookDB, webhookID, eventLogStatuses(showPending),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return failed, pending, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// resubmitCounts reports, for each of the page's events, how many
|
|
||||||
// events have been resubmitted from it.
|
|
||||||
//
|
|
||||||
// One grouped query covers the page rather than one query per event.
|
|
||||||
// The page shows at most recentEventLimit events, far below SQLite's
|
|
||||||
// bound parameter ceiling, so it needs no chunking as the delivery
|
|
||||||
// result load does.
|
|
||||||
func resubmitCounts(
|
|
||||||
webhookDB *gorm.DB, eventIDs []string,
|
|
||||||
) (map[string]int, error) {
|
|
||||||
counts := make(map[string]int, len(eventIDs))
|
|
||||||
|
|
||||||
if len(eventIDs) == 0 {
|
|
||||||
return counts, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var rows []struct {
|
|
||||||
ResubmittedFromID string
|
|
||||||
Total int
|
|
||||||
}
|
|
||||||
|
|
||||||
err := webhookDB.Model(&database.Event{}).
|
|
||||||
Select("resubmitted_from_id, count(*) AS total").
|
|
||||||
Where("resubmitted_from_id IN ?", eventIDs).
|
|
||||||
Group("resubmitted_from_id").
|
|
||||||
Find(&rows).Error
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, row := range rows {
|
|
||||||
counts[row.ResubmittedFromID] = row.Total
|
|
||||||
}
|
|
||||||
|
|
||||||
return counts, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// deliveryIDChunkSize bounds how many delivery IDs go into one
|
|
||||||
// IN clause. SQLite refuses a statement carrying more than
|
|
||||||
// SQLITE_MAX_VARIABLE_NUMBER (32766) bound parameters, and a
|
|
||||||
// page holds one delivery per target per event, so a webhook
|
|
||||||
// with enough targets would turn the whole query into an error
|
|
||||||
// and the page into zero attempts.
|
|
||||||
const deliveryIDChunkSize = 500
|
|
||||||
|
|
||||||
// loadDeliveryResults loads the recorded attempts for the
|
|
||||||
// page's deliveries, keyed by delivery ID.
|
|
||||||
//
|
|
||||||
// Each response body is cut by SQLite rather than in Go, for
|
|
||||||
// the reason deliveryResultColumns gives. How many attempts a
|
|
||||||
// delivery has is the target's MaxRetries, which the
|
|
||||||
// authenticated operator sets; how many of them reach the page
|
|
||||||
// is bounded again by maxRenderedAttempts.
|
|
||||||
func (h *Handlers) loadDeliveryResults(
|
|
||||||
webhookDB *gorm.DB,
|
|
||||||
deliveryIDs []string,
|
|
||||||
) (map[string][]deliveryResultRow, error) {
|
|
||||||
byDelivery := make(map[string][]deliveryResultRow)
|
|
||||||
|
|
||||||
for chunk := range slices.Chunk(
|
|
||||||
deliveryIDs, deliveryIDChunkSize,
|
|
||||||
) {
|
|
||||||
var rows []deliveryResultRow
|
|
||||||
|
|
||||||
err := webhookDB.Model(
|
|
||||||
&database.DeliveryResult{},
|
|
||||||
).Select(
|
|
||||||
deliveryResultColumns, maxRenderedResponseBytes,
|
|
||||||
).Where(
|
|
||||||
"delivery_id IN ?", chunk,
|
|
||||||
).Order("attempt_num ASC").Find(&rows).Error
|
|
||||||
if err != nil {
|
|
||||||
// Returning what was loaded so far renders the
|
|
||||||
// deliveries in the failed chunk as never having run,
|
|
||||||
// which is indistinguishable from ones that really
|
|
||||||
// never ran. The page fails instead.
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := range rows {
|
|
||||||
byDelivery[rows[i].DeliveryID] = append(
|
|
||||||
byDelivery[rows[i].DeliveryID], rows[i],
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return byDelivery, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// newDeliveryViews projects deliveries for rendering,
|
|
||||||
// resolving each one's target to its display-safe view and
|
|
||||||
// each one's attempts through that target's redactor. A
|
|
||||||
// retrying delivery also reads its target's circuit breaker.
|
|
||||||
func (h *Handlers) newDeliveryViews(
|
|
||||||
deliveries []database.Delivery,
|
|
||||||
targetMap map[string]eventLogTarget,
|
|
||||||
attempts map[string][]deliveryResultRow,
|
|
||||||
) []DeliveryView {
|
|
||||||
views := make([]DeliveryView, len(deliveries))
|
|
||||||
|
|
||||||
for i := range deliveries {
|
|
||||||
target := targetMap[deliveries[i].TargetID]
|
|
||||||
rows := attempts[deliveries[i].ID]
|
|
||||||
created := deliveries[i].CreatedAt
|
|
||||||
|
|
||||||
results, omitted := renderedAttempts(
|
|
||||||
rows, target.Redactor,
|
|
||||||
)
|
|
||||||
|
|
||||||
views[i] = DeliveryView{
|
|
||||||
ID: deliveries[i].ID,
|
|
||||||
Status: deliveries[i].Status,
|
|
||||||
Target: target.View,
|
|
||||||
Replay: deliveries[i].Replay,
|
|
||||||
Created: humanize.Time(created),
|
|
||||||
CreatedUTC: created.UTC().Format(time.DateTime) + " UTC",
|
|
||||||
Results: results,
|
|
||||||
AttemptCount: len(rows),
|
|
||||||
AttemptsOmitted: omitted,
|
|
||||||
}
|
|
||||||
|
|
||||||
if deliveries[i].Status == database.DeliveryStatusRetrying {
|
|
||||||
views[i].Paused = h.deliveryPausedView(
|
|
||||||
deliveries[i].TargetID, rows,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return views
|
|
||||||
}
|
|
||||||
|
|
||||||
// maxRenderedAttempts bounds how many of one delivery's
|
|
||||||
// attempts the page renders. Past it the middle is dropped and
|
|
||||||
// counted, keeping the first attempts and the last ones: how
|
|
||||||
// the delivery started failing and how it ended are what a
|
|
||||||
// reader needs, and the count says plainly that the rest was
|
|
||||||
// dropped rather than never recorded.
|
|
||||||
const (
|
|
||||||
renderedAttemptsHead = 10
|
|
||||||
renderedAttemptsTail = 10
|
|
||||||
maxRenderedAttempts = renderedAttemptsHead +
|
|
||||||
renderedAttemptsTail
|
|
||||||
)
|
|
||||||
|
|
||||||
// renderedAttempts projects a delivery's attempts through the
|
|
||||||
// target's redactor, at most maxRenderedAttempts of them, and
|
|
||||||
// reports how many it dropped.
|
|
||||||
func renderedAttempts(
|
|
||||||
rows []deliveryResultRow,
|
|
||||||
redactor delivery.Redactor,
|
|
||||||
) ([]DeliveryResultView, int) {
|
|
||||||
omitted := 0
|
|
||||||
|
|
||||||
if len(rows) > maxRenderedAttempts {
|
|
||||||
omitted = len(rows) - maxRenderedAttempts
|
|
||||||
|
|
||||||
kept := make(
|
|
||||||
[]deliveryResultRow, 0, maxRenderedAttempts,
|
|
||||||
)
|
|
||||||
kept = append(kept, rows[:renderedAttemptsHead]...)
|
|
||||||
kept = append(
|
|
||||||
kept, rows[len(rows)-renderedAttemptsTail:]...,
|
|
||||||
)
|
|
||||||
rows = kept
|
|
||||||
}
|
|
||||||
|
|
||||||
views := make([]DeliveryResultView, len(rows))
|
|
||||||
for i := range rows {
|
|
||||||
views[i] = rows[i].view(redactor)
|
|
||||||
}
|
|
||||||
|
|
||||||
return views, omitted
|
|
||||||
}
|
|
||||||
@@ -1,15 +1,10 @@
|
|||||||
package handlers
|
package handlers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
"unicode/utf8"
|
"unicode/utf8"
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
"github.com/dustin/go-humanize"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// eventLogColumns is the event log's projection. The casts to
|
// eventLogColumns is the event log's projection. The casts to
|
||||||
@@ -17,20 +12,16 @@ import (
|
|||||||
// bytes rather than characters, so the cap bounds the page in
|
// bytes rather than characters, so the cap bounds the page in
|
||||||
// bytes whatever the payload's encoding. Cutting in SQLite
|
// bytes whatever the payload's encoding. Cutting in SQLite
|
||||||
// rather than in Go is the point of the projection — an
|
// rather than in Go is the point of the projection — an
|
||||||
// oversized body or set of request headers never becomes a Go
|
// oversized body never becomes a Go string at all.
|
||||||
// string at all.
|
|
||||||
const eventLogColumns = "id, created_at, method, content_type, " +
|
const eventLogColumns = "id, created_at, method, content_type, " +
|
||||||
"resubmitted_from_id, entrypoint_id, " +
|
"resubmitted_from_id, " +
|
||||||
"substr(cast(headers as blob), 1, ?) AS headers, " +
|
|
||||||
"length(cast(headers as blob)) AS headers_bytes, " +
|
|
||||||
"substr(cast(body as blob), 1, ?) AS body, " +
|
"substr(cast(body as blob), 1, ?) AS body, " +
|
||||||
"length(cast(body as blob)) AS body_bytes"
|
"length(cast(body as blob)) AS body_bytes"
|
||||||
|
|
||||||
// eventColumns is eventLogColumns for the event's own page, which
|
// eventColumns is eventLogColumns for the event's own page, which
|
||||||
// shows the whole body and every request header.
|
// shows the whole body.
|
||||||
const eventColumns = "id, created_at, method, content_type, " +
|
const eventColumns = "id, created_at, method, content_type, " +
|
||||||
"resubmitted_from_id, entrypoint_id, headers, " +
|
"resubmitted_from_id, " +
|
||||||
"length(cast(headers as blob)) AS headers_bytes, " +
|
|
||||||
"cast(body as blob) AS body, " +
|
"cast(body as blob) AS body, " +
|
||||||
"length(cast(body as blob)) AS body_bytes"
|
"length(cast(body as blob)) AS body_bytes"
|
||||||
|
|
||||||
@@ -49,22 +40,6 @@ type EventLogView struct {
|
|||||||
|
|
||||||
Body BodyView
|
Body BodyView
|
||||||
|
|
||||||
// Entrypoint names the entrypoint the event arrived at. A
|
|
||||||
// resubmitted copy, even a copy of a copy, did not arrive; it
|
|
||||||
// names the one the request it copies arrived at. The name is
|
|
||||||
// the entrypoint's description, "Entrypoint" when it has none,
|
|
||||||
// or "deleted entrypoint", never its URL, which is the
|
|
||||||
// entrypoint's secret.
|
|
||||||
Entrypoint string
|
|
||||||
|
|
||||||
// Headers is the event's request headers as text, one
|
|
||||||
// "Name: value" line per value, sorted by name. HeadersCut
|
|
||||||
// reports headers left out because they hold more than
|
|
||||||
// maxRenderedBodyBytes, stored or as text; only the event log
|
|
||||||
// leaves them out.
|
|
||||||
Headers string
|
|
||||||
HeadersCut bool
|
|
||||||
|
|
||||||
// ResubmittedFromID names the event this one was copied
|
// ResubmittedFromID names the event this one was copied
|
||||||
// from, empty for an event that arrived on the receiver.
|
// from, empty for an event that arrived on the receiver.
|
||||||
ResubmittedFromID string
|
ResubmittedFromID string
|
||||||
@@ -85,35 +60,27 @@ func (v EventLogView) ResubmittedFrom() bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// eventLogRow is one row of the event log projection, or of
|
// eventLogRow is one row of the event log projection, or of
|
||||||
// eventColumns. In the event log its headers and body columns
|
// eventColumns. In the event log its body column arrives
|
||||||
// arrive already cut to the cap by SQLite, each with its true
|
// already cut to the cap by SQLite, with the true size beside
|
||||||
// size beside it.
|
// it.
|
||||||
type eventLogRow struct {
|
type eventLogRow struct {
|
||||||
ID string
|
ID string
|
||||||
CreatedAt time.Time
|
CreatedAt time.Time
|
||||||
Method string
|
Method string
|
||||||
ContentType string
|
ContentType string
|
||||||
ResubmittedFromID *string
|
ResubmittedFromID *string
|
||||||
EntrypointID string
|
|
||||||
Headers string
|
|
||||||
HeadersBytes int64
|
|
||||||
Body []byte
|
Body []byte
|
||||||
BodyBytes int64
|
BodyBytes int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// view projects a loaded row of the webhook's events for
|
// view projects a loaded row of the webhook's events for
|
||||||
// rendering. It shows the request headers when the row holds them
|
// rendering.
|
||||||
// whole and their text holds at most maxHeaderBytes.
|
func (r *eventLogRow) view(webhookID string) EventLogView {
|
||||||
func (r *eventLogRow) view(
|
|
||||||
webhookID string, maxHeaderBytes int,
|
|
||||||
) EventLogView {
|
|
||||||
var from string
|
var from string
|
||||||
if r.ResubmittedFromID != nil {
|
if r.ResubmittedFromID != nil {
|
||||||
from = *r.ResubmittedFromID
|
from = *r.ResubmittedFromID
|
||||||
}
|
}
|
||||||
|
|
||||||
headers, fit := requestHeaderLines(r.Headers, maxHeaderBytes)
|
|
||||||
|
|
||||||
return EventLogView{
|
return EventLogView{
|
||||||
ID: r.ID,
|
ID: r.ID,
|
||||||
Method: r.Method,
|
Method: r.Method,
|
||||||
@@ -123,83 +90,10 @@ func (r *eventLogRow) view(
|
|||||||
Body: newBodyView(
|
Body: newBodyView(
|
||||||
"/hook/"+webhookID+"/events/"+r.ID, r.Body, r.BodyBytes,
|
"/hook/"+webhookID+"/events/"+r.ID, r.Body, r.BodyBytes,
|
||||||
),
|
),
|
||||||
Headers: strings.Join(headers, "\n"),
|
|
||||||
HeadersCut: !fit || r.HeadersBytes > int64(len(r.Headers)),
|
|
||||||
ResubmittedFromID: from,
|
ResubmittedFromID: from,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// requestHeaderLines turns an event's stored request headers, the
|
|
||||||
// JSON the receiver writes, into one "Name: value" line per value,
|
|
||||||
// sorted by name. Headers that do not parse, as when the event log
|
|
||||||
// has cut them, show as none. It reports false, with no lines, when
|
|
||||||
// the lines, each with the newline that follows it, would hold more
|
|
||||||
// than maxBytes: a header sent many times is stored with its name
|
|
||||||
// once but shown with it on every line.
|
|
||||||
func requestHeaderLines(headersJSON string, maxBytes int) ([]string, bool) {
|
|
||||||
var headers http.Header
|
|
||||||
|
|
||||||
if json.Unmarshal([]byte(headersJSON), &headers) != nil {
|
|
||||||
return nil, true
|
|
||||||
}
|
|
||||||
|
|
||||||
names := make([]string, 0, len(headers))
|
|
||||||
for name := range headers {
|
|
||||||
names = append(names, name)
|
|
||||||
}
|
|
||||||
|
|
||||||
slices.Sort(names)
|
|
||||||
|
|
||||||
var lines []string
|
|
||||||
|
|
||||||
size := 0
|
|
||||||
|
|
||||||
for _, name := range names {
|
|
||||||
for _, value := range headers[name] {
|
|
||||||
line := name + ": " + value
|
|
||||||
|
|
||||||
size += len(line) + len("\n")
|
|
||||||
if size > maxBytes {
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
lines = append(lines, line)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return lines, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// entrypointNames maps each of the webhook's entrypoints to the name
|
|
||||||
// an event that arrived at it shows: its description, or "Entrypoint"
|
|
||||||
// when it has none, as the webhook page names it. A deleted
|
|
||||||
// entrypoint is left out.
|
|
||||||
func (h *Handlers) entrypointNames(
|
|
||||||
webhookID string,
|
|
||||||
) (map[string]string, error) {
|
|
||||||
var entrypoints []database.Entrypoint
|
|
||||||
|
|
||||||
err := h.db.DB().Where(
|
|
||||||
"webhook_id = ?", webhookID,
|
|
||||||
).Find(&entrypoints).Error
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
names := make(map[string]string, len(entrypoints))
|
|
||||||
|
|
||||||
for i := range entrypoints {
|
|
||||||
name := entrypoints[i].Description
|
|
||||||
if name == "" {
|
|
||||||
name = "Entrypoint"
|
|
||||||
}
|
|
||||||
|
|
||||||
names[entrypoints[i].ID] = name
|
|
||||||
}
|
|
||||||
|
|
||||||
return names, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// trimPartialRune drops a trailing UTF-8 sequence that the
|
// trimPartialRune drops a trailing UTF-8 sequence that the
|
||||||
// byte-wise cut left incomplete, so a multi-byte rune severed
|
// byte-wise cut left incomplete, so a multi-byte rune severed
|
||||||
// at the cap does not surface as a mojibake tail.
|
// at the cap does not surface as a mojibake tail.
|
||||||
|
|||||||
@@ -1,333 +0,0 @@
|
|||||||
package handlers_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/gorm/clause"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// arrivedAt is how a page names the entrypoint an event arrived at.
|
|
||||||
func arrivedAt(name string) string {
|
|
||||||
return `Arrived at <span class="text-gray-900 wrap-anywhere">` + name +
|
|
||||||
`</span>`
|
|
||||||
}
|
|
||||||
|
|
||||||
// copiedRequestArrivedAt is how a page names, for a resubmitted copy,
|
|
||||||
// the entrypoint the request it copies arrived at.
|
|
||||||
func copiedRequestArrivedAt(name string) string {
|
|
||||||
return `The request it copies arrived at ` +
|
|
||||||
`<span class="text-gray-900 wrap-anywhere">` + name + `</span>`
|
|
||||||
}
|
|
||||||
|
|
||||||
// headerBox is how a page shows an event's request header lines: as
|
|
||||||
// one block of text in a single box.
|
|
||||||
func headerBox(lines ...string) string {
|
|
||||||
return `<pre class="rounded-md border border-gray-200 bg-white p-2 ` +
|
|
||||||
`text-xs text-gray-700 overflow-x-auto whitespace-pre-wrap ` +
|
|
||||||
`break-all">` + strings.Join(lines, "\n") + `</pre>`
|
|
||||||
}
|
|
||||||
|
|
||||||
// showHeadersLink is the event log's link to an event's own page for
|
|
||||||
// request headers it leaves out.
|
|
||||||
func showHeadersLink(webhookID, eventID string) string {
|
|
||||||
return `<a href="/hook/` + webhookID + `/events/` + eventID +
|
|
||||||
`" class="btn-small">Show the request headers</a>`
|
|
||||||
}
|
|
||||||
|
|
||||||
// entrypoint records one of the fixture webhook's entrypoints.
|
|
||||||
func (f *recentEventsFixture) entrypoint(
|
|
||||||
t *testing.T, description string,
|
|
||||||
) *database.Entrypoint {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
ep := &database.Entrypoint{
|
|
||||||
WebhookID: f.webhook.ID,
|
|
||||||
Path: uuid.NewString(),
|
|
||||||
Description: description,
|
|
||||||
Active: true,
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, f.db.DB().Omit(clause.Associations).Create(ep).Error)
|
|
||||||
|
|
||||||
return ep
|
|
||||||
}
|
|
||||||
|
|
||||||
// eventAt records an event that arrived at the entrypoint with the
|
|
||||||
// given request headers, stored as JSON as the receiver stores them.
|
|
||||||
func (f *recentEventsFixture) eventAt(
|
|
||||||
t *testing.T,
|
|
||||||
ep *database.Entrypoint,
|
|
||||||
headersJSON string,
|
|
||||||
receivedAt time.Time,
|
|
||||||
) *database.Event {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
event := &database.Event{
|
|
||||||
WebhookID: f.webhook.ID,
|
|
||||||
EntrypointID: ep.ID,
|
|
||||||
Method: http.MethodPost,
|
|
||||||
Headers: headersJSON,
|
|
||||||
Body: "{}",
|
|
||||||
BodyBytes: 2,
|
|
||||||
ContentType: contentTypeJSON,
|
|
||||||
}
|
|
||||||
event.CreatedAt = receivedAt
|
|
||||||
|
|
||||||
require.NoError(t, f.webhookDB.Omit(
|
|
||||||
clause.Associations,
|
|
||||||
).Create(event).Error)
|
|
||||||
|
|
||||||
return event
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventRequest_EachEventShowsItsOwnEntrypointAndHeaders proves two
|
|
||||||
// events that arrived at two entrypoints each show their own
|
|
||||||
// entrypoint and request headers, in the event log and on their own
|
|
||||||
// pages, with the headers sorted by name, escaped and keeping their
|
|
||||||
// whitespace, and never the entrypoint's URL.
|
|
||||||
func TestEventRequest_EachEventShowsItsOwnEntrypointAndHeaders(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
billing := f.entrypoint(t, "Billing sender")
|
|
||||||
unnamed := f.entrypoint(t, "")
|
|
||||||
|
|
||||||
// Stored in reverse name order.
|
|
||||||
older := f.eventAt(t, billing,
|
|
||||||
`{"X-Shop-Event":["order.created"],`+
|
|
||||||
`"User-Agent":["shop/1 build\t7"],"Accept":["*/*"]}`,
|
|
||||||
time.Now().Add(-time.Minute))
|
|
||||||
newer := f.eventAt(t, unnamed,
|
|
||||||
`{"X-Shop-Event":["order.paid"],"X-Note":["<b>hi</b>"]}`,
|
|
||||||
time.Now())
|
|
||||||
|
|
||||||
olderShows := func(t *testing.T, page string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
assert.Contains(t, page, arrivedAt("Billing sender"))
|
|
||||||
assert.Contains(t, page, headerBox(
|
|
||||||
"Accept: */*",
|
|
||||||
"User-Agent: shop/1 build\t7",
|
|
||||||
"X-Shop-Event: order.created",
|
|
||||||
), "headers are sorted by name")
|
|
||||||
assert.NotContains(t, page, "order.paid")
|
|
||||||
assert.NotContains(t, page, billing.Path)
|
|
||||||
}
|
|
||||||
|
|
||||||
newerShows := func(t *testing.T, page string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
assert.Contains(t, page, arrivedAt("Entrypoint"))
|
|
||||||
assert.Contains(t, page, headerBox(
|
|
||||||
"X-Note: <b>hi</b>",
|
|
||||||
"X-Shop-Event: order.paid",
|
|
||||||
))
|
|
||||||
assert.NotContains(t, page, "<b>hi</b>")
|
|
||||||
assert.NotContains(t, page, "order.created")
|
|
||||||
assert.NotContains(t, page, unnamed.Path)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The log lists the newer event first, so everything between
|
|
||||||
// the two events' first mentions belongs to the newer one.
|
|
||||||
_, rest, found := strings.Cut(renderSourceLogsPage(
|
|
||||||
t, f.h, f.sess, f.webhook.ID,
|
|
||||||
), newer.ID)
|
|
||||||
require.True(t, found)
|
|
||||||
|
|
||||||
newerPart, olderPart, found := strings.Cut(rest, older.ID)
|
|
||||||
require.True(t, found)
|
|
||||||
|
|
||||||
newerShows(t, newerPart)
|
|
||||||
olderShows(t, olderPart)
|
|
||||||
|
|
||||||
w := serveEventPage(t, f.h, f.sess, f.webhook.ID, newer.ID)
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
newerShows(t, w.Body.String())
|
|
||||||
|
|
||||||
w = serveEventPage(t, f.h, f.sess, f.webhook.ID, older.ID)
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
olderShows(t, w.Body.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventRequest_DeletedEntrypoint proves an event whose entrypoint
|
|
||||||
// has since been deleted says so in the event log and on its own page.
|
|
||||||
func TestEventRequest_DeletedEntrypoint(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
ep := f.entrypoint(t, "Retired sender")
|
|
||||||
event := f.eventAt(t, ep, `{}`, time.Now())
|
|
||||||
|
|
||||||
require.NoError(t, f.db.DB().Delete(ep).Error)
|
|
||||||
|
|
||||||
page := renderSourceLogsPage(t, f.h, f.sess, f.webhook.ID)
|
|
||||||
assert.Contains(t, page, arrivedAt("deleted entrypoint"))
|
|
||||||
assert.NotContains(t, page, "Retired sender")
|
|
||||||
|
|
||||||
w := serveEventPage(t, f.h, f.sess, f.webhook.ID, event.ID)
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
assert.Contains(t, w.Body.String(), arrivedAt("deleted entrypoint"))
|
|
||||||
assert.NotContains(t, w.Body.String(), "Retired sender")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventRequest_ResubmittedCopy proves a resubmitted copy and a copy
|
|
||||||
// of that copy each say the request they copy arrived at the
|
|
||||||
// entrypoint, in the event log and on their own pages, and never that
|
|
||||||
// they did.
|
|
||||||
func TestEventRequest_ResubmittedCopy(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
ep := f.entrypoint(t, "Billing sender")
|
|
||||||
original := f.eventAt(t, ep, `{}`, time.Now().Add(-2*time.Minute))
|
|
||||||
copied := f.eventAt(t, ep, `{}`, time.Now().Add(-time.Minute))
|
|
||||||
copyOfCopy := f.eventAt(t, ep, `{}`, time.Now())
|
|
||||||
|
|
||||||
require.NoError(t, f.webhookDB.Model(copied).Update(
|
|
||||||
"resubmitted_from_id", original.ID,
|
|
||||||
).Error)
|
|
||||||
require.NoError(t, f.webhookDB.Model(copyOfCopy).Update(
|
|
||||||
"resubmitted_from_id", copied.ID,
|
|
||||||
).Error)
|
|
||||||
|
|
||||||
// The log lists the newest event first, and each event's Resubmit
|
|
||||||
// form comes before its entrypoint, so cutting the page at the
|
|
||||||
// copy's and the original's forms leaves each event's entrypoint
|
|
||||||
// in its own part.
|
|
||||||
copyOfCopyPart, rest, found := strings.Cut(
|
|
||||||
renderSourceLogsPage(t, f.h, f.sess, f.webhook.ID),
|
|
||||||
"/events/"+copied.ID+"/resubmit",
|
|
||||||
)
|
|
||||||
require.True(t, found)
|
|
||||||
|
|
||||||
copyPart, originalPart, found := strings.Cut(
|
|
||||||
rest, "/events/"+original.ID+"/resubmit",
|
|
||||||
)
|
|
||||||
require.True(t, found)
|
|
||||||
|
|
||||||
for _, part := range []string{copyOfCopyPart, copyPart} {
|
|
||||||
assert.Contains(t, part, copiedRequestArrivedAt("Billing sender"))
|
|
||||||
assert.NotContains(t, part, arrivedAt("Billing sender"))
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Contains(t, originalPart, arrivedAt("Billing sender"))
|
|
||||||
assert.NotContains(t, originalPart,
|
|
||||||
copiedRequestArrivedAt("Billing sender"))
|
|
||||||
|
|
||||||
for _, event := range []*database.Event{copied, copyOfCopy} {
|
|
||||||
w := serveEventPage(t, f.h, f.sess, f.webhook.ID, event.ID)
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
assert.Contains(t, w.Body.String(),
|
|
||||||
copiedRequestArrivedAt("Billing sender"))
|
|
||||||
assert.NotContains(t, w.Body.String(), arrivedAt("Billing sender"))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventRequest_HeadersOverTheLimit proves the event log leaves out
|
|
||||||
// request headers that hold more than it shows of a body, whether
|
|
||||||
// stored or as lines, and links to the event's own page, which shows
|
|
||||||
// them all.
|
|
||||||
func TestEventRequest_HeadersOverTheLimit(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// The receiver stores each "<" as six bytes of JSON, so this
|
|
||||||
// header is over the limit stored but not as a line.
|
|
||||||
const lessThans = bodyCap/6 + 1
|
|
||||||
|
|
||||||
// A header sent many times is stored with its name once, and
|
|
||||||
// shown with it on every line.
|
|
||||||
repeatedName := "X-Repeated-" + strings.Repeat("r", 1000)
|
|
||||||
|
|
||||||
tests := map[string]struct {
|
|
||||||
headers http.Header
|
|
||||||
line string
|
|
||||||
}{
|
|
||||||
"stored": {
|
|
||||||
headers: http.Header{"X-Long": {strings.Repeat("<", lessThans)}},
|
|
||||||
line: "X-Long: " + strings.Repeat("<", lessThans),
|
|
||||||
},
|
|
||||||
"as lines": {
|
|
||||||
headers: http.Header{repeatedName: slices.Repeat([]string{""}, 41)},
|
|
||||||
line: repeatedName + ": ",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, tc := range tests {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
headersJSON, err := json.Marshal(tc.headers)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
ep := f.entrypoint(t, "Billing sender")
|
|
||||||
event := f.eventAt(t, ep, string(headersJSON), time.Now())
|
|
||||||
|
|
||||||
page := renderSourceLogsPage(t, f.h, f.sess, f.webhook.ID)
|
|
||||||
assert.Contains(t, page, showHeadersLink(f.webhook.ID, event.ID))
|
|
||||||
assert.NotContains(t, page, tc.line)
|
|
||||||
assert.Less(t, len(page), 4*bodyCap)
|
|
||||||
|
|
||||||
w := serveEventPage(t, f.h, f.sess, f.webhook.ID, event.ID)
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
assert.Contains(t, w.Body.String(), tc.line)
|
|
||||||
assert.NotContains(t, w.Body.String(), "Show the request headers")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestEventRequest_ManyShortHeaderLines proves that for many short
|
|
||||||
// request header lines the event log writes no more than its limit,
|
|
||||||
// apart from escaping: lines that fill the limit show as one block of
|
|
||||||
// text, and one line more is left out with a link to the event's own
|
|
||||||
// page.
|
|
||||||
func TestEventRequest_ManyShortHeaderLines(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Each "A: " line and the newline after it hold four bytes, so
|
|
||||||
// this many lines fill the limit exactly. Each line in its own
|
|
||||||
// element would make the page many times the limit.
|
|
||||||
const fill = bodyCap / len("A: \n")
|
|
||||||
|
|
||||||
tests := map[string]struct {
|
|
||||||
lines int
|
|
||||||
shown bool
|
|
||||||
}{
|
|
||||||
"filling the limit": {lines: fill, shown: true},
|
|
||||||
"one over the limit": {lines: fill + 1, shown: false},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, tc := range tests {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
headersJSON, err := json.Marshal(http.Header{
|
|
||||||
"A": slices.Repeat([]string{""}, tc.lines),
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
ep := f.entrypoint(t, "Billing sender")
|
|
||||||
event := f.eventAt(t, ep, string(headersJSON), time.Now())
|
|
||||||
|
|
||||||
page := renderSourceLogsPage(t, f.h, f.sess, f.webhook.ID)
|
|
||||||
box := headerBox(slices.Repeat([]string{"A: "}, tc.lines)...)
|
|
||||||
link := showHeadersLink(f.webhook.ID, event.ID)
|
|
||||||
|
|
||||||
assert.Equal(t, tc.shown, strings.Contains(page, box))
|
|
||||||
assert.Equal(t, !tc.shown, strings.Contains(page, link))
|
|
||||||
assert.Less(t, len(page), 4*bodyCap)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -75,7 +75,7 @@ func (s *Handlers) LoadEventLogViewsForTest(
|
|||||||
webhook database.Webhook,
|
webhook database.Webhook,
|
||||||
) []EventLogView {
|
) []EventLogView {
|
||||||
views, _, _ := s.loadEventsWithDeliveries(
|
views, _, _ := s.loadEventsWithDeliveries(
|
||||||
w, newRequestForTest(), webhook, nil, nil,
|
w, newRequestForTest(), webhook, nil,
|
||||||
)
|
)
|
||||||
|
|
||||||
return views
|
return views
|
||||||
@@ -160,16 +160,7 @@ func (s *Handlers) BuildHTTPTargetConfigForTest(
|
|||||||
// buildDatabaseTargetConfig for use in the handlers_test
|
// buildDatabaseTargetConfig for use in the handlers_test
|
||||||
// package.
|
// package.
|
||||||
func BuildDatabaseTargetConfigForTest(
|
func BuildDatabaseTargetConfigForTest(
|
||||||
expiry, rotation string,
|
expiry string,
|
||||||
) (string, string, error) {
|
) (string, string, error) {
|
||||||
return buildDatabaseTargetConfig(expiry, rotation)
|
return buildDatabaseTargetConfig(expiry)
|
||||||
}
|
|
||||||
|
|
||||||
// ArchiveFileViewForTest exposes archiveFileView, which describes a
|
|
||||||
// database target's archive files as the target list shows them at
|
|
||||||
// now.
|
|
||||||
func (s *Handlers) ArchiveFileViewForTest(
|
|
||||||
webhook *database.Webhook, target *database.Target, now time.Time,
|
|
||||||
) *ArchiveFileView {
|
|
||||||
return s.archiveFileView(webhook, target, now)
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -99,9 +99,7 @@ type Handlers struct {
|
|||||||
// Interleaved, one could rename an archive between another's
|
// Interleaved, one could rename an archive between another's
|
||||||
// rename and save, leaving the file named for one edit and the
|
// rename and save, leaving the file named for one edit and the
|
||||||
// stored names from the other. An archive download holds it while
|
// stored names from the other. An archive download holds it while
|
||||||
// it reads the stored names and lists the files they give, and
|
// it reads the stored names and opens the file they give.
|
||||||
// again for each file while it finds the file under the names
|
|
||||||
// stored then and opens it.
|
|
||||||
renameMu sync.Mutex
|
renameMu sync.Mutex
|
||||||
|
|
||||||
// dummyVerifications counts the equivalent-cost verifications
|
// dummyVerifications counts the equivalent-cost verifications
|
||||||
@@ -167,12 +165,12 @@ func New(
|
|||||||
),
|
),
|
||||||
"source_edit.html": parsePageTemplate("source_edit.html"),
|
"source_edit.html": parsePageTemplate("source_edit.html"),
|
||||||
"source_logs.html": parsePageTemplate(
|
"source_logs.html": parsePageTemplate(
|
||||||
"source_logs.html", "event_request.html", "event_body.html",
|
"source_logs.html", "event_body.html", "delivery_row.html",
|
||||||
"delivery_row.html", "delivery_attempts.html",
|
"delivery_attempts.html",
|
||||||
),
|
),
|
||||||
"event_detail.html": parsePageTemplate(
|
"event_detail.html": parsePageTemplate(
|
||||||
"event_detail.html", "event_request.html", "event_body.html",
|
"event_detail.html", "event_body.html", "delivery_row.html",
|
||||||
"delivery_row.html", "delivery_attempts.html",
|
"delivery_attempts.html",
|
||||||
),
|
),
|
||||||
"target_edit.html": parsePageTemplate("target_edit.html"),
|
"target_edit.html": parsePageTemplate("target_edit.html"),
|
||||||
"error.html": parsePageTemplate("error.html"),
|
"error.html": parsePageTemplate("error.html"),
|
||||||
|
|||||||
@@ -477,37 +477,23 @@ func TestRenderTemplateMidRenderErrorSendsNoPartialBody(t *testing.T) {
|
|||||||
func TestBuildDatabaseTargetConfig_Valid(t *testing.T) {
|
func TestBuildDatabaseTargetConfig_Valid(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
// Empty expiry and rotation: the keep-forever, one-file default,
|
// Empty expiry: the keep-forever default, empty config.
|
||||||
// empty config.
|
cfg, errMsg, err := handlers.BuildDatabaseTargetConfigForTest("")
|
||||||
cfg, errMsg, err := handlers.BuildDatabaseTargetConfigForTest("", "")
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Empty(t, errMsg)
|
assert.Empty(t, errMsg)
|
||||||
assert.Empty(t, cfg)
|
assert.Empty(t, cfg)
|
||||||
|
|
||||||
// Explicit never is stored as config.
|
// Explicit never is stored as config.
|
||||||
cfg, errMsg, err = handlers.BuildDatabaseTargetConfigForTest("never", "")
|
cfg, errMsg, err = handlers.BuildDatabaseTargetConfigForTest("never")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Empty(t, errMsg)
|
assert.Empty(t, errMsg)
|
||||||
assert.JSONEq(t, `{"expiry":"never"}`, cfg)
|
assert.JSONEq(t, `{"expiry":"never"}`, cfg)
|
||||||
|
|
||||||
// A positive duration is stored as config.
|
// A positive duration is stored as config.
|
||||||
cfg, errMsg, err = handlers.BuildDatabaseTargetConfigForTest("720h", "")
|
cfg, errMsg, err = handlers.BuildDatabaseTargetConfigForTest("720h")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Empty(t, errMsg)
|
assert.Empty(t, errMsg)
|
||||||
assert.JSONEq(t, `{"expiry":"720h"}`, cfg)
|
assert.JSONEq(t, `{"expiry":"720h"}`, cfg)
|
||||||
|
|
||||||
// A rotation is stored as config, with or without an expiry.
|
|
||||||
cfg, errMsg, err = handlers.BuildDatabaseTargetConfigForTest("", "daily")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Empty(t, errMsg)
|
|
||||||
assert.JSONEq(t, `{"rotation":"daily"}`, cfg)
|
|
||||||
|
|
||||||
cfg, errMsg, err = handlers.BuildDatabaseTargetConfigForTest(
|
|
||||||
"720h", "hourly",
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Empty(t, errMsg)
|
|
||||||
assert.JSONEq(t, `{"expiry":"720h","rotation":"hourly"}`, cfg)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildDatabaseTargetConfig_RejectsBadExpiry(
|
func TestBuildDatabaseTargetConfig_RejectsBadExpiry(
|
||||||
@@ -516,7 +502,7 @@ func TestBuildDatabaseTargetConfig_RejectsBadExpiry(
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
for _, bad := range []string{"nonsense", "7d", "-5h"} {
|
for _, bad := range []string{"nonsense", "7d", "-5h"} {
|
||||||
cfg, errMsg, err := handlers.BuildDatabaseTargetConfigForTest(bad, "")
|
cfg, errMsg, err := handlers.BuildDatabaseTargetConfigForTest(bad)
|
||||||
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Contains(
|
assert.Contains(
|
||||||
@@ -526,20 +512,3 @@ func TestBuildDatabaseTargetConfig_RejectsBadExpiry(
|
|||||||
assert.Empty(t, cfg)
|
assert.Empty(t, cfg)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildDatabaseTargetConfig_RejectsBadRotation(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, bad := range []string{"weekly", "Daily", " none"} {
|
|
||||||
cfg, errMsg, err := handlers.BuildDatabaseTargetConfigForTest("", bad)
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Contains(
|
|
||||||
t, errMsg, "Invalid archive rotation",
|
|
||||||
"rotation %q should be refused", bad,
|
|
||||||
)
|
|
||||||
assert.Empty(t, cfg)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -96,7 +96,7 @@ func noticeFor(r *http.Request) *notice {
|
|||||||
},
|
},
|
||||||
resubmitNoTargets: {
|
resubmitNoTargets: {
|
||||||
Text: "Resubmitted: a new event was created, but this " +
|
Text: "Resubmitted: a new event was created, but this " +
|
||||||
"webhook has no active targets, so nothing was queued.",
|
"source has no active targets, so nothing was queued.",
|
||||||
},
|
},
|
||||||
}[noticeCode(r.URL.Query().Get(noticeParam))]
|
}[noticeCode(r.URL.Query().Get(noticeParam))]
|
||||||
if !ok {
|
if !ok {
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ func TestEveryPageRendersItsOwnTitle(t *testing.T) {
|
|||||||
data map[string]any
|
data map[string]any
|
||||||
title string
|
title string
|
||||||
}{
|
}{
|
||||||
{"login.html", map[string]any{}, "Sign in - Webhooker"},
|
{"login.html", map[string]any{}, "Login - Webhooker"},
|
||||||
{"profile.html", map[string]any{}, "Profile - Webhooker"},
|
{"profile.html", map[string]any{}, "Profile - Webhooker"},
|
||||||
{"settings.html", map[string]any{}, "Settings - Webhooker"},
|
{"settings.html", map[string]any{}, "Settings - Webhooker"},
|
||||||
{"sources_list.html", map[string]any{}, "Webhooks - Webhooker"},
|
{"sources_list.html", map[string]any{}, "Webhooks - Webhooker"},
|
||||||
|
|||||||
@@ -1,230 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// parseRetentionDays interprets a retention_days form value. It
|
|
||||||
// returns the number of days, or, for a value it refuses, the message
|
|
||||||
// the create and edit forms show; the message is empty when the value
|
|
||||||
// is accepted.
|
|
||||||
//
|
|
||||||
// An empty value yields fallback, which lets the create path apply the
|
|
||||||
// default and the edit path leave the stored value unchanged. A value
|
|
||||||
// of 0 is returned as 0 and is rewritten to the retain-forever
|
|
||||||
// sentinel by database.Webhook's BeforeSave hook. Anything unparseable
|
|
||||||
// or negative is refused rather than silently given a default.
|
|
||||||
//
|
|
||||||
// The upper bound is not cosmetic. The reaper computes its cutoff as a
|
|
||||||
// time.Duration, an int64 nanosecond count, so a day count above
|
|
||||||
// database.MaxFiniteRetentionDays overflows, puts the cutoff in the
|
|
||||||
// future, and deletes every event the webhook has. A finite value
|
|
||||||
// above that ceiling is therefore refused, and the message names the
|
|
||||||
// ceiling rather than implying the input was not a number.
|
|
||||||
//
|
|
||||||
// A value at or above the retain-forever sentinel is not out of range:
|
|
||||||
// it is what the edit form pre-fills for a retain-forever webhook, so
|
|
||||||
// submitting the form back unchanged has to keep meaning "forever"
|
|
||||||
// rather than being rejected.
|
|
||||||
func parseRetentionDays(raw string, fallback int) (int, string) {
|
|
||||||
raw = strings.TrimSpace(raw)
|
|
||||||
if raw == "" {
|
|
||||||
return fallback, ""
|
|
||||||
}
|
|
||||||
|
|
||||||
v, err := strconv.Atoi(raw)
|
|
||||||
if err != nil || v < 0 {
|
|
||||||
return 0, "Retention must be a whole number of days, or 0 to " +
|
|
||||||
"retain events forever."
|
|
||||||
}
|
|
||||||
|
|
||||||
if v >= database.RetentionForeverDays {
|
|
||||||
return database.RetentionForeverDays, ""
|
|
||||||
}
|
|
||||||
|
|
||||||
if v > database.MaxFiniteRetentionDays {
|
|
||||||
return 0, "Retention must be at most " +
|
|
||||||
strconv.Itoa(database.MaxFiniteRetentionDays) +
|
|
||||||
" days, or 0 to retain events forever."
|
|
||||||
}
|
|
||||||
|
|
||||||
return v, ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// ownedWebhook resolves the request's sourceID parameter to a
|
|
||||||
// webhook the session's user owns.
|
|
||||||
//
|
|
||||||
// Ownership and existence are decided by one query, so a
|
|
||||||
// webhook belonging to another user is indistinguishable from
|
|
||||||
// one that does not exist: both are a 404, and neither confirms
|
|
||||||
// the id. Callers that reach further into a webhook's data —
|
|
||||||
// the event log page and the event body download — share this
|
|
||||||
// one check rather than restating it, so the download cannot
|
|
||||||
// come to authorize differently from the page that links to it.
|
|
||||||
//
|
|
||||||
// It reports false once it has written the response, which is a
|
|
||||||
// redirect to the login page for an unauthenticated request and
|
|
||||||
// a 404 otherwise. The caller returns without writing more.
|
|
||||||
func (h *Handlers) ownedWebhook(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
r *http.Request,
|
|
||||||
) (database.Webhook, bool) {
|
|
||||||
var webhook database.Webhook
|
|
||||||
|
|
||||||
userID, ok := h.getUserID(r)
|
|
||||||
if !ok {
|
|
||||||
http.Redirect(
|
|
||||||
w, r, "/pages/login", http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
|
|
||||||
return database.Webhook{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
sourceID := chi.URLParam(r, "sourceID")
|
|
||||||
|
|
||||||
err := h.db.DB().Where(
|
|
||||||
"id = ? AND user_id = ?", sourceID, userID,
|
|
||||||
).First(&webhook).Error
|
|
||||||
if err != nil {
|
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
|
||||||
|
|
||||||
return database.Webhook{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
return webhook, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// deleteChildResource returns a handler that deletes a child
|
|
||||||
// resource (entrypoint or target) belonging to a webhook. The
|
|
||||||
// optional afterDelete hook runs with the child's id once the
|
|
||||||
// delete has removed it, before the redirect, which carries done as
|
|
||||||
// its notice.
|
|
||||||
func (h *Handlers) deleteChildResource(
|
|
||||||
idParam string,
|
|
||||||
model any,
|
|
||||||
errMsg string,
|
|
||||||
afterDelete func(childID string),
|
|
||||||
done noticeCode,
|
|
||||||
) http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
userID, ok := h.getUserID(r)
|
|
||||||
if !ok {
|
|
||||||
http.Redirect(
|
|
||||||
w, r, "/pages/login", http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
sourceID := chi.URLParam(r, "sourceID")
|
|
||||||
childID := chi.URLParam(r, idParam)
|
|
||||||
|
|
||||||
var webhook database.Webhook
|
|
||||||
|
|
||||||
err := h.db.DB().Where(
|
|
||||||
"id = ? AND user_id = ?", sourceID, userID,
|
|
||||||
).First(&webhook).Error
|
|
||||||
if err != nil {
|
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
result := h.db.DB().Where(
|
|
||||||
"id = ? AND webhook_id = ?",
|
|
||||||
childID, webhook.ID,
|
|
||||||
).Delete(model)
|
|
||||||
if result.Error != nil {
|
|
||||||
h.serverError(w, r, errMsg, result.Error)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Only for a row this webhook really had: the id came from
|
|
||||||
// the URL and may name another webhook's child.
|
|
||||||
if afterDelete != nil && result.RowsAffected > 0 {
|
|
||||||
afterDelete(childID)
|
|
||||||
}
|
|
||||||
|
|
||||||
http.Redirect(
|
|
||||||
w, r,
|
|
||||||
withNotice("/hook/"+webhook.ID, done),
|
|
||||||
http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// toggleChildResource returns a handler that toggles the active
|
|
||||||
// state of a child resource belonging to a webhook. toggleFn returns
|
|
||||||
// the new state, and the redirect carries activated or deactivated as
|
|
||||||
// its notice to match.
|
|
||||||
func (h *Handlers) toggleChildResource(
|
|
||||||
idParam string,
|
|
||||||
toggleFn func(webhookID, childID string) (bool, error),
|
|
||||||
errMsg string,
|
|
||||||
activated, deactivated noticeCode,
|
|
||||||
) http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
userID, ok := h.getUserID(r)
|
|
||||||
if !ok {
|
|
||||||
http.Redirect(
|
|
||||||
w, r, "/pages/login", http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
sourceID := chi.URLParam(r, "sourceID")
|
|
||||||
childID := chi.URLParam(r, idParam)
|
|
||||||
|
|
||||||
var webhook database.Webhook
|
|
||||||
|
|
||||||
err := h.db.DB().Where(
|
|
||||||
"id = ? AND user_id = ?", sourceID, userID,
|
|
||||||
).First(&webhook).Error
|
|
||||||
if err != nil {
|
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
active, err := toggleFn(webhook.ID, childID)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, r, errMsg, err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
done := deactivated
|
|
||||||
if active {
|
|
||||||
done = activated
|
|
||||||
}
|
|
||||||
|
|
||||||
http.Redirect(
|
|
||||||
w, r,
|
|
||||||
withNotice("/hook/"+webhook.ID, done),
|
|
||||||
http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// getUserID extracts the user ID from the session.
|
|
||||||
func (h *Handlers) getUserID(
|
|
||||||
r *http.Request,
|
|
||||||
) (string, bool) {
|
|
||||||
sess, err := h.session.Get(r)
|
|
||||||
if err != nil {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
if !h.session.IsAuthenticated(sess) {
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
return h.session.GetUserID(sess)
|
|
||||||
}
|
|
||||||
@@ -120,7 +120,7 @@ func TestHandleSourceCreateSubmit_CreatesRequestedTargets(t *testing.T) {
|
|||||||
// retention, each with archive on. Nothing is created, and the form
|
// retention, each with archive on. Nothing is created, and the form
|
||||||
// comes back with the reason and every value entered: name,
|
// comes back with the reason and every value entered: name,
|
||||||
// description, retention, URL, the checked archive box and the pruning
|
// description, retention, URL, the checked archive box and the pruning
|
||||||
// and rotation choices.
|
// choice.
|
||||||
func TestHandleSourceCreateSubmit_RefusedFormKeepsEveryValue(
|
func TestHandleSourceCreateSubmit_RefusedFormKeepsEveryValue(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
@@ -152,7 +152,6 @@ func TestHandleSourceCreateSubmit_RefusedFormKeepsEveryValue(
|
|||||||
form.Set("http_url", tc.httpURL)
|
form.Set("http_url", tc.httpURL)
|
||||||
form.Set("archive", "on")
|
form.Set("archive", "on")
|
||||||
form.Set("archive_expiry", "2160h")
|
form.Set("archive_expiry", "2160h")
|
||||||
form.Set("archive_rotation", "hourly")
|
|
||||||
|
|
||||||
w := submitCreateForm(env, form)
|
w := submitCreateForm(env, form)
|
||||||
require.Equal(t, http.StatusBadRequest, w.Code)
|
require.Equal(t, http.StatusBadRequest, w.Code)
|
||||||
@@ -167,7 +166,6 @@ func TestHandleSourceCreateSubmit_RefusedFormKeepsEveryValue(
|
|||||||
assert.Contains(t, page, `name="archive" value="on" checked`)
|
assert.Contains(t, page, `name="archive" value="on" checked`)
|
||||||
assert.Contains(t, page, `x-data="collapsible" data-open`)
|
assert.Contains(t, page, `x-data="collapsible" data-open`)
|
||||||
assert.Contains(t, page, `<option value="2160h" selected>`)
|
assert.Contains(t, page, `<option value="2160h" selected>`)
|
||||||
assert.Contains(t, page, `<option value="hourly" selected>`)
|
|
||||||
|
|
||||||
assertNothingCreated(t, env.db)
|
assertNothingCreated(t, env.db)
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -233,13 +233,8 @@ func TestHandleSourceDetail_RendersNamedTargetFields(
|
|||||||
assert.Contains(t, body, "1 configured")
|
assert.Contains(t, body, "1 configured")
|
||||||
assert.NotContains(t, body, "sekrit")
|
assert.NotContains(t, body, "sekrit")
|
||||||
|
|
||||||
// The database type is called an archive: on its badge, in the
|
assert.Contains(t, body, "Archive Expiry")
|
||||||
// add target form's type list and in its settings.
|
assert.Contains(t, body, "30 days")
|
||||||
list := targetList(t, body)
|
|
||||||
assert.Contains(t, list, "t-database archive Active")
|
|
||||||
assert.Contains(t, list, "Archive expiry: 30 days")
|
|
||||||
assert.Contains(t, list, "Archive rotation: none")
|
|
||||||
assert.Contains(t, body, `<option value="database">Archive</option>`)
|
|
||||||
|
|
||||||
// An unknown type gets the neutral placeholder, never the
|
// An unknown type gets the neutral placeholder, never the
|
||||||
// stored blob.
|
// stored blob.
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"slices"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -187,152 +186,6 @@ func TestHandleSourceLogs_ShowsFiftyNewestEvents(t *testing.T) {
|
|||||||
assert.Contains(t, body, "50 most recent of 51 events")
|
assert.Contains(t, body, "50 most recent of 51 events")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleSourceLogs_ShowsEventsByDeliveryStatus proves that the
|
|
||||||
// Failed list holds exactly the events with a failed delivery, the
|
|
||||||
// Pending list exactly those with a delivery pending or retrying, each
|
|
||||||
// once, and any other show value every event; that each link, and the
|
|
||||||
// line beside the heading, counts the events its list holds; and that
|
|
||||||
// the shown link is marked.
|
|
||||||
func TestHandleSourceLogs_ShowsEventsByDeliveryStatus(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
target := seedTarget(t, f.db, f.webhook.ID, database.TargetTypeLog)
|
|
||||||
now := time.Now()
|
|
||||||
|
|
||||||
const (
|
|
||||||
failed = database.DeliveryStatusFailed
|
|
||||||
delivered = database.DeliveryStatusDelivered
|
|
||||||
pending = database.DeliveryStatusPending
|
|
||||||
retrying = database.DeliveryStatusRetrying
|
|
||||||
)
|
|
||||||
|
|
||||||
// Each event is named by its content type. The first failed and
|
|
||||||
// was then replayed and delivered. The second failed, and so did
|
|
||||||
// its replay, and the fifth has one delivery pending and another
|
|
||||||
// retrying: each must still be listed and counted once.
|
|
||||||
events := []struct {
|
|
||||||
contentType string
|
|
||||||
deliveries []database.DeliveryStatus
|
|
||||||
}{
|
|
||||||
{"application/x-failed", []database.DeliveryStatus{failed, delivered}},
|
|
||||||
{"application/x-failed-twice", []database.DeliveryStatus{failed, failed}},
|
|
||||||
{"application/x-pending", []database.DeliveryStatus{pending}},
|
|
||||||
{"application/x-retrying", []database.DeliveryStatus{retrying}},
|
|
||||||
{"application/x-pending-retrying", []database.DeliveryStatus{pending, retrying}},
|
|
||||||
{"application/x-delivered", []database.DeliveryStatus{delivered}},
|
|
||||||
{"application/x-no-delivery", nil},
|
|
||||||
}
|
|
||||||
|
|
||||||
all := make([]string, len(events))
|
|
||||||
|
|
||||||
for i, e := range events {
|
|
||||||
event := f.event(
|
|
||||||
t, e.contentType, "{}", now.Add(time.Duration(i)*time.Second),
|
|
||||||
)
|
|
||||||
|
|
||||||
for _, status := range e.deliveries {
|
|
||||||
f.delivery(t, event, target.ID, status)
|
|
||||||
}
|
|
||||||
|
|
||||||
all[i] = e.contentType
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
query string
|
|
||||||
current string
|
|
||||||
heading string
|
|
||||||
listed []string
|
|
||||||
}{
|
|
||||||
{"", "All", "7 total events", all},
|
|
||||||
{"?show=failed", "Failed (2)", "2 events with a failed delivery",
|
|
||||||
[]string{"application/x-failed", "application/x-failed-twice"}},
|
|
||||||
{"?show=pending", "Pending (3)",
|
|
||||||
"3 events with a delivery pending or retrying", []string{
|
|
||||||
"application/x-pending", "application/x-retrying",
|
|
||||||
"application/x-pending-retrying",
|
|
||||||
}},
|
|
||||||
{"?show=unknown", "All", "7 total events", all},
|
|
||||||
} {
|
|
||||||
body := renderSourceLogsPageWithQuery(
|
|
||||||
t, f.h, f.sess, f.webhook.ID, tc.query,
|
|
||||||
)
|
|
||||||
|
|
||||||
// One row per listed event, so with each listed event shown
|
|
||||||
// no event is listed twice.
|
|
||||||
assert.Equal(t, len(tc.listed),
|
|
||||||
strings.Count(body, `role="button"`), tc.query)
|
|
||||||
|
|
||||||
for _, contentType := range all {
|
|
||||||
assert.Equal(t,
|
|
||||||
slices.Contains(tc.listed, contentType),
|
|
||||||
strings.Contains(body, ">"+contentType+"<"),
|
|
||||||
tc.query+" "+contentType)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Contains(t, body, ">"+tc.heading+"<", tc.query)
|
|
||||||
assert.Contains(t, body, "Failed (2)", tc.query)
|
|
||||||
assert.Contains(t, body, "Pending (3)", tc.query)
|
|
||||||
assert.Equal(t, 1, strings.Count(body, "aria-current"), tc.query)
|
|
||||||
assert.Contains(t, body,
|
|
||||||
`aria-current="page">`+tc.current+"</a>", tc.query)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceLogs_FilteredListShowsFiftyNewest proves a filtered
|
|
||||||
// list holds the 50 newest matching events, as the full log does,
|
|
||||||
// while its link and heading count every matching event.
|
|
||||||
func TestHandleSourceLogs_FilteredListShowsFiftyNewest(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
target := seedTarget(t, f.db, f.webhook.ID, database.TargetTypeLog)
|
|
||||||
base := time.Now().Add(-time.Hour)
|
|
||||||
|
|
||||||
for i := range 51 {
|
|
||||||
event := f.event(
|
|
||||||
t, fmt.Sprintf("application/x-failed-%02d", i), "{}",
|
|
||||||
base.Add(time.Duration(i)*time.Second),
|
|
||||||
)
|
|
||||||
f.delivery(t, event, target.ID, database.DeliveryStatusFailed)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The newest event has no failed delivery.
|
|
||||||
f.event(t, "application/x-no-delivery", "{}", time.Now())
|
|
||||||
|
|
||||||
body := renderSourceLogsPageWithQuery(
|
|
||||||
t, f.h, f.sess, f.webhook.ID, "?show=failed",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Equal(t, 50, strings.Count(body, `role="button"`))
|
|
||||||
assert.NotContains(t, body, "application/x-failed-00")
|
|
||||||
assert.NotContains(t, body, "application/x-no-delivery")
|
|
||||||
assert.Contains(t, body, "Failed (51)")
|
|
||||||
assert.Contains(t, body,
|
|
||||||
"50 most recent of 51 events with a failed delivery")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceLogs_EmptyFilteredList proves an empty filtered list
|
|
||||||
// says that no event matches rather than that none was recorded.
|
|
||||||
func TestHandleSourceLogs_EmptyFilteredList(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
target := seedTarget(t, f.db, f.webhook.ID, database.TargetTypeLog)
|
|
||||||
|
|
||||||
f.delivery(
|
|
||||||
t, f.event(t, contentTypeJSON, "{}", time.Now()),
|
|
||||||
target.ID, database.DeliveryStatusDelivered,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Contains(t, renderSourceLogsPageWithQuery(
|
|
||||||
t, f.h, f.sess, f.webhook.ID, "?show=failed",
|
|
||||||
), "No event has a failed delivery.")
|
|
||||||
assert.Contains(t, renderSourceLogsPageWithQuery(
|
|
||||||
t, f.h, f.sess, f.webhook.ID, "?show=pending",
|
|
||||||
), "No event has a delivery pending or retrying.")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceLogs_OnlyNewestStartsExpanded proves that of the
|
// TestHandleSourceLogs_OnlyNewestStartsExpanded proves that of the
|
||||||
// events in the log only the newest starts expanded.
|
// events in the log only the newest starts expanded.
|
||||||
func TestHandleSourceLogs_OnlyNewestStartsExpanded(t *testing.T) {
|
func TestHandleSourceLogs_OnlyNewestStartsExpanded(t *testing.T) {
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -1,384 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// HandleTargetCreate handles adding a new target to a webhook.
|
|
||||||
func (h *Handlers) HandleTargetCreate() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
userID, ok := h.getUserID(r)
|
|
||||||
if !ok {
|
|
||||||
http.Redirect(
|
|
||||||
w, r, "/pages/login", http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
sourceID := chi.URLParam(r, "sourceID")
|
|
||||||
|
|
||||||
h.renameMu.Lock()
|
|
||||||
defer h.renameMu.Unlock()
|
|
||||||
|
|
||||||
var webhook database.Webhook
|
|
||||||
|
|
||||||
err := h.db.DB().Where(
|
|
||||||
"id = ? AND user_id = ?", sourceID, userID,
|
|
||||||
).First(&webhook).Error
|
|
||||||
if err != nil {
|
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// The body size cap is enforced by the MaxBodySize
|
|
||||||
// middleware, which runs before CSRF parses the form.
|
|
||||||
err = r.ParseForm()
|
|
||||||
if err != nil {
|
|
||||||
h.renderError(w, r, http.StatusBadRequest)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.processTargetCreate(w, r, webhook)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// processTargetCreate validates and creates a new target. A refused
|
|
||||||
// submission shows the webhook page again, with the add target form
|
|
||||||
// open on the chosen type, the values entered, and the reason.
|
|
||||||
func (h *Handlers) processTargetCreate(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
r *http.Request,
|
|
||||||
webhook database.Webhook,
|
|
||||||
) {
|
|
||||||
in := targetFormInputFrom(r)
|
|
||||||
|
|
||||||
target, errMsg, err := h.newTarget(r.Context(), webhook.ID, in)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, r, "failed to encode target config", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if errMsg != "" {
|
|
||||||
h.renderSourceDetail(w, r, webhook, in, errMsg)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
err = h.db.DB().Create(target).Error
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, r, "failed to create target", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
http.Redirect(
|
|
||||||
w, r, withNotice("/hook/"+webhook.ID, targetAdded),
|
|
||||||
http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// newTarget validates a new target for a webhook and returns the row
|
|
||||||
// to create, or, when it refuses the target, the message the form
|
|
||||||
// shows. An error is the server's fault, not a refusal: the accepted
|
|
||||||
// configuration could not be encoded. Every form that creates a
|
|
||||||
// target goes through here, so they all accept and refuse the same
|
|
||||||
// things.
|
|
||||||
func (h *Handlers) newTarget(
|
|
||||||
ctx context.Context,
|
|
||||||
webhookID string,
|
|
||||||
in targetFormInput,
|
|
||||||
) (*database.Target, string, error) {
|
|
||||||
target := &database.Target{
|
|
||||||
WebhookID: webhookID,
|
|
||||||
Type: in.Type,
|
|
||||||
Active: true,
|
|
||||||
}
|
|
||||||
|
|
||||||
errMsg, err := h.setTargetFromForm(ctx, target, in)
|
|
||||||
if err != nil || errMsg != "" {
|
|
||||||
return nil, errMsg, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return target, "", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// setTargetFromForm validates a target form against the target's type
|
|
||||||
// and, when it accepts it, sets the target's name, configuration and
|
|
||||||
// retry count from it. It returns the message the form shows for
|
|
||||||
// anything it refuses, an unknown type among them, and then leaves the
|
|
||||||
// target unchanged; an error is the server's fault, as for newTarget.
|
|
||||||
// The add target form and the target edit form both go through here,
|
|
||||||
// so the two cannot come to disagree about what a target may be.
|
|
||||||
func (h *Handlers) setTargetFromForm(
|
|
||||||
ctx context.Context,
|
|
||||||
target *database.Target,
|
|
||||||
in targetFormInput,
|
|
||||||
) (string, error) {
|
|
||||||
if in.Name == "" {
|
|
||||||
return "Name is required", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
configJSON, errMsg, err := h.buildTargetConfig(ctx, target.Type, in)
|
|
||||||
if err != nil || errMsg != "" {
|
|
||||||
return errMsg, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// An empty max_retries keeps the target's count: the
|
|
||||||
// fire-and-forget default of 0 for a new target, and the stored
|
|
||||||
// count for an edited one, since the forms for target types that
|
|
||||||
// do not retry have no such field. A value that is filled in but
|
|
||||||
// invalid is refused rather than becoming that count, so a typo
|
|
||||||
// cannot destroy the count a target is delivering with.
|
|
||||||
maxRetries, err := parseMaxRetries(in.MaxRetries, target.MaxRetries)
|
|
||||||
if err != nil {
|
|
||||||
return "Invalid delivery attempts: " + retriesErrorMessage(err), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
target.Name = in.Name
|
|
||||||
target.Config = configJSON
|
|
||||||
target.MaxRetries = maxRetries
|
|
||||||
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// targetFormInput carries the raw values of a target form. Both the
|
|
||||||
// create and the edit path fill one and hand it to setTargetFromForm,
|
|
||||||
// so neither can come to validate a target differently from the
|
|
||||||
// other. Both forms are filled from one: the edit form with the
|
|
||||||
// stored values, and a refused form with the values submitted.
|
|
||||||
type targetFormInput struct {
|
|
||||||
// Name is the target's name.
|
|
||||||
Name string
|
|
||||||
// Type is the type chosen on the add target form. The edit form
|
|
||||||
// has none: a target's stored type decides.
|
|
||||||
Type database.TargetType
|
|
||||||
// URL is the destination for an HTTP target and the webhook URL
|
|
||||||
// for a Slack target.
|
|
||||||
URL string
|
|
||||||
// Headers is an HTTP target's headers, one "Name: value" per
|
|
||||||
// line.
|
|
||||||
Headers string
|
|
||||||
// Timeout is an HTTP target's per-request timeout in seconds.
|
|
||||||
Timeout string
|
|
||||||
// MaxRetries is an HTTP or Slack target's max_retries.
|
|
||||||
MaxRetries string
|
|
||||||
// Expiry is a database (archive) target's row expiry.
|
|
||||||
Expiry string
|
|
||||||
// Rotation is a database (archive) target's rotation.
|
|
||||||
Rotation string
|
|
||||||
}
|
|
||||||
|
|
||||||
// targetFormInputFrom reads a target form from a request body. The
|
|
||||||
// body size cap is enforced by the MaxBodySize middleware, which runs
|
|
||||||
// before CSRF parses the form.
|
|
||||||
//
|
|
||||||
// Every field is read with PostFormValue, not FormValue. FormValue
|
|
||||||
// falls back to the query string, which would let
|
|
||||||
// `POST /hook/{id}/targets?url=https://hooks.slack.com/...`
|
|
||||||
// configure a target from a value the request line carries — and the
|
|
||||||
// request line, unlike the body, is what logs, proxies, Referer
|
|
||||||
// headers and error trackers record. The headers field is under the
|
|
||||||
// same rule and for the same reason: its values are authorization
|
|
||||||
// tokens.
|
|
||||||
func targetFormInputFrom(r *http.Request) targetFormInput {
|
|
||||||
return targetFormInput{
|
|
||||||
Name: r.PostFormValue("name"),
|
|
||||||
Type: database.TargetType(r.PostFormValue("type")),
|
|
||||||
URL: r.PostFormValue("url"),
|
|
||||||
Headers: r.PostFormValue("headers"),
|
|
||||||
Timeout: r.PostFormValue("timeout"),
|
|
||||||
MaxRetries: r.PostFormValue("max_retries"),
|
|
||||||
Expiry: r.PostFormValue("expiry"),
|
|
||||||
Rotation: r.PostFormValue("rotation"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildTargetConfig builds the JSON config string for a target from
|
|
||||||
// the submitted form values, or returns the message the form shows
|
|
||||||
// for a value it refuses. An error is the server's fault, not a
|
|
||||||
// refusal: the accepted configuration could not be encoded. Which
|
|
||||||
// fields of in apply depends on the target type; a type without a URL
|
|
||||||
// ignores any URL submitted.
|
|
||||||
func (h *Handlers) buildTargetConfig(
|
|
||||||
ctx context.Context,
|
|
||||||
targetType database.TargetType,
|
|
||||||
in targetFormInput,
|
|
||||||
) (string, string, error) {
|
|
||||||
switch targetType {
|
|
||||||
case database.TargetTypeHTTP:
|
|
||||||
return h.buildHTTPTargetConfig(ctx, in)
|
|
||||||
case database.TargetTypeSlack:
|
|
||||||
return h.buildSlackTargetConfig(ctx, in.URL)
|
|
||||||
case database.TargetTypeDatabase:
|
|
||||||
return buildDatabaseTargetConfig(in.Expiry, in.Rotation)
|
|
||||||
case database.TargetTypeLog:
|
|
||||||
return "", "", nil
|
|
||||||
default:
|
|
||||||
return "", "Invalid target type", nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildHTTPTargetConfig builds config JSON for an HTTP target: an
|
|
||||||
// SSRF-validated destination plus the optional headers and timeout
|
|
||||||
// the delivery path honours.
|
|
||||||
func (h *Handlers) buildHTTPTargetConfig(
|
|
||||||
ctx context.Context,
|
|
||||||
in targetFormInput,
|
|
||||||
) (string, string, error) {
|
|
||||||
errMsg := h.validateTargetURL(
|
|
||||||
ctx, in.URL, "URL is required for HTTP targets",
|
|
||||||
)
|
|
||||||
if errMsg != "" {
|
|
||||||
return "", errMsg, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
headers, err := delivery.ParseTargetHeaders(in.Headers)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Sprintf("Invalid headers: %v", err), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
timeout, err := delivery.ParseTargetTimeout(in.Timeout)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Sprintf("Invalid timeout: %v", err), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
configJSON, err := marshalTargetConfig(delivery.HTTPTargetConfig{
|
|
||||||
URL: in.URL,
|
|
||||||
Headers: headers,
|
|
||||||
Timeout: timeout,
|
|
||||||
})
|
|
||||||
|
|
||||||
return configJSON, "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildSlackTargetConfig builds config JSON for a Slack target,
|
|
||||||
// whose whole configuration is one SSRF-validated webhook URL.
|
|
||||||
func (h *Handlers) buildSlackTargetConfig(
|
|
||||||
ctx context.Context,
|
|
||||||
targetURL string,
|
|
||||||
) (string, string, error) {
|
|
||||||
errMsg := h.validateTargetURL(
|
|
||||||
ctx, targetURL,
|
|
||||||
"Webhook URL is required for Slack targets",
|
|
||||||
)
|
|
||||||
if errMsg != "" {
|
|
||||||
return "", errMsg, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
configJSON, err := marshalTargetConfig(delivery.SlackTargetConfig{
|
|
||||||
WebhookURL: targetURL,
|
|
||||||
})
|
|
||||||
|
|
||||||
return configJSON, "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
// validateTargetURL refuses an empty or SSRF-blocked destination,
|
|
||||||
// returning the message the form shows, or "" when the destination
|
|
||||||
// is accepted. missingMsg is the message for no URL at all.
|
|
||||||
//
|
|
||||||
// It is the single point at which a user-supplied destination enters
|
|
||||||
// the SSRF guard, on create and on edit alike. An edit path that
|
|
||||||
// reached storage without passing through here would reopen the hole
|
|
||||||
// the guard closes.
|
|
||||||
func (h *Handlers) validateTargetURL(
|
|
||||||
ctx context.Context,
|
|
||||||
targetURL, missingMsg string,
|
|
||||||
) string {
|
|
||||||
if targetURL == "" {
|
|
||||||
return missingMsg
|
|
||||||
}
|
|
||||||
|
|
||||||
err := h.ssrf.ValidateTargetURL(ctx, targetURL)
|
|
||||||
if err != nil {
|
|
||||||
// The submitted URL can be a credential (a Slack
|
|
||||||
// incoming webhook URL is a bearer token), so the log
|
|
||||||
// records only its scheme and host.
|
|
||||||
h.log.Warn(
|
|
||||||
"target URL blocked by SSRF protection",
|
|
||||||
"url", delivery.MaskURL(targetURL),
|
|
||||||
"error", err,
|
|
||||||
)
|
|
||||||
|
|
||||||
msg := "Invalid target URL: " + err.Error()
|
|
||||||
|
|
||||||
// Only a private or reserved address's refusal says how
|
|
||||||
// to allow it. Other refusals never do: link-local, the
|
|
||||||
// unspecified addresses and the unconditional metadata
|
|
||||||
// addresses cannot be opened, and the default
|
|
||||||
// blocklist's public addresses, which listing does open,
|
|
||||||
// hand out credentials.
|
|
||||||
if errors.Is(err, delivery.ErrBlockedPrivateOrReservedIP) {
|
|
||||||
msg += ". Private and reserved addresses are refused " +
|
|
||||||
"by default; the server's ALLOWED_EGRESS_CIDRS " +
|
|
||||||
"setting allows named networks (see \"Allowing " +
|
|
||||||
"egress to your own network\" in the README)."
|
|
||||||
}
|
|
||||||
|
|
||||||
return msg
|
|
||||||
}
|
|
||||||
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// marshalTargetConfig serialises a target configuration for storage.
|
|
||||||
func marshalTargetConfig(cfg any) (string, error) {
|
|
||||||
configBytes, err := json.Marshal(cfg)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
return string(configBytes), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildDatabaseTargetConfig builds config JSON for a database
|
|
||||||
// (archive) target. The optional expiry and rotation are validated
|
|
||||||
// here, at creation time, so a bad value is refused instead of
|
|
||||||
// failing every subsequent delivery. Each is stored only when set,
|
|
||||||
// and with neither the config is empty (the keep-forever, one-file
|
|
||||||
// default).
|
|
||||||
func buildDatabaseTargetConfig(
|
|
||||||
expiry, rotation string,
|
|
||||||
) (string, string, error) {
|
|
||||||
expiry = strings.TrimSpace(expiry)
|
|
||||||
|
|
||||||
err := delivery.ValidateArchiveExpiry(expiry)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Sprintf("Invalid archive expiry: %v", err), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
err = delivery.ValidateArchiveRotation(rotation)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Sprintf("Invalid archive rotation: %v", err), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg := map[string]any{}
|
|
||||||
|
|
||||||
if expiry != "" {
|
|
||||||
cfg["expiry"] = expiry
|
|
||||||
}
|
|
||||||
|
|
||||||
if rotation != "" {
|
|
||||||
cfg["rotation"] = rotation
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(cfg) == 0 {
|
|
||||||
return "", "", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
configJSON, err := marshalTargetConfig(cfg)
|
|
||||||
|
|
||||||
return configJSON, "", err
|
|
||||||
}
|
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// HandleTargetDelete handles deleting a target. A deleted
|
|
||||||
// database target's archive writer is evicted and its handle
|
|
||||||
// closed; the archive file is left on disk.
|
|
||||||
func (h *Handlers) HandleTargetDelete() http.HandlerFunc {
|
|
||||||
return h.deleteChildResource(
|
|
||||||
"targetID", &database.Target{},
|
|
||||||
"failed to delete target",
|
|
||||||
h.evictTargetArchiveWriter,
|
|
||||||
targetDeleted,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// evictTargetArchiveWriter is evictArchiveWriter for one deleted
|
|
||||||
// target, and leaves its archive file on disk for the same reason.
|
|
||||||
// A target that is not a database target has no writer, and
|
|
||||||
// evicting it does nothing.
|
|
||||||
func (h *Handlers) evictTargetArchiveWriter(targetID string) {
|
|
||||||
if h.archives == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.archives.EvictTarget(targetID)
|
|
||||||
}
|
|
||||||
@@ -27,11 +27,13 @@ func (h *Handlers) HandleTargetDownload() http.HandlerFunc {
|
|||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := context.WithoutCancel(r.Context())
|
ctx := context.WithoutCancel(r.Context())
|
||||||
|
|
||||||
webhook, target, export, ok := h.listTargetArchive(w, r)
|
webhook, target, export, ok := h.openTargetArchive(ctx, w, r)
|
||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
defer func() { _ = export.Close() }()
|
||||||
|
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "application/gzip")
|
w.Header().Set("Content-Type", "application/gzip")
|
||||||
@@ -79,15 +81,16 @@ func (d downloadWriter) Write(b []byte) (int, error) {
|
|||||||
return d.w.Write(b)
|
return d.w.Write(b)
|
||||||
}
|
}
|
||||||
|
|
||||||
// listTargetArchive lists the archive files of the request's database
|
// openTargetArchive opens the archive of the request's database target
|
||||||
// target for export. It reports false once it has written the response.
|
// for export, with its reads under ctx. It reports false once it has
|
||||||
|
// written the response.
|
||||||
//
|
//
|
||||||
// It holds renameMu, which every archive rename runs under, while it
|
// It holds renameMu, which every archive rename runs under, while it
|
||||||
// reads the stored names and lists the files, so the files it lists
|
// reads the stored names and opens the file, so the file it opens is
|
||||||
// are the ones those names give. It lets go before the export is
|
// the one those names give. It lets go before the export is streamed:
|
||||||
// streamed, which takes renameMu again for each file only while it
|
// once the file is open, a rename does not affect the export.
|
||||||
// finds the file under the names stored then and opens it.
|
func (h *Handlers) openTargetArchive(
|
||||||
func (h *Handlers) listTargetArchive(
|
ctx context.Context,
|
||||||
w http.ResponseWriter,
|
w http.ResponseWriter,
|
||||||
r *http.Request,
|
r *http.Request,
|
||||||
) (database.Webhook, *database.Target, *delivery.ArchiveExport, bool) {
|
) (database.Webhook, *database.Target, *delivery.ArchiveExport, bool) {
|
||||||
@@ -105,43 +108,14 @@ func (h *Handlers) listTargetArchive(
|
|||||||
return database.Webhook{}, nil, nil, false
|
return database.Webhook{}, nil, nil, false
|
||||||
}
|
}
|
||||||
|
|
||||||
export, err := delivery.NewArchiveExport(
|
export, err := delivery.OpenArchiveExport(
|
||||||
delivery.ArchivePath(h.dbMgr, &webhook, target),
|
ctx, delivery.ArchivePath(h.dbMgr, &webhook, target), h.log,
|
||||||
&h.renameMu,
|
|
||||||
func() (string, error) {
|
|
||||||
return h.storedArchivePath(webhook.ID, target.ID)
|
|
||||||
},
|
|
||||||
h.log,
|
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.serverError(w, r, "failed to list archive for export", err)
|
h.serverError(w, r, "failed to open archive for export", err)
|
||||||
|
|
||||||
return database.Webhook{}, nil, nil, false
|
return database.Webhook{}, nil, nil, false
|
||||||
}
|
}
|
||||||
|
|
||||||
return webhook, target, export, true
|
return webhook, target, export, true
|
||||||
}
|
}
|
||||||
|
|
||||||
// storedArchivePath returns the path delivery.ArchivePath gives a
|
|
||||||
// database target under the names stored for it and its webhook now.
|
|
||||||
// Its caller holds renameMu. A webhook or target deleted since is still
|
|
||||||
// found, since deleting one leaves its archive files under their names.
|
|
||||||
func (h *Handlers) storedArchivePath(
|
|
||||||
webhookID, targetID string,
|
|
||||||
) (string, error) {
|
|
||||||
var webhook database.Webhook
|
|
||||||
|
|
||||||
err := h.db.DB().Unscoped().First(&webhook, "id = ?", webhookID).Error
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
var target database.Target
|
|
||||||
|
|
||||||
err = h.db.DB().Unscoped().First(&target, "id = ?", targetID).Error
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
return delivery.ArchivePath(h.dbMgr, &webhook, &target), nil
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -13,8 +13,6 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -87,7 +85,7 @@ func TestHandleTargetDownload(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleTargetDownload_WaitsForRename proves a download reads the
|
// TestHandleTargetDownload_WaitsForRename proves a download reads the
|
||||||
// target's names and lists its archive under the lock a rename holds:
|
// target's names and opens its archive under the lock a rename holds:
|
||||||
// started while an edit is renaming the archive, it waits, and is
|
// started while an edit is renaming the archive, it waits, and is
|
||||||
// named for the target's new name.
|
// named for the target's new name.
|
||||||
func TestHandleTargetDownload_WaitsForRename(t *testing.T) {
|
func TestHandleTargetDownload_WaitsForRename(t *testing.T) {
|
||||||
@@ -150,17 +148,18 @@ func (s *stalledWriter) Write(b []byte) (int, error) {
|
|||||||
return s.ResponseRecorder.Write(b)
|
return s.ResponseRecorder.Write(b)
|
||||||
}
|
}
|
||||||
|
|
||||||
// startStalledDownload starts a download of the target and returns once
|
// TestHandleTargetDownload_StreamsWithoutTheLock proves a download
|
||||||
// it is stalled at its first write, which comes before it opens any
|
// lets go of the rename lock once its archive is open: while the
|
||||||
// archive file. Closing the writer's resume lets it go on; the returned
|
// download is stalled writing, an edit can still rename the target.
|
||||||
// channel is closed when it has finished.
|
func TestHandleTargetDownload_StreamsWithoutTheLock(t *testing.T) {
|
||||||
func startStalledDownload(
|
t.Parallel()
|
||||||
t *testing.T, env *sourceTestEnv, webhookID, targetID string,
|
|
||||||
) (*stalledWriter, <-chan struct{}) {
|
env := setupSourceTest(t)
|
||||||
t.Helper()
|
wh := seedWebhookWithRetention(t, env.db, 7)
|
||||||
|
archive := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(
|
req := httptest.NewRequestWithContext(
|
||||||
t.Context(), http.MethodGet, downloadPath(webhookID, targetID), nil,
|
t.Context(), http.MethodGet, downloadPath(wh.ID, archive.ID), nil,
|
||||||
)
|
)
|
||||||
for _, c := range env.cookies {
|
for _, c := range env.cookies {
|
||||||
req.AddCookie(c)
|
req.AddCookie(c)
|
||||||
@@ -180,21 +179,6 @@ func startStalledDownload(
|
|||||||
|
|
||||||
<-sw.writing
|
<-sw.writing
|
||||||
|
|
||||||
return sw, downloaded
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleTargetDownload_StreamsWithoutTheLock proves a download
|
|
||||||
// lets go of the rename lock once it has listed its archive: while the
|
|
||||||
// download is stalled writing, an edit can still rename the target.
|
|
||||||
func TestHandleTargetDownload_StreamsWithoutTheLock(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
wh := seedWebhookWithRetention(t, env.db, 7)
|
|
||||||
archive := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
|
||||||
|
|
||||||
sw, downloaded := startStalledDownload(t, env, wh.ID, archive.ID)
|
|
||||||
|
|
||||||
edited := make(chan *httptest.ResponseRecorder, 1)
|
edited := make(chan *httptest.ResponseRecorder, 1)
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
@@ -213,62 +197,6 @@ func TestHandleTargetDownload_StreamsWithoutTheLock(t *testing.T) {
|
|||||||
assert.Equal(t, http.StatusOK, sw.Code)
|
assert.Equal(t, http.StatusOK, sw.Code)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleTargetDownload_FindsFilesAfterRename proves a download
|
|
||||||
// finds each of the target's files again under the names stored when
|
|
||||||
// it reaches the file: the target is renamed while the download is
|
|
||||||
// stalled before it has opened any file, and the rows of both its files
|
|
||||||
// are in the download.
|
|
||||||
func TestHandleTargetDownload_FindsFilesAfterRename(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
wh := seedWebhookWithRetention(t, env.db, 7)
|
|
||||||
archive := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
|
||||||
oldPath := delivery.ArchivePath(env.dbMgr, &wh, archive)
|
|
||||||
|
|
||||||
month := func(path string) string {
|
|
||||||
return strings.TrimSuffix(path, ".db") + "-2026-10.db"
|
|
||||||
}
|
|
||||||
|
|
||||||
seedArchive(t, oldPath, 1, 16)
|
|
||||||
seedArchive(t, month(oldPath), 1, 16)
|
|
||||||
|
|
||||||
sw, downloaded := startStalledDownload(t, env, wh.ID, archive.ID)
|
|
||||||
|
|
||||||
require.Equal(t,
|
|
||||||
http.StatusSeeOther, renameTarget(env, wh.ID, archive.ID).Code,
|
|
||||||
)
|
|
||||||
|
|
||||||
// The test's archives record a rename without moving any file, so
|
|
||||||
// the files are moved here, as the delivery engine moves them.
|
|
||||||
var renamed database.Target
|
|
||||||
|
|
||||||
require.NoError(t, env.db.DB().First(&renamed, "id = ?", archive.ID).Error)
|
|
||||||
|
|
||||||
newPath := delivery.ArchivePath(env.dbMgr, &wh, &renamed)
|
|
||||||
|
|
||||||
require.NoError(t, os.Rename(oldPath, newPath))
|
|
||||||
require.NoError(t, os.Rename(month(oldPath), month(newPath)))
|
|
||||||
|
|
||||||
close(sw.resume)
|
|
||||||
<-downloaded
|
|
||||||
require.Equal(t, http.StatusOK, sw.Code)
|
|
||||||
|
|
||||||
zr, err := gzip.NewReader(sw.Body)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var (
|
|
||||||
got map[string]json.RawMessage
|
|
||||||
events []map[string]any
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, json.NewDecoder(zr).Decode(&got))
|
|
||||||
require.NoError(t, json.Unmarshal(got["archived_events"], &events))
|
|
||||||
require.Len(t, events, 2)
|
|
||||||
assert.NotContains(t, events[0], "period")
|
|
||||||
assert.Equal(t, "2026-10", events[1]["period"])
|
|
||||||
}
|
|
||||||
|
|
||||||
// seedArchive writes rows to the archive file at path, each with a
|
// seedArchive writes rows to the archive file at path, each with a
|
||||||
// body of bodySize random bytes, which do not compress. Its table has
|
// body of bodySize random bytes, which do not compress. Its table has
|
||||||
// only the columns the test fills; an export writes the others empty.
|
// only the columns the test fills; an export writes the others empty.
|
||||||
|
|||||||
@@ -83,7 +83,6 @@ func (h *Handlers) HandleTargetEdit() http.HandlerFunc {
|
|||||||
Timeout: cfg.Timeout,
|
Timeout: cfg.Timeout,
|
||||||
MaxRetries: strconv.Itoa(target.MaxRetries),
|
MaxRetries: strconv.Itoa(target.MaxRetries),
|
||||||
Expiry: cfg.Expiry,
|
Expiry: cfg.Expiry,
|
||||||
Rotation: cfg.Rotation,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
h.renderTargetEdit(
|
h.renderTargetEdit(
|
||||||
@@ -243,9 +242,6 @@ func (h *Handlers) renderTargetEdit(
|
|||||||
tmplKeyMaxTimeout: delivery.MaxTargetTimeoutSeconds,
|
tmplKeyMaxTimeout: delivery.MaxTargetTimeoutSeconds,
|
||||||
tmplKeyError: errMsg,
|
tmplKeyError: errMsg,
|
||||||
tmplKeyArchiveExpiryChoices: archiveExpiryOptions(form.Expiry),
|
tmplKeyArchiveExpiryChoices: archiveExpiryOptions(form.Expiry),
|
||||||
tmplKeyArchiveRotationChoices: archiveRotationOptions(
|
|
||||||
form.Rotation,
|
|
||||||
),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
h.renderTemplateStatus(w, r, targetEditTemplate, data, status)
|
h.renderTemplateStatus(w, r, targetEditTemplate, data, status)
|
||||||
|
|||||||
@@ -418,30 +418,6 @@ func TestHandleTargetEdit_PrefillsTheStoredValuesUnmasked(
|
|||||||
assert.Contains(t, page, "original-name")
|
assert.Contains(t, page, "original-name")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleTargetEdit_CallsTheDatabaseTypeArchive pins the names the
|
|
||||||
// edit page of a database target gives its type and its settings to
|
|
||||||
// the ones its badge and the add target form use.
|
|
||||||
func TestHandleTargetEdit_CallsTheDatabaseTypeArchive(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
webhook := seedWebhookWithRetention(t, env.db, 30)
|
|
||||||
target := seedTarget(t, env.db, webhook.ID, database.TargetTypeDatabase)
|
|
||||||
|
|
||||||
w := serveTarget(
|
|
||||||
env, http.MethodGet,
|
|
||||||
"/hook/"+webhook.ID+"/targets/"+target.ID+"/edit",
|
|
||||||
nil,
|
|
||||||
)
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
|
|
||||||
page := w.Body.String()
|
|
||||||
|
|
||||||
assert.Contains(t, page, "Type: archive.")
|
|
||||||
assert.Contains(t, page, `class="label">Archive expiry</label>`)
|
|
||||||
assert.Contains(t, page, `class="label">Archive rotation</label>`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleTargetEditSubmit_Rejects covers every submission that
|
// TestHandleTargetEditSubmit_Rejects covers every submission that
|
||||||
// must not reach storage.
|
// must not reach storage.
|
||||||
//
|
//
|
||||||
@@ -612,11 +588,10 @@ func TestHandleTargetEditSubmit_RefusedFormComesBack(t *testing.T) {
|
|||||||
{
|
{
|
||||||
database.TargetTypeSlack,
|
database.TargetTypeSlack,
|
||||||
"name=edited&url=" + editOriginalURL + "&max_retries=25",
|
"name=edited&url=" + editOriginalURL + "&max_retries=25",
|
||||||
"Invalid delivery attempts",
|
"Invalid max retries",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
database.TargetTypeDatabase,
|
database.TargetTypeDatabase, "name=edited&expiry=7d",
|
||||||
"name=edited&expiry=7d&rotation=daily",
|
|
||||||
"Invalid archive expiry",
|
"Invalid archive expiry",
|
||||||
},
|
},
|
||||||
{database.TargetTypeLog, "name=", "Name is required"},
|
{database.TargetTypeLog, "name=", "Name is required"},
|
||||||
@@ -638,15 +613,15 @@ func TestHandleTargetEditSubmit_RefusedFormComesBack(t *testing.T) {
|
|||||||
page := w.Body.String()
|
page := w.Body.String()
|
||||||
assert.Contains(t, page, `class="alert-error">`+tc.reason)
|
assert.Contains(t, page, `class="alert-error">`+tc.reason)
|
||||||
|
|
||||||
// headers is the form's one textarea, and expiry and
|
// headers is the form's one textarea and expiry its one
|
||||||
// rotation its selects; every other field is an input.
|
// select; every other field is an input.
|
||||||
for field := range form {
|
for field := range form {
|
||||||
shown := `name="` + field + `" value="` + form.Get(field) + `"`
|
shown := `name="` + field + `" value="` + form.Get(field) + `"`
|
||||||
|
|
||||||
switch field {
|
switch field {
|
||||||
case "headers":
|
case "headers":
|
||||||
shown = ">" + form.Get(field) + "</textarea>"
|
shown = ">" + form.Get(field) + "</textarea>"
|
||||||
case "expiry", "rotation":
|
case "expiry":
|
||||||
shown = `<option value="` + form.Get(field) + `" selected>`
|
shown = `<option value="` + form.Get(field) + `" selected>`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -22,8 +21,8 @@ type TargetRowView struct {
|
|||||||
// and is nil when the webhook's event database could not be read.
|
// and is nil when the webhook's event database could not be read.
|
||||||
Deliveries *TargetDeliveries
|
Deliveries *TargetDeliveries
|
||||||
|
|
||||||
// Archive is a database target's archive files, and nil for a
|
// Archive is a database target's archive file, and nil for a target
|
||||||
// target of any other type.
|
// of any other type.
|
||||||
Archive *ArchiveFileView
|
Archive *ArchiveFileView
|
||||||
|
|
||||||
// Paused is set while the target's circuit breaker is turning its
|
// Paused is set while the target's circuit breaker is turning its
|
||||||
@@ -117,23 +116,17 @@ type TargetDeliveries struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ArchiveFileView is what a database target's row shows about its
|
// ArchiveFileView is what a database target's row shows about its
|
||||||
// archive files.
|
// archive file.
|
||||||
type ArchiveFileView struct {
|
type ArchiveFileView struct {
|
||||||
// Name is the name of the file an event received now goes to.
|
|
||||||
Name string
|
Name string
|
||||||
|
|
||||||
// Note says that file does not exist yet, or that the files could
|
// Note stands in for the size and the last write when there are
|
||||||
// not be read, and is empty otherwise.
|
// none to show, and is empty when there are.
|
||||||
Note string
|
Note string
|
||||||
|
|
||||||
// Files counts the target's archive files, and is 0 when there are
|
// Size is the size on disk. Written is how long ago the file was
|
||||||
// none, or they could not be read, and so no size or last write to
|
// last written, and WrittenUTC the full time the page shows on
|
||||||
// show.
|
// hover.
|
||||||
Files int
|
|
||||||
|
|
||||||
// Size is the size on disk of all the files together. Written is
|
|
||||||
// how long ago the latest of them was last written, and WrittenUTC
|
|
||||||
// the full time the page shows on hover.
|
|
||||||
Size string
|
Size string
|
||||||
Written string
|
Written string
|
||||||
WrittenUTC string
|
WrittenUTC string
|
||||||
@@ -146,7 +139,6 @@ func (h *Handlers) targetRows(
|
|||||||
) []TargetRowView {
|
) []TargetRowView {
|
||||||
views := delivery.NewTargetViews(targets)
|
views := delivery.NewTargetViews(targets)
|
||||||
rows := make([]TargetRowView, len(views))
|
rows := make([]TargetRowView, len(views))
|
||||||
now := time.Now()
|
|
||||||
|
|
||||||
deliveries, err := h.loadTargetDeliveries(webhook.ID)
|
deliveries, err := h.loadTargetDeliveries(webhook.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -167,7 +159,7 @@ func (h *Handlers) targetRows(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if targets[i].Type == database.TargetTypeDatabase {
|
if targets[i].Type == database.TargetTypeDatabase {
|
||||||
rows[i].Archive = h.archiveFileView(webhook, &targets[i], now)
|
rows[i].Archive = h.archiveFileView(webhook, &targets[i])
|
||||||
}
|
}
|
||||||
|
|
||||||
rows[i].Paused = h.pausedView(targets[i].ID)
|
rows[i].Paused = h.pausedView(targets[i].ID)
|
||||||
@@ -234,57 +226,34 @@ func readTargetDeliveries(
|
|||||||
return byTarget, nil
|
return byTarget, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// archiveFileView describes a database target's archive files from
|
// archiveFileView describes a database target's archive file from the
|
||||||
// their metadata alone; the archive is never opened. It names the file
|
// file's metadata alone; the archive is never opened. The file is found
|
||||||
// an event received at now goes to. The files are found by the name
|
// by the name the archive writer uses, so it follows a rename of the
|
||||||
// the archive writer uses, so they follow a rename of the webhook or
|
// webhook or the target.
|
||||||
// the target.
|
|
||||||
func (h *Handlers) archiveFileView(
|
func (h *Handlers) archiveFileView(
|
||||||
webhook *database.Webhook, target *database.Target, now time.Time,
|
webhook *database.Webhook, target *database.Target,
|
||||||
) *ArchiveFileView {
|
) *ArchiveFileView {
|
||||||
current, err := delivery.ArchivePathAt(h.dbMgr, webhook, target, now)
|
path := delivery.ArchivePath(h.dbMgr, webhook, target)
|
||||||
if err != nil {
|
view := &ArchiveFileView{Name: filepath.Base(path)}
|
||||||
return h.archiveUnreadable(&ArchiveFileView{}, target, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
view := &ArchiveFileView{Name: filepath.Base(current)}
|
file, err := delivery.StatArchive(path)
|
||||||
|
|
||||||
_, statErr := os.Stat(current)
|
|
||||||
if errors.Is(statErr, fs.ErrNotExist) {
|
|
||||||
view.Note = "not created yet"
|
|
||||||
}
|
|
||||||
|
|
||||||
files, err := delivery.StatArchive(
|
|
||||||
delivery.ArchivePath(h.dbMgr, webhook, target),
|
|
||||||
)
|
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case errors.Is(err, fs.ErrNotExist):
|
case errors.Is(err, fs.ErrNotExist):
|
||||||
// No files: no size or last write to show.
|
view.Note = "not created yet"
|
||||||
case err != nil:
|
case err != nil:
|
||||||
return h.archiveUnreadable(view, target, err)
|
h.log.Error(
|
||||||
|
"failed to read archive file metadata",
|
||||||
|
"target_id", target.ID,
|
||||||
|
"error", err,
|
||||||
|
)
|
||||||
|
|
||||||
|
view.Note = "could not be read"
|
||||||
default:
|
default:
|
||||||
view.Files = files.Files
|
view.Size = humanize.Bytes(uint64(file.Size)) //nolint:gosec // never negative
|
||||||
view.Size = humanize.Bytes(uint64(files.Size)) //nolint:gosec // never negative
|
view.Written = humanize.Time(file.Written)
|
||||||
view.Written = humanize.Time(files.Written)
|
view.WrittenUTC = file.Written.UTC().Format(time.DateTime) + " UTC"
|
||||||
view.WrittenUTC = files.Written.UTC().Format(time.DateTime) + " UTC"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return view
|
return view
|
||||||
}
|
}
|
||||||
|
|
||||||
// archiveUnreadable logs why a database target's archive files could
|
|
||||||
// not be described, and returns view saying so.
|
|
||||||
func (h *Handlers) archiveUnreadable(
|
|
||||||
view *ArchiveFileView, target *database.Target, err error,
|
|
||||||
) *ArchiveFileView {
|
|
||||||
h.log.Error(
|
|
||||||
"failed to read archive file metadata",
|
|
||||||
"target_id", target.ID,
|
|
||||||
"error", err,
|
|
||||||
)
|
|
||||||
|
|
||||||
view.Note = "could not be read"
|
|
||||||
|
|
||||||
return view
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -44,10 +44,10 @@ func TestHandleSourceDetail_ShowsArchiveFile(t *testing.T) {
|
|||||||
path := delivery.ArchivePath(dbMgr, wh, archive)
|
path := delivery.ArchivePath(dbMgr, wh, archive)
|
||||||
|
|
||||||
body := renderSourceDetailPage(t, h, sess, wh.ID)
|
body := renderSourceDetailPage(t, h, sess, wh.ID)
|
||||||
assert.Equal(t, 1, strings.Count(body, "Archive file:"))
|
assert.Equal(t, 1, strings.Count(body, "Archive File:"))
|
||||||
assert.Contains(t, body, filepath.Base(path))
|
assert.Contains(t, body, filepath.Base(path))
|
||||||
assert.Contains(t, body, "not created yet")
|
assert.Contains(t, body, "not created yet")
|
||||||
assert.NotContains(t, body, "Archive size:")
|
assert.NotContains(t, body, "Archive Size:")
|
||||||
|
|
||||||
seedArchive(t, path, 1, 100)
|
seedArchive(t, path, 1, 100)
|
||||||
|
|
||||||
@@ -58,7 +58,7 @@ func TestHandleSourceDetail_ShowsArchiveFile(t *testing.T) {
|
|||||||
assert.Contains(t, body, filepath.Base(path))
|
assert.Contains(t, body, filepath.Base(path))
|
||||||
assert.NotContains(t, body, "not created yet")
|
assert.NotContains(t, body, "not created yet")
|
||||||
assert.Regexp(t,
|
assert.Regexp(t,
|
||||||
`Archive size:</span>\s*<span>[1-9][0-9.]* [kM]?B</span>`, body,
|
`Archive Size:</span>\s*<span>[1-9][0-9.]* [kM]?B</span>`, body,
|
||||||
)
|
)
|
||||||
assert.Contains(t, body,
|
assert.Contains(t, body,
|
||||||
`title="`+file.ModTime().UTC().Format(time.DateTime)+` UTC"`,
|
`title="`+file.ModTime().UTC().Format(time.DateTime)+` UTC"`,
|
||||||
@@ -69,7 +69,7 @@ func TestHandleSourceDetail_ShowsArchiveFile(t *testing.T) {
|
|||||||
body = renderSourceDetailPage(t, h, sess, wh.ID)
|
body = renderSourceDetailPage(t, h, sess, wh.ID)
|
||||||
assert.Contains(t, body, filepath.Base(path))
|
assert.Contains(t, body, filepath.Base(path))
|
||||||
assert.Contains(t, body, "not created yet")
|
assert.Contains(t, body, "not created yet")
|
||||||
assert.NotContains(t, body, "Archive size:")
|
assert.NotContains(t, body, "Archive Size:")
|
||||||
}
|
}
|
||||||
|
|
||||||
// targetList returns the text of the targets section in a rendered
|
// targetList returns the text of the targets section in a rendered
|
||||||
|
|||||||
@@ -81,18 +81,17 @@ func TestPausedTarget_ShownUntilBreakerCloses(t *testing.T) {
|
|||||||
|
|
||||||
list := targetList(t, renderSourceDetailPage(t, h, sess, wh.ID))
|
list := targetList(t, renderSourceDetailPage(t, h, sess, wh.ID))
|
||||||
assert.Regexp(t, "t-http http Active Edit Deactivate Delete "+
|
assert.Regexp(t, "t-http http Active Edit Deactivate Delete "+
|
||||||
"Deliveries paused: after repeated failures, until "+cooldownEnds+
|
"Deliveries Paused: after repeated failures, until "+cooldownEnds+
|
||||||
", then one waiting delivery is sent to test the target while "+
|
", then one waiting delivery is sent to test the target while "+
|
||||||
"the others wait at least one more cooldown", list)
|
"the others wait at least one more cooldown", list)
|
||||||
assert.Equal(t, 1, strings.Count(list, "Deliveries paused"))
|
assert.Equal(t, 1, strings.Count(list, "Paused"))
|
||||||
|
|
||||||
log := renderSourceLogsPage(t, h, sess, wh.ID)
|
log := renderSourceLogsPage(t, h, sess, wh.ID)
|
||||||
assert.Equal(t, 2, strings.Count(log, "t-http: waiting"))
|
assert.Equal(t, 2, strings.Count(log, "t-http: waiting"))
|
||||||
assert.Contains(t, log, "t-http: delivered")
|
assert.Contains(t, log, "t-http: delivered")
|
||||||
assert.Regexp(t, waiting+cooldownEnds, log)
|
assert.Regexp(t, waiting+cooldownEnds, log)
|
||||||
assert.Contains(t, log, waiting+backoffEnds)
|
assert.Contains(t, log, waiting+backoffEnds)
|
||||||
// No delivery shows as retrying; the Pending link's title says it.
|
assert.NotContains(t, log, "retrying")
|
||||||
assert.NotRegexp(t, `t-http: retrying|>retrying<`, log)
|
|
||||||
|
|
||||||
page := eventPage(t, h, sess, wh.ID, retrying.ID)
|
page := eventPage(t, h, sess, wh.ID, retrying.ID)
|
||||||
assert.Regexp(t, waiting+cooldownEnds, page)
|
assert.Regexp(t, waiting+cooldownEnds, page)
|
||||||
@@ -106,30 +105,20 @@ func TestPausedTarget_ShownUntilBreakerCloses(t *testing.T) {
|
|||||||
|
|
||||||
list = targetList(t, renderSourceDetailPage(t, h, sess, wh.ID))
|
list = targetList(t, renderSourceDetailPage(t, h, sess, wh.ID))
|
||||||
assert.Contains(t, list, "t-http http Active Edit Deactivate Delete "+
|
assert.Contains(t, list, "t-http http Active Edit Deactivate Delete "+
|
||||||
"Deliveries paused: held while one delivery tests whether the "+
|
"Deliveries Paused: held while one delivery tests whether the "+
|
||||||
"target has recovered")
|
"target has recovered")
|
||||||
// Not the whole list: the add target form above the rows says UTC.
|
assert.NotContains(t, list, "UTC")
|
||||||
assert.NotContains(t, targetRow(list, "t-http", "t-log"), "UTC")
|
|
||||||
|
|
||||||
assertRetryingNotWaiting(t, h, sess, wh.ID, retrying, backedOff)
|
assertRetryingNotWaiting(t, h, sess, wh.ID, retrying, backedOff)
|
||||||
|
|
||||||
breakers.Set(target.ID, delivery.CircuitClosed, 0)
|
breakers.Set(target.ID, delivery.CircuitClosed, 0)
|
||||||
|
|
||||||
list = targetList(t, renderSourceDetailPage(t, h, sess, wh.ID))
|
list = targetList(t, renderSourceDetailPage(t, h, sess, wh.ID))
|
||||||
assert.NotContains(t, list, "Deliveries paused")
|
assert.NotContains(t, list, "Paused")
|
||||||
|
|
||||||
assertRetryingNotWaiting(t, h, sess, wh.ID, retrying, backedOff)
|
assertRetryingNotWaiting(t, h, sess, wh.ID, retrying, backedOff)
|
||||||
}
|
}
|
||||||
|
|
||||||
// targetRow returns the row of the target named name in a targetList:
|
|
||||||
// from its name to the name of the target listed after it, next.
|
|
||||||
func targetRow(list, name, next string) string {
|
|
||||||
_, row, _ := strings.Cut(list, name+" ")
|
|
||||||
row, _, _ = strings.Cut(row, next+" ")
|
|
||||||
|
|
||||||
return row
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertRetryingNotWaiting checks that the event log and each event's
|
// assertRetryingNotWaiting checks that the event log and each event's
|
||||||
// page show the http target's delivery of the event as retrying, and
|
// page show the http target's delivery of the event as retrying, and
|
||||||
// none of them as waiting.
|
// none of them as waiting.
|
||||||
|
|||||||
@@ -25,14 +25,14 @@ var (
|
|||||||
// errRetriesInvalid signals a max_retries form value that is not
|
// errRetriesInvalid signals a max_retries form value that is not
|
||||||
// a non-negative whole number.
|
// a non-negative whole number.
|
||||||
errRetriesInvalid = errors.New(
|
errRetriesInvalid = errors.New(
|
||||||
"must be a whole number",
|
"retries must be a whole number of attempts",
|
||||||
)
|
)
|
||||||
|
|
||||||
// errRetriesTooLarge signals a max_retries form value that is a
|
// errRetriesTooLarge signals a max_retries form value that is a
|
||||||
// whole number but above maxTargetRetries. It is distinguished
|
// whole number but above maxTargetRetries. It is distinguished
|
||||||
// from errRetriesInvalid so the message can name the ceiling
|
// from errRetriesInvalid so the message can name the ceiling
|
||||||
// instead of implying the input was not a number.
|
// instead of implying the input was not a number.
|
||||||
errRetriesTooLarge = errors.New("out of range")
|
errRetriesTooLarge = errors.New("retries out of range")
|
||||||
)
|
)
|
||||||
|
|
||||||
// parseMaxRetries interprets a max_retries form value.
|
// parseMaxRetries interprets a max_retries form value.
|
||||||
@@ -82,7 +82,7 @@ func retriesErrorMessage(err error) string {
|
|||||||
if errors.Is(err, errRetriesTooLarge) {
|
if errors.Is(err, errRetriesTooLarge) {
|
||||||
return errRetriesTooLarge.Error() +
|
return errRetriesTooLarge.Error() +
|
||||||
": at most " + strconv.Itoa(maxTargetRetries) +
|
": at most " + strconv.Itoa(maxTargetRetries) +
|
||||||
" attempts"
|
" retries"
|
||||||
}
|
}
|
||||||
|
|
||||||
return errRetriesInvalid.Error() +
|
return errRetriesInvalid.Error() +
|
||||||
|
|||||||
@@ -1,35 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// HandleTargetToggle handles toggling a target's active state.
|
|
||||||
func (h *Handlers) HandleTargetToggle() http.HandlerFunc {
|
|
||||||
return h.toggleChildResource(
|
|
||||||
"targetID",
|
|
||||||
func(webhookID, childID string) (bool, error) {
|
|
||||||
var tgt database.Target
|
|
||||||
|
|
||||||
err := h.db.DB().Where(
|
|
||||||
"id = ? AND webhook_id = ?",
|
|
||||||
childID, webhookID,
|
|
||||||
).First(&tgt).Error
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Only the active column: saving the whole row would
|
|
||||||
// write back the name and settings read above over an
|
|
||||||
// edit saved since.
|
|
||||||
active := !tgt.Active
|
|
||||||
|
|
||||||
return active, h.db.DB().Model(&tgt).
|
|
||||||
Update("active", active).Error
|
|
||||||
},
|
|
||||||
"failed to toggle target",
|
|
||||||
targetActivated, targetDeactivated,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -354,14 +354,15 @@ func TestEntrypointCopyButtonIsProgressiveEnhancement(t *testing.T) {
|
|||||||
// and target_http gives up once the attempt number reaches
|
// and target_http gives up once the attempt number reaches
|
||||||
// max_retries), and 0 is special-cased to a single fire-and-forget
|
// max_retries), and 0 is special-cased to a single fire-and-forget
|
||||||
// attempt with no circuit breaker.
|
// attempt with no circuit breaker.
|
||||||
const maxRetriesHelp = "How many times each delivery is attempted in " +
|
const maxRetriesHelp = "This is the total number of delivery attempts, " +
|
||||||
"all, the first attempt included. 0 means a single attempt with no " +
|
"not retries on top of the first: a value of 3 makes three attempts " +
|
||||||
"retries and no circuit breaker."
|
"in all. 0 means a single attempt with no retries and no circuit " +
|
||||||
|
"breaker."
|
||||||
|
|
||||||
// TestTargetFormMaxRetriesCopyMatchesBehaviour pins the max_retries
|
// TestTargetFormMaxRetriesCopyMatchesBehaviour pins the max_retries
|
||||||
// label, "Delivery attempts", and help text on both the create form
|
// help text on both the create form (the add-target form on the webhook
|
||||||
// (the add-target form on the webhook detail page) and the edit form,
|
// detail page) and the edit form, so the copy cannot drift back to
|
||||||
// so the copy cannot drift back to calling the number a retry count.
|
// calling the number a retry count.
|
||||||
func TestTargetFormMaxRetriesCopyMatchesBehaviour(t *testing.T) {
|
func TestTargetFormMaxRetriesCopyMatchesBehaviour(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -392,7 +393,6 @@ func TestTargetFormMaxRetriesCopyMatchesBehaviour(t *testing.T) {
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Contains(t, createBody, "Delivery attempts:</label>")
|
|
||||||
assert.Contains(
|
assert.Contains(
|
||||||
t, createBody, maxRetriesHelp,
|
t, createBody, maxRetriesHelp,
|
||||||
"the add-target form must explain max_retries as total attempts",
|
"the add-target form must explain max_retries as total attempts",
|
||||||
@@ -419,33 +419,8 @@ func TestTargetFormMaxRetriesCopyMatchesBehaviour(t *testing.T) {
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Contains(t, editBody, `class="label">Delivery attempts</label>`)
|
|
||||||
assert.Contains(
|
assert.Contains(
|
||||||
t, editBody, maxRetriesHelp,
|
t, editBody, maxRetriesHelp,
|
||||||
"the target edit form must explain max_retries as total attempts",
|
"the target edit form must explain max_retries as total attempts",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestCreateFormCallsTheDatabaseTargetAnArchive pins the names the new
|
|
||||||
// webhook page gives the database target its Archive checkbox creates,
|
|
||||||
// and that target's settings, to the ones the target forms use.
|
|
||||||
func TestCreateFormCallsTheDatabaseTargetAnArchive(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var h *handlers.Handlers
|
|
||||||
|
|
||||||
var sess *session.Session
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
body := renderPage(t, h, sess, "sources_new.html", map[string]any{
|
|
||||||
dataKeyError: "",
|
|
||||||
})
|
|
||||||
|
|
||||||
assert.Contains(t, body, "created with an archive target")
|
|
||||||
assert.Contains(t, body, `class="label">Archive expiry</label>`)
|
|
||||||
assert.Contains(t, body, `class="label">Archive rotation</label>`)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,263 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net/http"
|
|
||||||
"strconv"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// HandleSourceCreate shows the form to create a new webhook.
|
|
||||||
func (h *Handlers) HandleSourceCreate() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
h.renderTemplate(
|
|
||||||
w, r, "sources_new.html",
|
|
||||||
newSourceFormData("", sourceFormInput{
|
|
||||||
RetentionDays: strconv.Itoa(
|
|
||||||
database.DefaultRetentionDays,
|
|
||||||
),
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// sourceFormInput carries the raw values of the new webhook form. A
|
|
||||||
// refused submission is shown again from it, so every value entered
|
|
||||||
// comes back, retention included.
|
|
||||||
type sourceFormInput struct {
|
|
||||||
Name string
|
|
||||||
Description string
|
|
||||||
RetentionDays string
|
|
||||||
// HTTPURL, when not empty, asks for an HTTP target with this
|
|
||||||
// destination.
|
|
||||||
HTTPURL string
|
|
||||||
// Archive asks for a database (archive) target, whose rows expire
|
|
||||||
// after ArchiveExpiry and whose files rotate by ArchiveRotation.
|
|
||||||
Archive bool
|
|
||||||
ArchiveExpiry string
|
|
||||||
ArchiveRotation string
|
|
||||||
}
|
|
||||||
|
|
||||||
// newSourceFormData builds the template data for the webhook creation
|
|
||||||
// form. It carries the retention default, which the form's help text
|
|
||||||
// names, from database.DefaultRetentionDays rather than a hardcoded
|
|
||||||
// copy of the same policy.
|
|
||||||
func newSourceFormData(
|
|
||||||
errMsg string, in sourceFormInput,
|
|
||||||
) map[string]any {
|
|
||||||
return map[string]any{
|
|
||||||
tmplKeyError: errMsg,
|
|
||||||
"Form": in,
|
|
||||||
"DefaultRetentionDays": database.DefaultRetentionDays,
|
|
||||||
tmplKeyArchiveExpiryChoices: archiveExpiryOptions(
|
|
||||||
in.ArchiveExpiry,
|
|
||||||
),
|
|
||||||
tmplKeyArchiveRotationChoices: archiveRotationOptions(
|
|
||||||
in.ArchiveRotation,
|
|
||||||
),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleSourceCreateSubmit handles the webhook creation form
|
|
||||||
// submission.
|
|
||||||
func (h *Handlers) HandleSourceCreateSubmit() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
userID, ok := h.getUserID(r)
|
|
||||||
if !ok {
|
|
||||||
http.Redirect(
|
|
||||||
w, r, "/pages/login", http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// The body size cap is enforced by the MaxBodySize
|
|
||||||
// middleware, which runs before CSRF parses the form.
|
|
||||||
err := r.ParseForm()
|
|
||||||
if err != nil {
|
|
||||||
h.renderError(w, r, http.StatusBadRequest)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
in := sourceFormInput{
|
|
||||||
Name: r.PostFormValue("name"),
|
|
||||||
Description: r.PostFormValue("description"),
|
|
||||||
RetentionDays: r.PostFormValue("retention_days"),
|
|
||||||
HTTPURL: r.PostFormValue("http_url"),
|
|
||||||
Archive: r.PostFormValue("archive") != "",
|
|
||||||
ArchiveExpiry: r.PostFormValue("archive_expiry"),
|
|
||||||
ArchiveRotation: r.PostFormValue("archive_rotation"),
|
|
||||||
}
|
|
||||||
|
|
||||||
refuse := func(errMsg string) {
|
|
||||||
h.renderTemplateStatus(
|
|
||||||
w, r, "sources_new.html",
|
|
||||||
newSourceFormData(errMsg, in),
|
|
||||||
http.StatusBadRequest,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if in.Name == "" {
|
|
||||||
refuse("Name is required")
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
retentionDays, errMsg := parseRetentionDays(
|
|
||||||
in.RetentionDays, database.DefaultRetentionDays,
|
|
||||||
)
|
|
||||||
if errMsg != "" {
|
|
||||||
refuse(errMsg)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
targets, errMsg, err := h.newWebhookTargets(r.Context(), in)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, r, "failed to encode target config", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if errMsg != "" {
|
|
||||||
refuse(errMsg)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.createWebhookWithEntrypoint(w, r, &database.Webhook{
|
|
||||||
UserID: userID,
|
|
||||||
Name: in.Name,
|
|
||||||
Description: in.Description,
|
|
||||||
RetentionDays: retentionDays,
|
|
||||||
}, targets)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// newWebhookTargets validates the targets the new webhook form asks
|
|
||||||
// for and returns the rows to create with the webhook, or the message
|
|
||||||
// the form shows for the first one it refuses. A filled-in HTTP URL
|
|
||||||
// asks for an HTTP target named "HTTP", and the archive checkbox for a
|
|
||||||
// database target named "Archive". Each goes through newTarget, as on
|
|
||||||
// the webhook page's add target form. The rows have no WebhookID yet:
|
|
||||||
// the webhook has no ID until it is created.
|
|
||||||
func (h *Handlers) newWebhookTargets(
|
|
||||||
ctx context.Context,
|
|
||||||
in sourceFormInput,
|
|
||||||
) ([]*database.Target, string, error) {
|
|
||||||
var requested []targetFormInput
|
|
||||||
|
|
||||||
if in.HTTPURL != "" {
|
|
||||||
requested = append(requested, targetFormInput{
|
|
||||||
Name: "HTTP",
|
|
||||||
Type: database.TargetTypeHTTP,
|
|
||||||
URL: in.HTTPURL,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if in.Archive {
|
|
||||||
requested = append(requested, targetFormInput{
|
|
||||||
Name: "Archive",
|
|
||||||
Type: database.TargetTypeDatabase,
|
|
||||||
Expiry: in.ArchiveExpiry,
|
|
||||||
Rotation: in.ArchiveRotation,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
targets := make([]*database.Target, 0, len(requested))
|
|
||||||
|
|
||||||
for _, form := range requested {
|
|
||||||
target, errMsg, err := h.newTarget(ctx, "", form)
|
|
||||||
if err != nil || errMsg != "" {
|
|
||||||
return nil, errMsg, err
|
|
||||||
}
|
|
||||||
|
|
||||||
targets = append(targets, target)
|
|
||||||
}
|
|
||||||
|
|
||||||
return targets, "", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// createWebhookWithEntrypoint creates a webhook, its default
|
|
||||||
// entrypoint and the given targets in a transaction.
|
|
||||||
func (h *Handlers) createWebhookWithEntrypoint(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
r *http.Request,
|
|
||||||
webhook *database.Webhook,
|
|
||||||
targets []*database.Target,
|
|
||||||
) {
|
|
||||||
err := h.commitWebhook(webhook, targets)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, r, "failed to create webhook", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
err = h.dbMgr.CreateDB(webhook.ID)
|
|
||||||
if err != nil {
|
|
||||||
h.log.Error(
|
|
||||||
"failed to create webhook event database",
|
|
||||||
"webhook_id", webhook.ID, "error", err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
h.log.Info("webhook created",
|
|
||||||
"webhook_id", webhook.ID,
|
|
||||||
"name", webhook.Name, "user_id", webhook.UserID,
|
|
||||||
)
|
|
||||||
|
|
||||||
http.Redirect(
|
|
||||||
w, r, withNotice("/hook/"+webhook.ID, webhookCreated),
|
|
||||||
http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// commitWebhook creates a webhook, its default entrypoint and the
|
|
||||||
// given targets in a transaction. Returns an error on failure (rolls
|
|
||||||
// back).
|
|
||||||
func (h *Handlers) commitWebhook(
|
|
||||||
webhook *database.Webhook,
|
|
||||||
targets []*database.Target,
|
|
||||||
) error {
|
|
||||||
tx := h.db.DB().Begin()
|
|
||||||
if tx.Error != nil {
|
|
||||||
return tx.Error
|
|
||||||
}
|
|
||||||
|
|
||||||
err := tx.Create(webhook).Error
|
|
||||||
if err != nil {
|
|
||||||
tx.Rollback()
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
entrypoint := &database.Entrypoint{
|
|
||||||
WebhookID: webhook.ID,
|
|
||||||
Path: uuid.New().String(),
|
|
||||||
Description: "Default entrypoint",
|
|
||||||
Active: true,
|
|
||||||
}
|
|
||||||
|
|
||||||
err = tx.Create(entrypoint).Error
|
|
||||||
if err != nil {
|
|
||||||
tx.Rollback()
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, target := range targets {
|
|
||||||
target.WebhookID = webhook.ID
|
|
||||||
|
|
||||||
err = tx.Create(target).Error
|
|
||||||
if err != nil {
|
|
||||||
tx.Rollback()
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return tx.Commit().Error
|
|
||||||
}
|
|
||||||
@@ -1,170 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"net/http"
|
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// HandleSourceDelete handles webhook deletion.
|
|
||||||
func (h *Handlers) HandleSourceDelete() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
userID, ok := h.getUserID(r)
|
|
||||||
if !ok {
|
|
||||||
http.Redirect(
|
|
||||||
w, r, "/pages/login", http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
sourceID := chi.URLParam(r, "sourceID")
|
|
||||||
|
|
||||||
var webhook database.Webhook
|
|
||||||
|
|
||||||
err := h.db.DB().Where(
|
|
||||||
"id = ? AND user_id = ?", sourceID, userID,
|
|
||||||
).First(&webhook).Error
|
|
||||||
if err != nil {
|
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.deleteWebhookResources(w, r, webhook, userID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The messages deleteWebhookResources logs when a file of the event
|
|
||||||
// database cannot be removed: the database file itself, or only a
|
|
||||||
// sidecar once the database file is gone.
|
|
||||||
const (
|
|
||||||
eventDBLeftMsg = "webhook deleted, but its event database file is " +
|
|
||||||
"still on disk; remove it by hand"
|
|
||||||
sidecarLeftMsg = "webhook deleted and its events are gone, but a " +
|
|
||||||
"-wal or -shm sidecar of its event database is " +
|
|
||||||
"still on disk; remove it by hand"
|
|
||||||
)
|
|
||||||
|
|
||||||
// deleteWebhookResources soft-deletes config and hard-deletes
|
|
||||||
// the per-webhook event database.
|
|
||||||
func (h *Handlers) deleteWebhookResources(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
r *http.Request,
|
|
||||||
webhook database.Webhook,
|
|
||||||
userID string,
|
|
||||||
) {
|
|
||||||
// The configuration delete commits before the event database
|
|
||||||
// is touched. No transaction spans the main database and the
|
|
||||||
// filesystem, so one side has to go first: committing the
|
|
||||||
// configuration first means a later failure leaves an unused
|
|
||||||
// event database file on disk, while removing the event
|
|
||||||
// database first would mean a failed commit destroys the
|
|
||||||
// history of a webhook that still exists. A leftover file can
|
|
||||||
// be removed by hand; deleted history cannot be recovered.
|
|
||||||
err := h.commitWebhookDeletion(&webhook)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, r, "failed to delete webhook", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.log.Info(
|
|
||||||
"webhook deleted",
|
|
||||||
"webhook_id", webhook.ID,
|
|
||||||
"user_id", userID,
|
|
||||||
)
|
|
||||||
|
|
||||||
// Release the delivery engine's per-webhook archiving state
|
|
||||||
// so a deleted webhook's archive writer (and any handle open
|
|
||||||
// within its debounce window) does not linger for the
|
|
||||||
// process lifetime. The archive file itself is deliberately
|
|
||||||
// left on disk; see evictArchiveWriter.
|
|
||||||
h.evictArchiveWriter(webhook.ID)
|
|
||||||
|
|
||||||
err = h.dbMgr.DeleteDB(webhook.ID)
|
|
||||||
if err != nil {
|
|
||||||
// The configuration is committed, so the webhook is gone,
|
|
||||||
// but a file of its event database is still on disk with
|
|
||||||
// nothing referencing it. Report the failure rather than
|
|
||||||
// redirecting as though everything succeeded: the file
|
|
||||||
// needs removing by hand, and the logged error names it.
|
|
||||||
// When only a sidecar is left, the events are already
|
|
||||||
// gone, and the message must not suggest they survive.
|
|
||||||
msg := eventDBLeftMsg
|
|
||||||
if errors.Is(err, database.ErrSidecarNotRemoved) {
|
|
||||||
msg = sidecarLeftMsg
|
|
||||||
}
|
|
||||||
|
|
||||||
h.serverError(w, r, msg, err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
http.Redirect(
|
|
||||||
w, r, withNotice("/hooks", webhookDeleted), http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// commitWebhookDeletion soft-deletes a webhook's entrypoints,
|
|
||||||
// targets and the webhook row in one transaction. Every
|
|
||||||
// statement is checked and any failure rolls the whole
|
|
||||||
// transaction back, so a caller that gets an error knows the
|
|
||||||
// configuration is untouched and the event database must be
|
|
||||||
// left alone.
|
|
||||||
func (h *Handlers) commitWebhookDeletion(
|
|
||||||
webhook *database.Webhook,
|
|
||||||
) error {
|
|
||||||
tx := h.db.DB().Begin()
|
|
||||||
if tx.Error != nil {
|
|
||||||
return tx.Error
|
|
||||||
}
|
|
||||||
|
|
||||||
err := tx.Where(
|
|
||||||
"webhook_id = ?", webhook.ID,
|
|
||||||
).Delete(&database.Entrypoint{}).Error
|
|
||||||
if err != nil {
|
|
||||||
tx.Rollback()
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = tx.Where(
|
|
||||||
"webhook_id = ?", webhook.ID,
|
|
||||||
).Delete(&database.Target{}).Error
|
|
||||||
if err != nil {
|
|
||||||
tx.Rollback()
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = tx.Delete(webhook).Error
|
|
||||||
if err != nil {
|
|
||||||
tx.Rollback()
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return tx.Commit().Error
|
|
||||||
}
|
|
||||||
|
|
||||||
// evictArchiveWriter asks the delivery engine to drop the cached
|
|
||||||
// archive writers of a webhook's database targets, closing their
|
|
||||||
// archive file handles.
|
|
||||||
//
|
|
||||||
// The archive database files are NOT deleted. Unlike the event
|
|
||||||
// database — which is per-webhook working storage and is
|
|
||||||
// hard-deleted with the webhook — an archive is explicitly
|
|
||||||
// long-term storage that an operator may want to keep or move
|
|
||||||
// away for offline retention. Destroying it as a side effect of
|
|
||||||
// deleting a webhook would be a surprising and unrecoverable
|
|
||||||
// data loss, so the file is left for the operator to handle.
|
|
||||||
func (h *Handlers) evictArchiveWriter(webhookID string) {
|
|
||||||
if h.archives == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.archives.EvictWebhook(webhookID)
|
|
||||||
}
|
|
||||||
@@ -1,133 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/reqtls"
|
|
||||||
)
|
|
||||||
|
|
||||||
// HandleSourceDetail shows details for a specific webhook.
|
|
||||||
func (h *Handlers) HandleSourceDetail() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
userID, ok := h.getUserID(r)
|
|
||||||
if !ok {
|
|
||||||
http.Redirect(
|
|
||||||
w, r, "/pages/login", http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
sourceID := chi.URLParam(r, "sourceID")
|
|
||||||
|
|
||||||
var webhook database.Webhook
|
|
||||||
|
|
||||||
err := h.db.DB().Where(
|
|
||||||
"id = ? AND user_id = ?", sourceID, userID,
|
|
||||||
).First(&webhook).Error
|
|
||||||
if err != nil {
|
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.renderSourceDetail(w, r, webhook, targetFormInput{}, "")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// renderSourceDetail loads and renders a source detail page. With a
|
|
||||||
// targetErr, it is the page shown again for a refused add target
|
|
||||||
// form: it answers 400, and the form opens on targetForm's type with
|
|
||||||
// its values and the message.
|
|
||||||
func (h *Handlers) renderSourceDetail(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
r *http.Request,
|
|
||||||
webhook database.Webhook,
|
|
||||||
targetForm targetFormInput,
|
|
||||||
targetErr string,
|
|
||||||
) {
|
|
||||||
var entrypoints []database.Entrypoint
|
|
||||||
|
|
||||||
h.db.DB().Where(
|
|
||||||
"webhook_id = ?", webhook.ID,
|
|
||||||
).Find(&entrypoints)
|
|
||||||
|
|
||||||
var targets []database.Target
|
|
||||||
|
|
||||||
h.db.DB().Where(
|
|
||||||
"webhook_id = ?", webhook.ID,
|
|
||||||
).Find(&targets)
|
|
||||||
|
|
||||||
entrypointViews := NewEntrypointViews(entrypoints)
|
|
||||||
|
|
||||||
var events []RecentEventView
|
|
||||||
|
|
||||||
if h.dbMgr.DBExists(webhook.ID) {
|
|
||||||
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, r, "failed to get webhook database", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
events, err = loadRecentEvents(
|
|
||||||
webhookDB, webhook.ID, singleHTTPTargetID(targets),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, r, "failed to load recent events", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
err = addEntrypointEvents(
|
|
||||||
webhookDB, &webhook, entrypointViews, time.Now(),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, r, "failed to count entrypoint events", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
scheme := "http"
|
|
||||||
if reqtls.IsTLS(r) {
|
|
||||||
scheme = "https"
|
|
||||||
}
|
|
||||||
|
|
||||||
// The host is the client's Host header, unvalidated. It is
|
|
||||||
// inert only because source_detail.html renders BaseURL as
|
|
||||||
// text, inside a <code> element and in an entrypoint's delete
|
|
||||||
// prompt; putting it in an href or any other URL context
|
|
||||||
// needs it constrained first.
|
|
||||||
baseURL := scheme + "://" + r.Host
|
|
||||||
|
|
||||||
// The template calls Webhook methods, which take pointer
|
|
||||||
// receivers; html/template cannot address a value stored in a map.
|
|
||||||
data := map[string]any{
|
|
||||||
tmplKeyWebhook: &webhook,
|
|
||||||
// Targets are projected to a display-safe view: a
|
|
||||||
// target's stored config blob holds a credential, and it
|
|
||||||
// must never reach a template.
|
|
||||||
"Entrypoints": entrypointViews,
|
|
||||||
"Targets": h.targetRows(&webhook, targets),
|
|
||||||
"Events": events,
|
|
||||||
"BaseURL": baseURL,
|
|
||||||
"Stats": h.loadWebhookStats(webhook.ID, entrypoints, targets),
|
|
||||||
tmplKeyTargetForm: targetForm,
|
|
||||||
"TargetError": targetErr,
|
|
||||||
// The add target form's selects start on its expiry and
|
|
||||||
// rotation through Alpine, so no choice is selected here.
|
|
||||||
tmplKeyArchiveExpiryChoices: archiveExpiryChoices(),
|
|
||||||
tmplKeyArchiveRotationChoices: archiveRotationChoices(),
|
|
||||||
}
|
|
||||||
|
|
||||||
status := http.StatusOK
|
|
||||||
if targetErr != "" {
|
|
||||||
status = http.StatusBadRequest
|
|
||||||
}
|
|
||||||
|
|
||||||
h.renderTemplateStatus(w, r, "source_detail.html", data, status)
|
|
||||||
}
|
|
||||||
@@ -1,245 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"net/http"
|
|
||||||
"strconv"
|
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// HandleSourceEdit shows the form to edit a webhook.
|
|
||||||
func (h *Handlers) HandleSourceEdit() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
userID, ok := h.getUserID(r)
|
|
||||||
if !ok {
|
|
||||||
http.Redirect(
|
|
||||||
w, r, "/pages/login", http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
sourceID := chi.URLParam(r, "sourceID")
|
|
||||||
|
|
||||||
var webhook database.Webhook
|
|
||||||
|
|
||||||
err := h.db.DB().Where(
|
|
||||||
"id = ? AND user_id = ?", sourceID, userID,
|
|
||||||
).First(&webhook).Error
|
|
||||||
if err != nil {
|
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.renderWebhookEdit(
|
|
||||||
w, r, &webhook,
|
|
||||||
webhook.Name, webhook.Description,
|
|
||||||
strconv.Itoa(webhook.RetentionDays),
|
|
||||||
"", http.StatusOK,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleSourceEditSubmit handles the webhook edit form
|
|
||||||
// submission.
|
|
||||||
func (h *Handlers) HandleSourceEditSubmit() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
userID, ok := h.getUserID(r)
|
|
||||||
if !ok {
|
|
||||||
http.Redirect(
|
|
||||||
w, r, "/pages/login", http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
sourceID := chi.URLParam(r, "sourceID")
|
|
||||||
|
|
||||||
h.renameMu.Lock()
|
|
||||||
defer h.renameMu.Unlock()
|
|
||||||
|
|
||||||
var webhook database.Webhook
|
|
||||||
|
|
||||||
err := h.db.DB().Where(
|
|
||||||
"id = ? AND user_id = ?", sourceID, userID,
|
|
||||||
).First(&webhook).Error
|
|
||||||
if err != nil {
|
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// The body size cap is enforced by the MaxBodySize
|
|
||||||
// middleware, which runs before CSRF parses the form.
|
|
||||||
err = r.ParseForm()
|
|
||||||
if err != nil {
|
|
||||||
h.renderError(w, r, http.StatusBadRequest)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.applyWebhookEdit(w, r, &webhook)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// applyWebhookEdit validates and saves webhook edits. A refused save
|
|
||||||
// shows the edit form again with the values submitted and the reason.
|
|
||||||
func (h *Handlers) applyWebhookEdit(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
r *http.Request,
|
|
||||||
webhook *database.Webhook,
|
|
||||||
) {
|
|
||||||
// The body size cap is enforced by the MaxBodySize middleware,
|
|
||||||
// which runs before CSRF parses the form.
|
|
||||||
name := r.PostFormValue("name")
|
|
||||||
description := r.PostFormValue("description")
|
|
||||||
retention := r.PostFormValue("retention_days")
|
|
||||||
|
|
||||||
if name == "" {
|
|
||||||
h.renderWebhookEdit(
|
|
||||||
w, r, webhook, name, description, retention,
|
|
||||||
"Name is required", http.StatusBadRequest,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// An empty field falls back to the stored value, so submitting the
|
|
||||||
// form without touching retention leaves the policy alone.
|
|
||||||
retentionDays, errMsg := parseRetentionDays(
|
|
||||||
retention, webhook.RetentionDays,
|
|
||||||
)
|
|
||||||
if errMsg != "" {
|
|
||||||
h.renderWebhookEdit(
|
|
||||||
w, r, webhook, name, description, retention,
|
|
||||||
errMsg, http.StatusBadRequest,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// edited is the webhook as the submission leaves it; webhook stays
|
|
||||||
// as stored, for the page shown again when the save is refused.
|
|
||||||
edited := *webhook
|
|
||||||
edited.Name = name
|
|
||||||
edited.Description = description
|
|
||||||
edited.RetentionDays = retentionDays
|
|
||||||
|
|
||||||
// A new name renames the archive files before it is saved (see
|
|
||||||
// delivery.Engine.Rename). If either step fails, the same targets'
|
|
||||||
// archives go back to the name that is still stored, without
|
|
||||||
// reading the main database again.
|
|
||||||
targets, err := h.renameWebhookArchives(
|
|
||||||
webhook.ID, webhook.Name, edited.Name,
|
|
||||||
)
|
|
||||||
if err == nil {
|
|
||||||
err = h.db.DB().Save(&edited).Error
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
restoreErr := h.renameArchives(targets, webhook.Name)
|
|
||||||
if restoreErr != nil {
|
|
||||||
h.log.Error(
|
|
||||||
"failed to rename archives back",
|
|
||||||
"webhook_id", webhook.ID,
|
|
||||||
"error", restoreErr,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if errors.Is(err, delivery.ErrArchiveNameTaken) {
|
|
||||||
h.renderWebhookEdit(
|
|
||||||
w, r, webhook, name, description, retention,
|
|
||||||
"Not saved: "+err.Error()+
|
|
||||||
". Move that archive out of the data directory, "+
|
|
||||||
"its .db together with any -wal and -shm beside "+
|
|
||||||
"it, then save again.",
|
|
||||||
http.StatusConflict,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
h.serverError(w, r, "failed to update webhook", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
http.Redirect(
|
|
||||||
w, r, withNotice("/hook/"+webhook.ID, webhookSaved),
|
|
||||||
http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// renderWebhookEdit renders the webhook edit page for the webhook as
|
|
||||||
// stored, its form showing name, description and retentionDays, with
|
|
||||||
// an optional error message above it.
|
|
||||||
func (h *Handlers) renderWebhookEdit(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
r *http.Request,
|
|
||||||
webhook *database.Webhook,
|
|
||||||
name, description, retentionDays, errMsg string,
|
|
||||||
status int,
|
|
||||||
) {
|
|
||||||
data := map[string]any{
|
|
||||||
tmplKeyWebhook: webhook,
|
|
||||||
tmplKeyError: errMsg,
|
|
||||||
"Name": name,
|
|
||||||
"Description": description,
|
|
||||||
"RetentionDays": retentionDays,
|
|
||||||
}
|
|
||||||
|
|
||||||
h.renderTemplateStatus(w, r, "source_edit.html", data, status)
|
|
||||||
}
|
|
||||||
|
|
||||||
// renameWebhookArchives renames the archive file of every database
|
|
||||||
// target of a webhook from the webhook name oldName to newName,
|
|
||||||
// keeping each target's own name. It does nothing when the name is
|
|
||||||
// unchanged. It returns the targets it read, so that a failed edit can
|
|
||||||
// move those same archives back with renameArchives.
|
|
||||||
func (h *Handlers) renameWebhookArchives(
|
|
||||||
webhookID, oldName, newName string,
|
|
||||||
) ([]database.Target, error) {
|
|
||||||
if h.archives == nil || oldName == newName {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var targets []database.Target
|
|
||||||
|
|
||||||
err := h.db.DB().
|
|
||||||
Where(
|
|
||||||
"webhook_id = ? AND type = ?",
|
|
||||||
webhookID, database.TargetTypeDatabase,
|
|
||||||
).
|
|
||||||
Find(&targets).Error
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return targets, h.renameArchives(targets, newName)
|
|
||||||
}
|
|
||||||
|
|
||||||
// renameArchives renames the archive file of each of the given
|
|
||||||
// database targets to the webhook name webhookName, keeping each
|
|
||||||
// target's own name. It tries every target even after one fails, so
|
|
||||||
// that moving the archives back after a failed edit leaves none under
|
|
||||||
// the new name, and returns every failure joined.
|
|
||||||
func (h *Handlers) renameArchives(
|
|
||||||
targets []database.Target, webhookName string,
|
|
||||||
) error {
|
|
||||||
var errs []error
|
|
||||||
|
|
||||||
for i := range targets {
|
|
||||||
err := h.archives.Rename(
|
|
||||||
targets[i].ID, webhookName, targets[i].Name,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
errs = append(errs, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return errors.Join(errs...)
|
|
||||||
}
|
|
||||||
@@ -1,178 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// WebhookListItem holds data for the webhook list view.
|
|
||||||
type WebhookListItem struct {
|
|
||||||
database.Webhook
|
|
||||||
|
|
||||||
EntrypointCount int
|
|
||||||
InactiveEntrypointCount int
|
|
||||||
TargetCount int
|
|
||||||
InactiveTargetCount int
|
|
||||||
|
|
||||||
// EventCount is how many events the webhook holds, LastEventAt
|
|
||||||
// when the newest arrived (nil before the first), and
|
|
||||||
// FailedLast24Hours how many of its deliveries failed in the last
|
|
||||||
// 24 hours. When the webhook's event database could not be read,
|
|
||||||
// EventsUnreadable is set and these three are not known.
|
|
||||||
EventCount int64
|
|
||||||
LastEventAt *time.Time
|
|
||||||
FailedLast24Hours int64
|
|
||||||
EventsUnreadable bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleSourceList shows a list of user's webhooks.
|
|
||||||
func (h *Handlers) HandleSourceList() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
userID, ok := h.getUserID(r)
|
|
||||||
if !ok {
|
|
||||||
http.Redirect(
|
|
||||||
w, r, "/pages/login", http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
var webhooks []database.Webhook
|
|
||||||
|
|
||||||
err := h.db.DB().Where(
|
|
||||||
"user_id = ?", userID,
|
|
||||||
).Order("created_at DESC").Find(&webhooks).Error
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, r, "failed to list webhooks", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
items, err := h.buildWebhookListItems(webhooks)
|
|
||||||
if err != nil {
|
|
||||||
h.serverError(w, r, "failed to list webhooks", err)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
data := map[string]any{
|
|
||||||
"Webhooks": items,
|
|
||||||
}
|
|
||||||
|
|
||||||
h.renderTemplate(w, r, "sources_list.html", data)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// buildWebhookListItems builds the list's entry for each webhook. It
|
|
||||||
// fails when the main database cannot be read. A webhook whose event
|
|
||||||
// database cannot be read is marked on its own entry, and the error is
|
|
||||||
// logged.
|
|
||||||
func (h *Handlers) buildWebhookListItems(
|
|
||||||
webhooks []database.Webhook,
|
|
||||||
) ([]WebhookListItem, error) {
|
|
||||||
items := make([]WebhookListItem, len(webhooks))
|
|
||||||
since := time.Now().Add(-longWindow)
|
|
||||||
|
|
||||||
for i := range webhooks {
|
|
||||||
item := &items[i]
|
|
||||||
item.Webhook = webhooks[i]
|
|
||||||
|
|
||||||
var err error
|
|
||||||
|
|
||||||
item.EntrypointCount, item.InactiveEntrypointCount, err =
|
|
||||||
h.countWithInactive(&database.Entrypoint{}, item.ID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
item.TargetCount, item.InactiveTargetCount, err =
|
|
||||||
h.countWithInactive(&database.Target{}, item.ID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Opening an event database that does not exist would create
|
|
||||||
// it, and it would hold nothing to count.
|
|
||||||
if !h.dbMgr.DBExists(item.ID) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
err = h.readListEventFigures(item, since)
|
|
||||||
if err != nil {
|
|
||||||
h.log.Error(
|
|
||||||
"failed to read webhook list figures",
|
|
||||||
"webhook_id", item.ID,
|
|
||||||
"error", err,
|
|
||||||
)
|
|
||||||
|
|
||||||
item.EventsUnreadable = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return items, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// countWithInactive returns how many entrypoints or targets, as model
|
|
||||||
// says, a webhook has, and how many of them are inactive.
|
|
||||||
func (h *Handlers) countWithInactive(
|
|
||||||
model any, webhookID string,
|
|
||||||
) (int, int, error) {
|
|
||||||
var active []bool
|
|
||||||
|
|
||||||
err := h.db.DB().Model(model).
|
|
||||||
Where("webhook_id = ?", webhookID).
|
|
||||||
Pluck("active", &active).Error
|
|
||||||
if err != nil {
|
|
||||||
return 0, 0, fmt.Errorf(
|
|
||||||
"reading active flags of webhook %s: %w", webhookID, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
inactive := 0
|
|
||||||
|
|
||||||
for _, a := range active {
|
|
||||||
if !a {
|
|
||||||
inactive++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return len(active), inactive, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// readListEventFigures fills in the figures the list shows from the
|
|
||||||
// webhook's event database, with the statistics pane's own queries:
|
|
||||||
// the event count and last arrival from the event totals row, and the
|
|
||||||
// deliveries that failed since the given time from the deliveries'
|
|
||||||
// status index.
|
|
||||||
func (h *Handlers) readListEventFigures(
|
|
||||||
item *WebhookListItem, since time.Time,
|
|
||||||
) error {
|
|
||||||
webhookDB, err := h.dbMgr.GetDB(item.ID)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
var totals database.EventTotals
|
|
||||||
|
|
||||||
err = webhookDB.Take(&totals).Error
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("reading event totals: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
item.EventCount = totals.Events - totals.EventsRemoved
|
|
||||||
item.LastEventAt = totals.LastEventAt
|
|
||||||
|
|
||||||
byTarget, err := finishedByTarget(webhookDB, since)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, f := range byTarget {
|
|
||||||
item.FailedLast24Hours += f.Failed
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -131,9 +131,8 @@ const (
|
|||||||
//
|
//
|
||||||
// - Lines carrying an AUTHENTICATED operator's own input, which
|
// - Lines carrying an AUTHENTICATED operator's own input, which
|
||||||
// are not truncated at all: the webhook name on "webhook
|
// are not truncated at all: the webhook name on "webhook
|
||||||
// created" (internal/handlers/webhook_create.go) and the target
|
// created" and the target host on "target URL blocked by SSRF
|
||||||
// host on "target URL blocked by SSRF protection"
|
// protection" (both internal/handlers/source_management.go),
|
||||||
// (internal/handlers/target_create.go),
|
|
||||||
// and target_name in internal/delivery/engine.go and
|
// and target_name in internal/delivery/engine.go and
|
||||||
// target_http.go. Each is bounded only by the 1 MB form body
|
// target_http.go. Each is bounded only by the 1 MB form body
|
||||||
// cap, so a 100 KB name writes one line of roughly 600 KB.
|
// cap, so a 100 KB name writes one line of roughly 600 KB.
|
||||||
@@ -280,9 +279,8 @@ func (lrw *loggingResponseWriter) Unwrap() http.ResponseWriter {
|
|||||||
// this branch as bounded as the pattern branches below.
|
// this branch as bounded as the pattern branches below.
|
||||||
//
|
//
|
||||||
// Nothing debuggable is lost. The only query parameters the service
|
// Nothing debuggable is lost. The only query parameters the service
|
||||||
// reads are the login page's `next`, the page to return to,
|
// reads are the login page's `next`, the page to return to, and
|
||||||
// `notice`, which names the line a page shows after an action, and
|
// `notice`, which names the line a page shows after an action. The
|
||||||
// the event log's `show`, which picks the events it lists. The
|
|
||||||
// alternatives that would preserve more (a key count, a key
|
// alternatives that would preserve more (a key count, a key
|
||||||
// allowlist) all require parsing an attacker-sized query on every
|
// allowlist) all require parsing an attacker-sized query on every
|
||||||
// request, which is work an unauthenticated client would then be
|
// request, which is work an unauthenticated client would then be
|
||||||
|
|||||||
@@ -84,7 +84,7 @@ func TestAlpineRunsUnderTheSecurityPolicy(t *testing.T) {
|
|||||||
// this order. A new check is one more line here.
|
// this order. A new check is one more line here.
|
||||||
checkAddEntrypoint(ctx, t, page)
|
checkAddEntrypoint(ctx, t, page)
|
||||||
checkAddEachTargetType(ctx, t, page)
|
checkAddEachTargetType(ctx, t, page)
|
||||||
checkArchiveChoices(ctx, t, page)
|
checkArchiveExpiry(ctx, t, page)
|
||||||
checkRefusedTarget(ctx, t, page)
|
checkRefusedTarget(ctx, t, page)
|
||||||
checkTargetDeliveries(ctx, t, page, target.Name,
|
checkTargetDeliveries(ctx, t, page, target.Name,
|
||||||
"0 in total, 0 in the last 24 hours",
|
"0 in total, 0 in the last 24 hours",
|
||||||
@@ -98,25 +98,20 @@ func TestAlpineRunsUnderTheSecurityPolicy(t *testing.T) {
|
|||||||
checkRefusedNewWebhook(ctx, t, srv.URL+"/hooks/new")
|
checkRefusedNewWebhook(ctx, t, srv.URL+"/hooks/new")
|
||||||
checkEventLog(ctx, t, page+"/events", event.ID, older.ID, target.Name)
|
checkEventLog(ctx, t, page+"/events", event.ID, older.ID, target.Name)
|
||||||
checkMobileMenu(ctx, t, page)
|
checkMobileMenu(ctx, t, page)
|
||||||
checkPhoneWidth(ctx, t, page, page+"/events", target.Name)
|
|
||||||
|
|
||||||
assert.Empty(t, problems(), "the browser reported problems")
|
assert.Empty(t, problems(), "the browser reported problems")
|
||||||
}
|
}
|
||||||
|
|
||||||
// seedBrowserWebhook seeds the webhook the browser test loads, owned by
|
// seedBrowserWebhook seeds the webhook the browser test loads, owned by
|
||||||
// userID: an entrypoint, two events, and a target whose delivery of the
|
// userID: an entrypoint, two events, and a target whose delivery of the
|
||||||
// newer event failed once with a 502. The webhook's name and the newer
|
// newer event failed once with a 502. It returns the webhook, the older
|
||||||
// event's content type are each too long for one line on a phone. It
|
// and the newer event, and the target.
|
||||||
// returns the webhook, the older and the newer event, and the target.
|
|
||||||
func seedBrowserWebhook(
|
func seedBrowserWebhook(
|
||||||
t *testing.T, env *testEnv, userID string,
|
t *testing.T, env *testEnv, userID string,
|
||||||
) (*database.Webhook, *database.Event, *database.Event, *database.Target) {
|
) (*database.Webhook, *database.Event, *database.Event, *database.Target) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
webhook := env.seedWebhook(t, userID)
|
webhook := env.seedWebhook(t, userID)
|
||||||
require.NoError(t, env.db.DB().Model(webhook).Update(
|
|
||||||
"name", "payment_provider_production_notifications",
|
|
||||||
).Error)
|
|
||||||
require.NoError(t, env.db.DB().Omit(clause.Associations).Create(
|
require.NoError(t, env.db.DB().Omit(clause.Associations).Create(
|
||||||
&database.Entrypoint{
|
&database.Entrypoint{
|
||||||
WebhookID: webhook.ID,
|
WebhookID: webhook.ID,
|
||||||
@@ -131,9 +126,6 @@ func seedBrowserWebhook(
|
|||||||
|
|
||||||
webhookDB, err := env.dbMgr.GetDB(webhook.ID)
|
webhookDB, err := env.dbMgr.GetDB(webhook.ID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NoError(t, webhookDB.Model(event).Update(
|
|
||||||
"content_type", "application/vnd.paymentprovider.event+json",
|
|
||||||
).Error)
|
|
||||||
require.NoError(t, webhookDB.Omit(clause.Associations).Create(
|
require.NoError(t, webhookDB.Omit(clause.Associations).Create(
|
||||||
&database.DeliveryResult{
|
&database.DeliveryResult{
|
||||||
DeliveryID: dlv.ID,
|
DeliveryID: dlv.ID,
|
||||||
@@ -326,35 +318,31 @@ const (
|
|||||||
func checkAddEachTargetType(ctx context.Context, t *testing.T, url string) {
|
func checkAddEachTargetType(ctx context.Context, t *testing.T, url string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
// Each target type, with the badge its targets are listed with and
|
// Each target type, with the fields its add target form submits, in
|
||||||
// the fields its add target form submits, in page order. Only http
|
// page order. Only http and slack have a url field.
|
||||||
// and slack have a url field.
|
|
||||||
targetTypes := []struct {
|
targetTypes := []struct {
|
||||||
name string
|
name string
|
||||||
badge string
|
|
||||||
fields string
|
fields string
|
||||||
values map[string]string
|
values map[string]string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
"http", "http",
|
"http", "csrf_token name type url headers timeout max_retries",
|
||||||
"csrf_token name type url headers timeout max_retries",
|
|
||||||
map[string]string{"url": publicTargetURL},
|
map[string]string{"url": publicTargetURL},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"slack", "slack", "csrf_token name type url max_retries",
|
"slack", "csrf_token name type url max_retries",
|
||||||
map[string]string{"url": publicTargetURL},
|
map[string]string{"url": publicTargetURL},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"database", "archive", "csrf_token name type expiry rotation",
|
"database", "csrf_token name type expiry",
|
||||||
map[string]string{"expiry": "720h", "rotation": "daily"},
|
map[string]string{"expiry": "720h"},
|
||||||
},
|
},
|
||||||
{"log", "log", "csrf_token name type", nil},
|
{"log", "csrf_token name type", nil},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tt := range targetTypes {
|
for _, tt := range targetTypes {
|
||||||
checkAddTarget(
|
checkAddTarget(
|
||||||
ctx, t, url, tt.name, tt.badge,
|
ctx, t, url, tt.name, strings.Fields(tt.fields), tt.values,
|
||||||
strings.Fields(tt.fields), tt.values,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -365,11 +353,11 @@ func checkAddEachTargetType(ctx context.Context, t *testing.T, url string) {
|
|||||||
// type's own fields in place of the choice, and the form then submits
|
// type's own fields in place of the choice, and the form then submits
|
||||||
// exactly fields, so a field another type uses, such as url, is absent;
|
// exactly fields, so a field another type uses, such as url, is absent;
|
||||||
// Cancel closes it again. It then adds a target of the type, filling in
|
// Cancel closes it again. It then adds a target of the type, filling in
|
||||||
// values, and checks that the section lists it with badge.
|
// values, and checks that the section lists it with that type.
|
||||||
func checkAddTarget(
|
func checkAddTarget(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
url, targetType, badge string,
|
url, targetType string,
|
||||||
fields []string,
|
fields []string,
|
||||||
values map[string]string,
|
values map[string]string,
|
||||||
) {
|
) {
|
||||||
@@ -424,8 +412,8 @@ func checkAddTarget(
|
|||||||
|
|
||||||
click(ctx, t, saveButton)
|
click(ctx, t, saveButton)
|
||||||
assert.Truef(t, shown(ctx, `//span[text()="`+name+
|
assert.Truef(t, shown(ctx, `//span[text()="`+name+
|
||||||
`"]/following-sibling::div/span[text()="`+badge+`"]`),
|
`"]/following-sibling::div/span[text()="`+targetType+`"]`),
|
||||||
"%s: the added target is not listed as %s", targetType, badge)
|
"%s: the added target is not listed with its type", targetType)
|
||||||
}
|
}
|
||||||
|
|
||||||
// chooseTargetType clicks Add, picks targetType and clicks Next, and
|
// chooseTargetType clicks Add, picks targetType and clicks Next, and
|
||||||
@@ -447,18 +435,14 @@ func chooseTargetType(ctx context.Context, t *testing.T, targetType string) {
|
|||||||
"%s: Add still shows while the form is open", targetType)
|
"%s: Add still shows while the form is open", targetType)
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkArchiveChoices loads a webhook page and checks that the add
|
// checkArchiveExpiry loads a webhook page and checks that the add
|
||||||
// target form's archive expiry starts on never and its archive
|
// target form's archive expiry starts on never, that the database
|
||||||
// rotation on none, that the database target checkAddTarget added with
|
// target checkAddTarget added with 720h is listed as 30 days, and that
|
||||||
// 720h and daily is listed as 30 days and daily, and that its edit
|
// its edit form starts on 720h.
|
||||||
// form starts on 720h and daily.
|
func checkArchiveExpiry(ctx context.Context, t *testing.T, url string) {
|
||||||
func checkArchiveChoices(ctx context.Context, t *testing.T, url string) {
|
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
const (
|
const expiry = `form[action$="/targets"] select[name="expiry"]`
|
||||||
expiry = `form[action$="/targets"] select[name="expiry"]`
|
|
||||||
rotation = `form[action$="/targets"] select[name="rotation"]`
|
|
||||||
)
|
|
||||||
|
|
||||||
row := `//span[text()="added-database"]/ancestor::div[@class="p-4"][1]`
|
row := `//span[text()="added-database"]/ancestor::div[@class="p-4"][1]`
|
||||||
|
|
||||||
@@ -466,36 +450,26 @@ func checkArchiveChoices(ctx context.Context, t *testing.T, url string) {
|
|||||||
|
|
||||||
chooseTargetType(ctx, t, "database")
|
chooseTargetType(ctx, t, "database")
|
||||||
|
|
||||||
var startExpiry, startRotation, editedExpiry, editedRotation string
|
var start, edited string
|
||||||
|
|
||||||
require.NoError(t, chromedp.Run(
|
require.NoError(t, chromedp.Run(
|
||||||
ctx,
|
ctx, chromedp.Value(expiry, &start, chromedp.ByQuery),
|
||||||
chromedp.Value(expiry, &startExpiry, chromedp.ByQuery),
|
|
||||||
chromedp.Value(rotation, &startRotation, chromedp.ByQuery),
|
|
||||||
))
|
))
|
||||||
assert.Equal(t, "never", startExpiry,
|
assert.Equal(t, "never", start,
|
||||||
"the add target form's archive expiry does not start on never")
|
"the add target form's archive expiry does not start on never")
|
||||||
assert.Equal(t, "none", startRotation,
|
|
||||||
"the add target form's archive rotation does not start on none")
|
|
||||||
|
|
||||||
assert.True(t, shown(ctx, row+`//span[text()="Archive expiry:"]`+
|
assert.True(t, shown(ctx, row+`//span[text()="Archive Expiry:"]`+
|
||||||
`/following-sibling::span[text()="30 days"]`),
|
`/following-sibling::span[text()="30 days"]`),
|
||||||
"a database target added with 720h is not listed as 30 days")
|
"a database target added with 720h is not listed as 30 days")
|
||||||
assert.True(t, shown(ctx, row+`//span[text()="Archive rotation:"]`+
|
|
||||||
`/following-sibling::span[text()="daily"]`),
|
|
||||||
"a database target added with daily is not listed as daily")
|
|
||||||
|
|
||||||
click(ctx, t, row+`//a[text()="Edit"]`)
|
click(ctx, t, row+`//a[text()="Edit"]`)
|
||||||
require.NoError(t, chromedp.Run(
|
require.NoError(t, chromedp.Run(
|
||||||
ctx,
|
ctx,
|
||||||
chromedp.WaitReady("#expiry", chromedp.ByQuery),
|
chromedp.WaitReady("#expiry", chromedp.ByQuery),
|
||||||
chromedp.Value("#expiry", &editedExpiry, chromedp.ByQuery),
|
chromedp.Value("#expiry", &edited, chromedp.ByQuery),
|
||||||
chromedp.Value("#rotation", &editedRotation, chromedp.ByQuery),
|
|
||||||
))
|
))
|
||||||
assert.Equal(t, "720h", editedExpiry,
|
assert.Equal(t, "720h", edited,
|
||||||
"the edit form does not start on the stored archive expiry")
|
"the edit form does not start on the stored archive expiry")
|
||||||
assert.Equal(t, "daily", editedRotation,
|
|
||||||
"the edit form does not start on the stored archive rotation")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkRefusedTarget submits an http target the server refuses, a
|
// checkRefusedTarget submits an http target the server refuses, a
|
||||||
@@ -955,15 +929,15 @@ func checkEventSelection(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// checkEventKeyboard loads the event log and checks that Tab from the
|
// checkEventKeyboard loads the event log and checks that Tab from the
|
||||||
// page's Pending link, the last link above the list, reaches the row of
|
// page's Back link reaches the row of the newest event, the first after
|
||||||
// the newest event, the first after it, and that Enter then collapses
|
// it, and that Enter then collapses that event, which starts expanded,
|
||||||
// that event, which starts expanded, and Space expands it again.
|
// and Space expands it again.
|
||||||
func checkEventKeyboard(
|
func checkEventKeyboard(
|
||||||
ctx context.Context, t *testing.T, url, eventID string,
|
ctx context.Context, t *testing.T, url, eventID string,
|
||||||
) {
|
) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
pending := `//a[starts-with(text(), "Pending")]`
|
back := `//a[contains(text(), "Back to")]`
|
||||||
expanded := `form[action$="/` + eventID + `/resubmit"]`
|
expanded := `form[action$="/` + eventID + `/resubmit"]`
|
||||||
|
|
||||||
var focused string
|
var focused string
|
||||||
@@ -971,12 +945,12 @@ func checkEventKeyboard(
|
|||||||
require.NoError(t, chromedp.Run(
|
require.NoError(t, chromedp.Run(
|
||||||
ctx,
|
ctx,
|
||||||
loadPage(url),
|
loadPage(url),
|
||||||
chromedp.Focus(pending, chromedp.BySearch),
|
chromedp.Focus(back, chromedp.BySearch),
|
||||||
chromedp.KeyEvent(kb.Tab),
|
chromedp.KeyEvent(kb.Tab),
|
||||||
chromedp.Evaluate(`document.activeElement.textContent`, &focused),
|
chromedp.Evaluate(`document.activeElement.textContent`, &focused),
|
||||||
))
|
))
|
||||||
require.Contains(t, focused, eventID,
|
require.Contains(t, focused, eventID,
|
||||||
"Tab from the Pending link does not reach the event's row")
|
"Tab from the Back link does not reach the event's row")
|
||||||
|
|
||||||
require.NoError(t, chromedp.Run(ctx, chromedp.KeyEvent(kb.Enter)))
|
require.NoError(t, chromedp.Run(ctx, chromedp.KeyEvent(kb.Enter)))
|
||||||
assert.True(t, hidden(ctx, expanded), "Enter does not collapse the event")
|
assert.True(t, hidden(ctx, expanded), "Enter does not collapse the event")
|
||||||
@@ -1179,9 +1153,7 @@ func checkNewWebhookTargets(
|
|||||||
pruningChoice, expiry, chromedp.BySearch,
|
pruningChoice, expiry, chromedp.BySearch,
|
||||||
)))
|
)))
|
||||||
|
|
||||||
// The rotation choice is left on none.
|
want[database.TargetTypeDatabase] = `{"expiry":"` + expiry + `"}`
|
||||||
want[database.TargetTypeDatabase] = `{"expiry":"` + expiry +
|
|
||||||
`","rotation":"none"}`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
click(ctx, t, createButton)
|
click(ctx, t, createButton)
|
||||||
@@ -1222,8 +1194,8 @@ func targetConfigs(
|
|||||||
// checkRefusedNewWebhook submits the new webhook page with archive
|
// checkRefusedNewWebhook submits the new webhook page with archive
|
||||||
// checked and an HTTP target URL the server refuses, a loopback
|
// checked and an HTTP target URL the server refuses, a loopback
|
||||||
// destination, and checks that the page comes back with the reason and
|
// destination, and checks that the page comes back with the reason and
|
||||||
// every value entered, archive still checked and its pruning and
|
// every value entered, archive still checked and its pruning choice
|
||||||
// rotation choices showing.
|
// showing.
|
||||||
func checkRefusedNewWebhook(ctx context.Context, t *testing.T, url string) {
|
func checkRefusedNewWebhook(ctx context.Context, t *testing.T, url string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
@@ -1240,19 +1212,17 @@ func checkRefusedNewWebhook(ctx context.Context, t *testing.T, url string) {
|
|||||||
click(ctx, t, archiveBox)
|
click(ctx, t, archiveBox)
|
||||||
require.True(t, shown(ctx, pruningChoice),
|
require.True(t, shown(ctx, pruningChoice),
|
||||||
"checking archive does not show the pruning choice")
|
"checking archive does not show the pruning choice")
|
||||||
require.NoError(t, chromedp.Run(
|
require.NoError(t, chromedp.Run(ctx, chromedp.SetValue(
|
||||||
ctx,
|
pruningChoice, "2160h", chromedp.BySearch,
|
||||||
chromedp.SetValue(pruningChoice, "2160h", chromedp.BySearch),
|
)))
|
||||||
chromedp.SetValue("#archive_rotation", "monthly", chromedp.ByQuery),
|
|
||||||
))
|
|
||||||
click(ctx, t, createButton)
|
click(ctx, t, createButton)
|
||||||
|
|
||||||
assert.True(t, shown(ctx, `//div[@class="alert-error"]`),
|
assert.True(t, shown(ctx, `//div[@class="alert-error"]`),
|
||||||
"a refused webhook does not show the reason")
|
"a refused webhook does not show the reason")
|
||||||
|
|
||||||
var (
|
var (
|
||||||
name, description, retention, typed, expiry, rotation string
|
name, description, retention, typed, expiry string
|
||||||
checked bool
|
checked bool
|
||||||
)
|
)
|
||||||
|
|
||||||
require.NoError(t, chromedp.Run(
|
require.NoError(t, chromedp.Run(
|
||||||
@@ -1262,7 +1232,6 @@ func checkRefusedNewWebhook(ctx context.Context, t *testing.T, url string) {
|
|||||||
chromedp.Value("#retention_days", &retention, chromedp.ByQuery),
|
chromedp.Value("#retention_days", &retention, chromedp.ByQuery),
|
||||||
chromedp.Value("#http_url", &typed, chromedp.ByQuery),
|
chromedp.Value("#http_url", &typed, chromedp.ByQuery),
|
||||||
chromedp.Value("#archive_expiry", &expiry, chromedp.ByQuery),
|
chromedp.Value("#archive_expiry", &expiry, chromedp.ByQuery),
|
||||||
chromedp.Value("#archive_rotation", &rotation, chromedp.ByQuery),
|
|
||||||
chromedp.Evaluate(archiveIsOn, &checked),
|
chromedp.Evaluate(archiveIsOn, &checked),
|
||||||
))
|
))
|
||||||
|
|
||||||
@@ -1273,7 +1242,6 @@ func checkRefusedNewWebhook(ctx context.Context, t *testing.T, url string) {
|
|||||||
assert.True(t, checked, "archive is no longer checked")
|
assert.True(t, checked, "archive is no longer checked")
|
||||||
assert.True(t, shown(ctx, pruningChoice), "the pruning choice is hidden")
|
assert.True(t, shown(ctx, pruningChoice), "the pruning choice is hidden")
|
||||||
assert.Equal(t, "2160h", expiry, "the pruning chosen is lost")
|
assert.Equal(t, "2160h", expiry, "the pruning chosen is lost")
|
||||||
assert.Equal(t, "monthly", rotation, "the rotation chosen is lost")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkMobileMenu loads a page in a phone-sized window and checks that
|
// checkMobileMenu loads a page in a phone-sized window and checks that
|
||||||
@@ -1300,60 +1268,3 @@ func checkMobileMenu(ctx context.Context, t *testing.T, url string) {
|
|||||||
click(ctx, t, button)
|
click(ctx, t, button)
|
||||||
assert.True(t, hidden(ctx, menu), "the menu button does not close the menu")
|
assert.True(t, hidden(ctx, menu), "the menu button does not close the menu")
|
||||||
}
|
}
|
||||||
|
|
||||||
// scrollsSideways reports whether the page is wider than the window. A
|
|
||||||
// page's clientWidth is the window's width less its scroll bar.
|
|
||||||
const scrollsSideways = `document.documentElement.scrollWidth >
|
|
||||||
document.documentElement.clientWidth`
|
|
||||||
|
|
||||||
// cutOffElements lists each element, without elements inside it, that
|
|
||||||
// is shown but runs past the page's edge or its card's, by more than a
|
|
||||||
// pixel of rounding. A card hides what runs past its edge.
|
|
||||||
const cutOffElements = `[...document.querySelectorAll("body *")]
|
|
||||||
.filter((el) => {
|
|
||||||
const box = el.getBoundingClientRect();
|
|
||||||
const card = el.closest(".card")?.getBoundingClientRect();
|
|
||||||
const left = card ? card.left : 0;
|
|
||||||
const right = card ? card.right : document.documentElement.clientWidth;
|
|
||||||
return el.children.length === 0 && box.width > 0 &&
|
|
||||||
(box.left < left - 1 || box.right > right + 1);
|
|
||||||
})
|
|
||||||
.map((el) => el.outerHTML.slice(0, 120))`
|
|
||||||
|
|
||||||
// checkPhoneWidth loads the webhook page, url, and its event log,
|
|
||||||
// eventLog, in a phone-sized window, the event log with the attempts of
|
|
||||||
// the newest event's delivery to targetName shown. It checks that
|
|
||||||
// neither page scrolls sideways and that nothing shown on either, no
|
|
||||||
// status, time or control, is cut off at the page's or its card's edge.
|
|
||||||
func checkPhoneWidth(
|
|
||||||
ctx context.Context, t *testing.T, url, eventLog, targetName string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var (
|
|
||||||
sideways bool
|
|
||||||
cutOff []string
|
|
||||||
)
|
|
||||||
|
|
||||||
measure := chromedp.Tasks{
|
|
||||||
chromedp.Evaluate(scrollsSideways, &sideways),
|
|
||||||
chromedp.Evaluate(cutOffElements, &cutOff),
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, chromedp.Run(
|
|
||||||
ctx,
|
|
||||||
chromedp.EmulateViewport(phoneWidth, phoneHeight),
|
|
||||||
loadPage(url),
|
|
||||||
measure,
|
|
||||||
))
|
|
||||||
assert.False(t, sideways, "the webhook page scrolls sideways on a phone")
|
|
||||||
assert.Empty(t, cutOff, "the webhook page cuts these off on a phone")
|
|
||||||
|
|
||||||
require.NoError(t, chromedp.Run(ctx, loadPage(eventLog)))
|
|
||||||
click(ctx, t, `//span[text()="`+targetName+`"]`)
|
|
||||||
require.True(t, shown(ctx, `//span[text()="Attempt 1"]`),
|
|
||||||
"clicking the delivery does not show its attempts")
|
|
||||||
require.NoError(t, chromedp.Run(ctx, measure))
|
|
||||||
assert.False(t, sideways, "the event log scrolls sideways on a phone")
|
|
||||||
assert.Empty(t, cutOff, "the event log cuts these off on a phone")
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -72,7 +72,7 @@ func TestEventResubmit_SignedOutRequestsNeverReachTheRateLimit(
|
|||||||
env.requireNotice(
|
env.requireNotice(
|
||||||
t, env.post(path, csrfForm(token), cookies),
|
t, env.post(path, csrfForm(token), cookies),
|
||||||
logsPath, "resubmit-no-targets",
|
logsPath, "resubmit-no-targets",
|
||||||
"this webhook has no active targets", cookies,
|
"this source has no active targets", cookies,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -117,7 +117,7 @@ func TestEventResubmit_RefusedWithoutAValidCSRFToken(t *testing.T) {
|
|||||||
env.requireNotice(
|
env.requireNotice(
|
||||||
t, env.post(path, csrfForm(token), cookies),
|
t, env.post(path, csrfForm(token), cookies),
|
||||||
logsPath, "resubmit-no-targets",
|
logsPath, "resubmit-no-targets",
|
||||||
"this webhook has no active targets", cookies,
|
"this source has no active targets", cookies,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -171,7 +171,7 @@ func TestEventResubmit_AnotherWebhooksEvent404s(t *testing.T) {
|
|||||||
csrfForm(token), cookies,
|
csrfForm(token), cookies,
|
||||||
),
|
),
|
||||||
intrudersLogs, "resubmit-no-targets",
|
intrudersLogs, "resubmit-no-targets",
|
||||||
"this webhook has no active targets", cookies,
|
"this source has no active targets", cookies,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -914,26 +914,6 @@ func TestPagesLogout_SaysSignedOut(t *testing.T) {
|
|||||||
env.requireNotice(t, w, "/pages/login", "signed-out", "Signed out.", nil)
|
env.requireNotice(t, w, "/pages/login", "signed-out", "Signed out.", nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestSignInAndSignOutWording pins one wording for both: the sign-in
|
|
||||||
// page's button reads "Sign in" and the navbar's buttons "Sign out", as
|
|
||||||
// the sign-in page's heading, the error page's link and the notice
|
|
||||||
// after signing out do.
|
|
||||||
func TestSignInAndSignOutWording(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := newTestEnv(t)
|
|
||||||
|
|
||||||
assert.Contains(
|
|
||||||
t, env.get("/pages/login", nil).Body.String(), ">Sign in</button>",
|
|
||||||
)
|
|
||||||
|
|
||||||
userID, _ := env.seedUser(t, "reader", "somepassword")
|
|
||||||
page := env.get("/hooks", env.authCookies(t, userID, "reader")).Body.String()
|
|
||||||
|
|
||||||
assert.Equal(t, 2, strings.Count(page, ">Sign out</button>"),
|
|
||||||
"the desktop and the mobile navbar each say Sign out")
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- /user/{username} group ---
|
// --- /user/{username} group ---
|
||||||
|
|
||||||
// TestPasswordChange_OversizeBody_RejectedAndPasswordUnchanged
|
// TestPasswordChange_OversizeBody_RejectedAndPasswordUnchanged
|
||||||
@@ -1383,7 +1363,7 @@ func TestHook_ResubmitFromEventLog(t *testing.T) {
|
|||||||
)
|
)
|
||||||
env.requireNotice(
|
env.requireNotice(
|
||||||
t, w, logsPath, "resubmit-no-targets",
|
t, w, logsPath, "resubmit-no-targets",
|
||||||
"this webhook has no active targets", cookies,
|
"this source has no active targets", cookies,
|
||||||
)
|
)
|
||||||
|
|
||||||
webhookDB, err := env.dbMgr.GetDB(wh.ID)
|
webhookDB, err := env.dbMgr.GetDB(wh.ID)
|
||||||
|
|||||||
@@ -219,8 +219,8 @@ func keptSentryHeaders(headers map[string]string) map[string]string {
|
|||||||
// sentryKeepsHeader reports whether a request header is routing or
|
// sentryKeepsHeader reports whether a request header is routing or
|
||||||
// content metadata rather than client-chosen payload. Referer is kept
|
// content metadata rather than client-chosen payload. Referer is kept
|
||||||
// on the reasoning that it is browser-set, that the only query
|
// on the reasoning that it is browser-set, that the only query
|
||||||
// parameters in this service's own URLs are the login page's `next`,
|
// parameters in this service's own URLs are the login page's `next`
|
||||||
// `notice` and the event log's `show`, and that Referrer-Policy is set to
|
// and `notice`, and that Referrer-Policy is set to
|
||||||
// strict-origin-when-cross-origin. X-Request-Id ties the event to the
|
// strict-origin-when-cross-origin. X-Request-Id ties the event to the
|
||||||
// local access log line, which holds the rest of the detail.
|
// local access log line, which holds the rest of the detail.
|
||||||
func sentryKeepsHeader(name string) bool {
|
func sentryKeepsHeader(name string) bool {
|
||||||
|
|||||||
+2
-4
@@ -1,8 +1,6 @@
|
|||||||
{
|
{
|
||||||
"private": true,
|
"private": true,
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"eslint": "10.11.0",
|
"eslint": "10.11.0"
|
||||||
"prettier": "3.9.9"
|
}
|
||||||
},
|
|
||||||
"packageManager": "yarn@4.18.1+sha512.b2e1e7524f654f2749d32b4ebcb4622473cb5bcbc485df2007e12a154e50162a4d795526768bc5f5b8f81717bfd79deb2472813d86fb5ae2eb551fa9c872b08f"
|
|
||||||
}
|
}
|
||||||
|
|||||||
+4
-5
@@ -3,8 +3,8 @@
|
|||||||
# this repo. Idempotent: every install is guarded by a check so already
|
# this repo. Idempotent: every install is guarded by a check so already
|
||||||
# installed tools are skipped. Base tooling comes from nix, apt, brew,
|
# installed tools are skipped. Base tooling comes from nix, apt, brew,
|
||||||
# or apk (detected in that order); assumes NOTHING is present (not git,
|
# or apk (detected in that order); assumes NOTHING is present (not git,
|
||||||
# make, or go). golangci-lint, node, ESLint and prettier are deliberately
|
# make, or go). golangci-lint, node and ESLint are deliberately not
|
||||||
# not installed: they run only in docker, via script/lint and script/fmt.
|
# installed: linting runs only in docker, via script/lint.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
@@ -60,10 +60,9 @@ main() {
|
|||||||
if missing go; then pkg_install go golang go go; fi
|
if missing go; then pkg_install go golang go go; fi
|
||||||
|
|
||||||
# Not installed here: docker is platform-specific and out of scope for a
|
# Not installed here: docker is platform-specific and out of scope for a
|
||||||
# package-manager bootstrap, but script/lint, script/fmt and script/css
|
# package-manager bootstrap, but script/lint and script/css need it.
|
||||||
# need it.
|
|
||||||
if missing docker; then
|
if missing docker; then
|
||||||
echo "bootstrap: docker not found; script/lint, script/fmt and script/css require it" >&2
|
echo "bootstrap: docker not found; script/lint and script/css require it" >&2
|
||||||
fi
|
fi
|
||||||
|
|
||||||
go mod download
|
go mod download
|
||||||
|
|||||||
+4
-4
@@ -1,8 +1,8 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/cibuild: run the CI build. The Dockerfile runs the checks (the
|
# script/cibuild: run the CI build. The Dockerfile runs the checks
|
||||||
# gofmt check, golangci-lint, the stylesheet check, ESLint, the Markdown
|
# (make fmt-check, lint, test), so a successful build implies a green
|
||||||
# check, make test), so a successful build implies a green repo. Generic:
|
# repo. Generic: needs no adaptation. The Gitea workflow runs this on
|
||||||
# needs no adaptation. The Gitea workflow runs this on push.
|
# push.
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
|
|||||||
+1
-4
@@ -1,7 +1,5 @@
|
|||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
# script/fmt: format all files (writes): the Go code with gofmt and
|
# script/fmt: format all files (writes).
|
||||||
# goimports, the Markdown with prettier. prettier is never installed
|
|
||||||
# locally: it runs in docker, in the Dockerfile's Markdown stages.
|
|
||||||
set -eu
|
set -eu
|
||||||
|
|
||||||
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
ROOT="$(cd "$(dirname "$0")/.." && pwd -P)"
|
||||||
@@ -12,7 +10,6 @@ main() {
|
|||||||
if command -v goimports >/dev/null 2>&1; then
|
if command -v goimports >/dev/null 2>&1; then
|
||||||
goimports -w .
|
goimports -w .
|
||||||
fi
|
fi
|
||||||
docker build --target markdown-output --output type=local,dest=. .
|
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ main() {
|
|||||||
gofmt -s -l .
|
gofmt -s -l .
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
docker build --target markdown-check --output type=cacheonly .
|
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
main "$@"
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
+2
-5
@@ -72,8 +72,8 @@ document.addEventListener("alpine:init", function () {
|
|||||||
// Something a click shows and hides: the mobile menu, an add form,
|
// Something a click shows and hides: the mobile menu, an add form,
|
||||||
// an entrypoint's edit form, an event in the event log or in the
|
// an entrypoint's edit form, an event in the event log or in the
|
||||||
// recent events, a delivery's attempts, the new webhook page's
|
// recent events, a delivery's attempts, the new webhook page's
|
||||||
// archive pruning and rotation choices. It starts hidden, or shown
|
// archive pruning choice. It starts hidden, or shown when its
|
||||||
// when its element has the data-open attribute.
|
// element has the data-open attribute.
|
||||||
window.Alpine.data("collapsible", function () {
|
window.Alpine.data("collapsible", function () {
|
||||||
return {
|
return {
|
||||||
open: false,
|
open: false,
|
||||||
@@ -140,7 +140,6 @@ document.addEventListener("alpine:init", function () {
|
|||||||
timeout: "",
|
timeout: "",
|
||||||
maxRetries: "",
|
maxRetries: "",
|
||||||
expiry: "",
|
expiry: "",
|
||||||
rotation: "",
|
|
||||||
init() {
|
init() {
|
||||||
const refused = this.$root.dataset;
|
const refused = this.$root.dataset;
|
||||||
|
|
||||||
@@ -152,7 +151,6 @@ document.addEventListener("alpine:init", function () {
|
|||||||
this.timeout = refused.timeout;
|
this.timeout = refused.timeout;
|
||||||
this.maxRetries = refused.maxRetries;
|
this.maxRetries = refused.maxRetries;
|
||||||
this.expiry = refused.expiry;
|
this.expiry = refused.expiry;
|
||||||
this.rotation = refused.rotation;
|
|
||||||
},
|
},
|
||||||
add() {
|
add() {
|
||||||
this.choosing = true;
|
this.choosing = true;
|
||||||
@@ -171,7 +169,6 @@ document.addEventListener("alpine:init", function () {
|
|||||||
this.timeout = "";
|
this.timeout = "";
|
||||||
this.maxRetries = "";
|
this.maxRetries = "";
|
||||||
this.expiry = "";
|
this.expiry = "";
|
||||||
this.rotation = "";
|
|
||||||
this.$refs.form.reset();
|
this.$refs.form.reset();
|
||||||
},
|
},
|
||||||
get filling() {
|
get filling() {
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
an event's own page. Spans only, since a button may hold no div. -->
|
an event's own page. Spans only, since a button may hold no div. -->
|
||||||
<span class="flex flex-1 flex-wrap items-center justify-between gap-3">
|
<span class="flex flex-1 flex-wrap items-center justify-between gap-3">
|
||||||
<span class="flex flex-wrap items-center gap-3">
|
<span class="flex flex-wrap items-center gap-3">
|
||||||
<span class="text-sm text-gray-700 wrap-anywhere">{{.Target.DisplayName}}</span>
|
<span class="text-sm text-gray-700">{{.Target.DisplayName}}</span>
|
||||||
{{if .Replay}}<span class="text-xs text-gray-500">replay</span>{{end}}
|
{{if .Replay}}<span class="text-xs text-gray-500">replay</span>{{end}}
|
||||||
<span class="text-xs {{if eq .Status "delivered"}}text-green-600{{else if eq .Status "failed"}}text-red-600{{else if eq .Status "retrying"}}text-yellow-600{{else}}text-gray-400{{end}}">{{with .Paused}}waiting: target paused after repeated failures, next try no earlier than {{.Until}} ({{.Relative}}){{else}}{{.Status}}{{end}}</span>
|
<span class="text-xs {{if eq .Status "delivered"}}text-green-600{{else if eq .Status "failed"}}text-red-600{{else if eq .Status "retrying"}}text-yellow-600{{else}}text-gray-400{{end}}">{{with .Paused}}waiting: target paused after repeated failures, next try no earlier than {{.Until}} ({{.Relative}}){{else}}{{.Status}}{{end}}</span>
|
||||||
</span>
|
</span>
|
||||||
|
|||||||
@@ -50,15 +50,6 @@
|
|||||||
</dl>
|
</dl>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="card mt-6">
|
|
||||||
<div class="p-4 border-b border-gray-200">
|
|
||||||
<h2 class="text-lg font-medium text-gray-900">Request</h2>
|
|
||||||
</div>
|
|
||||||
<div class="p-4">
|
|
||||||
{{template "event_request" .}}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="card mt-6">
|
<div class="card mt-6">
|
||||||
<div class="p-4 border-b border-gray-200">
|
<div class="p-4 border-b border-gray-200">
|
||||||
<h2 class="text-lg font-medium text-gray-900">Body</h2>
|
<h2 class="text-lg font-medium text-gray-900">Body</h2>
|
||||||
|
|||||||
@@ -1,22 +0,0 @@
|
|||||||
{{define "event_request"}}
|
|
||||||
<!-- The entrypoint an event arrived at and its request headers, as
|
|
||||||
handlers.EventLogView carries them: the same in the event log and
|
|
||||||
the event's own page. The entrypoint's URL is never shown. A
|
|
||||||
resubmitted copy, even a copy of a copy, did not arrive at an
|
|
||||||
entrypoint; the request it copies did. -->
|
|
||||||
<div class="space-y-2 text-xs">
|
|
||||||
{{if .ResubmittedFrom}}
|
|
||||||
<p class="text-gray-500">The request it copies arrived at <span class="text-gray-900 wrap-anywhere">{{.Entrypoint}}</span></p>
|
|
||||||
{{else}}
|
|
||||||
<p class="text-gray-500">Arrived at <span class="text-gray-900 wrap-anywhere">{{.Entrypoint}}</span></p>
|
|
||||||
{{end}}
|
|
||||||
{{if .HeadersCut}}
|
|
||||||
<p class="text-gray-500">The request headers are larger than the event log shows. <a href="{{.Body.EventURL}}" class="btn-small">Show the request headers</a></p>
|
|
||||||
{{else if .Headers}}
|
|
||||||
<p class="text-gray-500">Request headers</p>
|
|
||||||
<pre class="rounded-md border border-gray-200 bg-white p-2 text-xs text-gray-700 overflow-x-auto whitespace-pre-wrap break-all">{{.Headers}}</pre>
|
|
||||||
{{else}}
|
|
||||||
<p class="text-gray-500">No request headers.</p>
|
|
||||||
{{end}}
|
|
||||||
</div>
|
|
||||||
{{end}}
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
{{template "base" .}}
|
{{template "base" .}}
|
||||||
|
|
||||||
{{define "title"}}Sign in - Webhooker{{end}}
|
{{define "title"}}Login - Webhooker{{end}}
|
||||||
|
|
||||||
{{define "content"}}
|
{{define "content"}}
|
||||||
<div class="min-h-screen flex items-center justify-center py-12 px-4">
|
<div class="min-h-screen flex items-center justify-center py-12 px-4">
|
||||||
@@ -52,7 +52,7 @@
|
|||||||
>
|
>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<button type="submit" class="btn-primary w-full py-3">Sign in</button>
|
<button type="submit" class="btn-primary w-full py-3">Sign In</button>
|
||||||
</form>
|
</form>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -32,7 +32,7 @@
|
|||||||
{{if .CSRFToken}}
|
{{if .CSRFToken}}
|
||||||
<form method="POST" action="/pages/logout" class="inline">
|
<form method="POST" action="/pages/logout" class="inline">
|
||||||
<input type="hidden" name="csrf_token" value="{{.CSRFToken}}">
|
<input type="hidden" name="csrf_token" value="{{.CSRFToken}}">
|
||||||
<button type="submit" class="btn-secondary">Sign out</button>
|
<button type="submit" class="btn-secondary">Logout</button>
|
||||||
</form>
|
</form>
|
||||||
{{end}}
|
{{end}}
|
||||||
{{end}}
|
{{end}}
|
||||||
@@ -49,7 +49,7 @@
|
|||||||
{{if .CSRFToken}}
|
{{if .CSRFToken}}
|
||||||
<form method="POST" action="/pages/logout">
|
<form method="POST" action="/pages/logout">
|
||||||
<input type="hidden" name="csrf_token" value="{{.CSRFToken}}">
|
<input type="hidden" name="csrf_token" value="{{.CSRFToken}}">
|
||||||
<button type="submit" class="btn-secondary w-full">Sign out</button>
|
<button type="submit" class="btn-secondary w-full">Logout</button>
|
||||||
</form>
|
</form>
|
||||||
{{end}}
|
{{end}}
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|||||||
@@ -6,18 +6,15 @@
|
|||||||
<!-- 108rem, half again the 72rem (max-w-6xl) of the webhook list, the
|
<!-- 108rem, half again the 72rem (max-w-6xl) of the webhook list, the
|
||||||
event log, the navbar and the footer, so an entrypoint URL fits on
|
event log, the navbar and the footer, so an entrypoint URL fits on
|
||||||
one line. An inline style, because the committed tailwind.css has
|
one line. An inline style, because the committed tailwind.css has
|
||||||
no class this wide. wrap-anywhere goes only on names and
|
no class this wide. -->
|
||||||
descriptions: a row too wide for a phone must still run past the
|
|
||||||
edge, where the browser test sees it, rather than break its
|
|
||||||
controls mid-word. -->
|
|
||||||
<div class="mx-auto px-6 py-8" style="max-width: 108rem">
|
<div class="mx-auto px-6 py-8" style="max-width: 108rem">
|
||||||
<div class="mb-6">
|
<div class="mb-6">
|
||||||
<a href="/hooks" class="btn-small">← Back to webhooks</a>
|
<a href="/hooks" class="btn-small">← Back to webhooks</a>
|
||||||
<div class="flex flex-wrap justify-between items-center gap-2 mt-2">
|
<div class="flex flex-wrap justify-between items-center gap-2 mt-2">
|
||||||
<div>
|
<div>
|
||||||
<h1 class="text-2xl font-medium text-gray-900 wrap-anywhere">{{.Webhook.Name}}</h1>
|
<h1 class="text-2xl font-medium text-gray-900">{{.Webhook.Name}}</h1>
|
||||||
{{if .Webhook.Description}}
|
{{if .Webhook.Description}}
|
||||||
<p class="text-sm text-gray-500 mt-1 wrap-anywhere">{{.Webhook.Description}}</p>
|
<p class="text-sm text-gray-500 mt-1">{{.Webhook.Description}}</p>
|
||||||
{{end}}
|
{{end}}
|
||||||
</div>
|
</div>
|
||||||
<div class="flex gap-2">
|
<div class="flex gap-2">
|
||||||
@@ -63,7 +60,7 @@
|
|||||||
{{range .Entrypoints}}
|
{{range .Entrypoints}}
|
||||||
<div class="p-4" x-data="collapsible">
|
<div class="p-4" x-data="collapsible">
|
||||||
<div class="flex flex-wrap items-center justify-between gap-2 mb-1">
|
<div class="flex flex-wrap items-center justify-between gap-2 mb-1">
|
||||||
<span x-show="closed" class="text-sm font-medium text-gray-900 wrap-anywhere">{{if .Description}}{{.Description}}{{else}}Entrypoint{{end}}</span>
|
<span x-show="closed" class="text-sm font-medium text-gray-900">{{if .Description}}{{.Description}}{{else}}Entrypoint{{end}}</span>
|
||||||
<!-- Edit shows this form in place of the
|
<!-- Edit shows this form in place of the
|
||||||
description and hides until it closes, and
|
description and hides until it closes, and
|
||||||
Cancel resets what was typed. With
|
Cancel resets what was typed. With
|
||||||
@@ -136,8 +133,7 @@
|
|||||||
data-headers="{{.TargetForm.Headers}}"
|
data-headers="{{.TargetForm.Headers}}"
|
||||||
data-timeout="{{.TargetForm.Timeout}}"
|
data-timeout="{{.TargetForm.Timeout}}"
|
||||||
data-max-retries="{{.TargetForm.MaxRetries}}"
|
data-max-retries="{{.TargetForm.MaxRetries}}"
|
||||||
data-expiry="{{.TargetForm.Expiry}}"
|
data-expiry="{{.TargetForm.Expiry}}">
|
||||||
data-rotation="{{.TargetForm.Rotation}}">
|
|
||||||
<div class="p-4 border-b border-gray-200 flex justify-between items-center">
|
<div class="p-4 border-b border-gray-200 flex justify-between items-center">
|
||||||
<h2 class="text-lg font-medium text-gray-900">Targets</h2>
|
<h2 class="text-lg font-medium text-gray-900">Targets</h2>
|
||||||
<button type="button" @click="add" x-show="closed" class="btn-small">
|
<button type="button" @click="add" x-show="closed" class="btn-small">
|
||||||
@@ -162,7 +158,7 @@
|
|||||||
<select x-ref="type" aria-label="Target type" class="input text-sm p-2 flex-1">
|
<select x-ref="type" aria-label="Target type" class="input text-sm p-2 flex-1">
|
||||||
<option value="http">HTTP</option>
|
<option value="http">HTTP</option>
|
||||||
<option value="slack">Slack</option>
|
<option value="slack">Slack</option>
|
||||||
<option value="database">Archive</option>
|
<option value="database">Database</option>
|
||||||
<option value="log">Log</option>
|
<option value="log">Log</option>
|
||||||
</select>
|
</select>
|
||||||
<button type="button" @click="next" class="btn-primary text-sm">Next</button>
|
<button type="button" @click="next" class="btn-primary text-sm">Next</button>
|
||||||
@@ -185,10 +181,10 @@
|
|||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<div class="flex gap-2 items-center">
|
<div class="flex gap-2 items-center">
|
||||||
<label class="text-sm text-gray-700">Delivery attempts:</label>
|
<label class="text-sm text-gray-700">Max retries:</label>
|
||||||
<input type="number" name="max_retries" :value="maxRetries" placeholder="0" min="0" max="20" class="input text-sm w-24">
|
<input type="number" name="max_retries" :value="maxRetries" placeholder="0" min="0" max="20" class="input text-sm w-24">
|
||||||
</div>
|
</div>
|
||||||
<p class="text-xs text-gray-500 mt-1">How many times each delivery is attempted in all, the first attempt included. 0 means a single attempt with no retries and no circuit breaker.</p>
|
<p class="text-xs text-gray-500 mt-1">This is the total number of delivery attempts, not retries on top of the first: a value of 3 makes three attempts in all. 0 means a single attempt with no retries and no circuit breaker.</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
@@ -201,38 +197,25 @@
|
|||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
<div class="flex gap-2 items-center">
|
<div class="flex gap-2 items-center">
|
||||||
<label class="text-sm text-gray-700">Delivery attempts:</label>
|
<label class="text-sm text-gray-700">Max retries:</label>
|
||||||
<input type="number" name="max_retries" :value="maxRetries" placeholder="0" min="0" max="20" class="input text-sm w-24">
|
<input type="number" name="max_retries" :value="maxRetries" placeholder="0" min="0" max="20" class="input text-sm w-24">
|
||||||
</div>
|
</div>
|
||||||
<p class="text-xs text-gray-500 mt-1">How many times each delivery is attempted in all, the first attempt included. 0 means a single attempt with no retries and no circuit breaker.</p>
|
<p class="text-xs text-gray-500 mt-1">This is the total number of delivery attempts, not retries on top of the first: a value of 3 makes three attempts in all. 0 means a single attempt with no retries and no circuit breaker.</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
<template x-if="isDatabase">
|
<template x-if="isDatabase">
|
||||||
<div class="space-y-3">
|
<div>
|
||||||
<input type="hidden" name="type" value="database">
|
<input type="hidden" name="type" value="database">
|
||||||
<div>
|
<div class="flex gap-2 items-center">
|
||||||
<div class="flex gap-2 items-center">
|
<label class="text-sm text-gray-700">Archive expiry:</label>
|
||||||
<label class="text-sm text-gray-700">Archive expiry:</label>
|
<select name="expiry" :value="expiry" class="input text-sm w-24">
|
||||||
<select name="expiry" :value="expiry" class="input text-sm w-24">
|
{{range .ArchiveExpiryChoices}}
|
||||||
{{range .ArchiveExpiryChoices}}
|
<option value="{{.Value}}">{{.Label}}</option>
|
||||||
<option value="{{.Value}}">{{.Label}}</option>
|
{{end}}
|
||||||
{{end}}
|
</select>
|
||||||
</select>
|
|
||||||
</div>
|
|
||||||
<p class="text-xs text-gray-500 mt-1">Archived events older than this are deleted from the archive; never keeps them all.</p>
|
|
||||||
</div>
|
|
||||||
<div>
|
|
||||||
<div class="flex gap-2 items-center">
|
|
||||||
<label class="text-sm text-gray-700">Archive rotation:</label>
|
|
||||||
<select name="rotation" :value="rotation" class="input text-sm w-28">
|
|
||||||
{{range .ArchiveRotationChoices}}
|
|
||||||
<option value="{{.Value}}">{{.Label}}</option>
|
|
||||||
{{end}}
|
|
||||||
</select>
|
|
||||||
</div>
|
|
||||||
<p class="text-xs text-gray-500 mt-1">Starts a new archive file each month, day or hour (UTC), named for it; none keeps one file.</p>
|
|
||||||
</div>
|
</div>
|
||||||
|
<p class="text-xs text-gray-500 mt-1">Archived events older than this are deleted from the archive; never keeps them all.</p>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
<template x-if="isLog">
|
<template x-if="isLog">
|
||||||
@@ -252,9 +235,9 @@
|
|||||||
{{range .Targets}}
|
{{range .Targets}}
|
||||||
<div class="p-4">
|
<div class="p-4">
|
||||||
<div class="flex flex-wrap items-center justify-between gap-2 mb-1">
|
<div class="flex flex-wrap items-center justify-between gap-2 mb-1">
|
||||||
<span class="text-sm font-medium text-gray-900 wrap-anywhere">{{.Name}}</span>
|
<span class="text-sm font-medium text-gray-900">{{.Name}}</span>
|
||||||
<div class="flex flex-wrap items-center gap-2">
|
<div class="flex flex-wrap items-center gap-2">
|
||||||
<span class="badge-info">{{if eq .Type "database"}}archive{{else}}{{.Type}}{{end}}</span>
|
<span class="badge-info">{{.Type}}</span>
|
||||||
{{if .Active}}
|
{{if .Active}}
|
||||||
<span class="badge-success">Active</span>
|
<span class="badge-success">Active</span>
|
||||||
{{else}}
|
{{else}}
|
||||||
@@ -278,7 +261,7 @@
|
|||||||
</div>
|
</div>
|
||||||
{{with .Paused}}
|
{{with .Paused}}
|
||||||
<div class="text-xs text-yellow-600 mt-1">
|
<div class="text-xs text-yellow-600 mt-1">
|
||||||
<span class="font-medium">Deliveries paused:</span>
|
<span class="font-medium">Deliveries Paused:</span>
|
||||||
<span>{{if .Until}}after repeated failures, until {{.Until}} ({{.Relative}}), then one waiting delivery is sent to test the target while the others wait at least one more cooldown{{else}}held while one delivery tests whether the target has recovered{{end}}</span>
|
<span>{{if .Until}}after repeated failures, until {{.Until}} ({{.Relative}}), then one waiting delivery is sent to test the target while the others wait at least one more cooldown{{else}}held while one delivery tests whether the target has recovered{{end}}</span>
|
||||||
</div>
|
</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
@@ -290,17 +273,17 @@
|
|||||||
{{end}}
|
{{end}}
|
||||||
{{with .Archive}}
|
{{with .Archive}}
|
||||||
<div class="text-xs text-gray-500 mt-1">
|
<div class="text-xs text-gray-500 mt-1">
|
||||||
<span class="font-medium text-gray-700">Archive file:</span>
|
<span class="font-medium text-gray-700">Archive File:</span>
|
||||||
<span class="break-all">{{.Name}}</span>
|
<span class="break-all">{{.Name}}</span>
|
||||||
{{with .Note}}<span>({{.}})</span>{{end}}
|
{{with .Note}}<span>({{.}})</span>{{end}}
|
||||||
</div>
|
</div>
|
||||||
{{if .Files}}
|
{{if .Size}}
|
||||||
<div class="text-xs text-gray-500 mt-1">
|
<div class="text-xs text-gray-500 mt-1">
|
||||||
<span class="font-medium text-gray-700">Archive size:</span>
|
<span class="font-medium text-gray-700">Archive Size:</span>
|
||||||
<span>{{.Size}}{{if gt .Files 1}} in {{.Files}} files{{end}}</span>
|
<span>{{.Size}}</span>
|
||||||
</div>
|
</div>
|
||||||
<div class="text-xs text-gray-500 mt-1">
|
<div class="text-xs text-gray-500 mt-1">
|
||||||
<span class="font-medium text-gray-700">Last written:</span>
|
<span class="font-medium text-gray-700">Last Written:</span>
|
||||||
<span title="{{.WrittenUTC}}">{{.Written}}</span>
|
<span title="{{.WrittenUTC}}">{{.Written}}</span>
|
||||||
</div>
|
</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|||||||
@@ -3,24 +3,12 @@
|
|||||||
{{define "title"}}Full Event Log - {{.Webhook.Name}} - Webhooker{{end}}
|
{{define "title"}}Full Event Log - {{.Webhook.Name}} - Webhooker{{end}}
|
||||||
|
|
||||||
{{define "content"}}
|
{{define "content"}}
|
||||||
<!-- wrap-anywhere goes only on names, IDs and content types: a row too
|
|
||||||
wide for a phone must still run past the edge, where the browser
|
|
||||||
test sees it, rather than break its statuses, times or controls
|
|
||||||
mid-word. So a target's name in an event's row, which shares its
|
|
||||||
element with the delivery's status, goes without. -->
|
|
||||||
<div class="max-w-6xl mx-auto px-6 py-8">
|
<div class="max-w-6xl mx-auto px-6 py-8">
|
||||||
<div class="mb-6">
|
<div class="mb-6">
|
||||||
<a href="/hook/{{.Webhook.ID}}" class="btn-small wrap-anywhere">← Back to {{.Webhook.Name}}</a>
|
<a href="/hook/{{.Webhook.ID}}" class="btn-small">← Back to {{.Webhook.Name}}</a>
|
||||||
<div class="flex flex-wrap justify-between items-center gap-2 mt-2">
|
<div class="flex justify-between items-center mt-2">
|
||||||
<h1 class="text-2xl font-medium text-gray-900">Full Event Log</h1>
|
<h1 class="text-2xl font-medium text-gray-900">Full Event Log</h1>
|
||||||
<!-- Under a filter, this counts the events the filter lists. -->
|
<span class="text-sm text-gray-500">{{if gt .TotalEvents (len .Events)}}{{len .Events}} most recent of {{.TotalEvents}} events{{else}}{{.TotalEvents}} total event{{if ne .TotalEvents 1}}s{{end}}{{end}}</span>
|
||||||
<span class="text-sm text-gray-500">{{if gt .TotalEvents (len .Events)}}{{len .Events}} most recent of {{.TotalEvents}} events{{else}}{{.TotalEvents}}{{if not .Show}} total{{end}} event{{if ne .TotalEvents 1}}s{{end}}{{end}}{{if eq .Show "failed"}} with a failed delivery{{else if eq .Show "pending"}} with a delivery pending or retrying{{end}}</span>
|
|
||||||
</div>
|
|
||||||
<!-- Plain links, so they work without the page's script library. The current one is marked. -->
|
|
||||||
<div class="mt-3 flex flex-wrap gap-2">
|
|
||||||
<a href="/hook/{{.Webhook.ID}}/events" class="btn-small{{if eq .Show ""}} bg-primary-50 border-primary-500{{end}}"{{if eq .Show ""}} aria-current="page"{{end}}>All</a>
|
|
||||||
<a href="/hook/{{.Webhook.ID}}/events?show=failed" title="Events with at least one failed delivery" class="btn-small{{if eq .Show "failed"}} bg-primary-50 border-primary-500{{end}}"{{if eq .Show "failed"}} aria-current="page"{{end}}>Failed ({{.FailedEvents}})</a>
|
|
||||||
<a href="/hook/{{.Webhook.ID}}/events?show=pending" title="Events with a delivery still pending or retrying" class="btn-small{{if eq .Show "pending"}} bg-primary-50 border-primary-500{{end}}"{{if eq .Show "pending"}} aria-current="page"{{end}}>Pending ({{.PendingEvents}})</a>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -33,8 +21,8 @@
|
|||||||
<div role="button" tabindex="0" class="btn-small w-full flex flex-wrap justify-between gap-2" :aria-expanded="open" @mousedown="cancelPendingToggle" @click="toggleUnlessSelecting" @keydown.enter.prevent="toggle" @keydown.space.prevent="toggle">
|
<div role="button" tabindex="0" class="btn-small w-full flex flex-wrap justify-between gap-2" :aria-expanded="open" @mousedown="cancelPendingToggle" @click="toggleUnlessSelecting" @keydown.enter.prevent="toggle" @keydown.space.prevent="toggle">
|
||||||
<span class="flex flex-wrap items-center gap-3">
|
<span class="flex flex-wrap items-center gap-3">
|
||||||
<span class="badge-info">{{.Method}}</span>
|
<span class="badge-info">{{.Method}}</span>
|
||||||
<span class="text-sm font-mono text-gray-700 wrap-anywhere">{{.ID}}</span>
|
<span class="text-sm font-mono text-gray-700">{{.ID}}</span>
|
||||||
<span class="text-sm text-gray-500 wrap-anywhere">{{.ContentType}}</span>
|
<span class="text-sm text-gray-500">{{.ContentType}}</span>
|
||||||
{{if .ResubmittedFrom}}
|
{{if .ResubmittedFrom}}
|
||||||
<span class="text-xs text-gray-500" title="This event is a copy of {{.ResubmittedFromID}}">resubmitted copy</span>
|
<span class="text-xs text-gray-500" title="This event is a copy of {{.ResubmittedFromID}}">resubmitted copy</span>
|
||||||
{{end}}
|
{{end}}
|
||||||
@@ -59,7 +47,7 @@
|
|||||||
<div x-show="open" x-cloak class="mt-3 p-3 bg-gray-50 rounded-md">
|
<div x-show="open" x-cloak class="mt-3 p-3 bg-gray-50 rounded-md">
|
||||||
<div class="mb-3 flex flex-wrap items-center justify-between gap-2">
|
<div class="mb-3 flex flex-wrap items-center justify-between gap-2">
|
||||||
<div class="text-xs text-gray-500">
|
<div class="text-xs text-gray-500">
|
||||||
{{if .ResubmittedFrom}}Resubmitted from event <a href="/hook/{{$.Webhook.ID}}/events/{{.ResubmittedFromID}}" class="btn-small font-mono wrap-anywhere">{{.ResubmittedFromID}}</a>.{{end}}
|
{{if .ResubmittedFrom}}Resubmitted from event <a href="/hook/{{$.Webhook.ID}}/events/{{.ResubmittedFromID}}" class="btn-small font-mono">{{.ResubmittedFromID}}</a>.{{end}}
|
||||||
{{if .ResubmitCount}}Resubmitted as {{.ResubmitCount}} new event{{if ne .ResubmitCount 1}}s{{end}}.{{end}}
|
{{if .ResubmitCount}}Resubmitted as {{.ResubmitCount}} new event{{if ne .ResubmitCount 1}}s{{end}}.{{end}}
|
||||||
</div>
|
</div>
|
||||||
<form method="POST" action="/hook/{{$.Webhook.ID}}/events/{{.ID}}/resubmit" class="inline">
|
<form method="POST" action="/hook/{{$.Webhook.ID}}/events/{{.ID}}/resubmit" class="inline">
|
||||||
@@ -67,9 +55,6 @@
|
|||||||
<button type="submit" class="btn-small" title="Submit this event again as a new event, to every currently active target">Resubmit</button>
|
<button type="submit" class="btn-small" title="Submit this event again as a new event, to every currently active target">Resubmit</button>
|
||||||
</form>
|
</form>
|
||||||
</div>
|
</div>
|
||||||
<div class="mb-3">
|
|
||||||
{{template "event_request" .}}
|
|
||||||
</div>
|
|
||||||
{{template "event_body" .Body}}
|
{{template "event_body" .Body}}
|
||||||
|
|
||||||
{{if .Deliveries}}
|
{{if .Deliveries}}
|
||||||
@@ -88,8 +73,6 @@
|
|||||||
{{if and .Status.Terminal (not .Target.Deleted)}}
|
{{if and .Status.Terminal (not .Target.Deleted)}}
|
||||||
<form method="POST" action="/hook/{{$.Webhook.ID}}/deliveries/{{.ID}}/replay" class="inline">
|
<form method="POST" action="/hook/{{$.Webhook.ID}}/deliveries/{{.ID}}/replay" class="inline">
|
||||||
<input type="hidden" name="csrf_token" value="{{$.CSRFToken}}">
|
<input type="hidden" name="csrf_token" value="{{$.CSRFToken}}">
|
||||||
<!-- The list to return to. -->
|
|
||||||
<input type="hidden" name="show" value="{{$.Show}}">
|
|
||||||
<button type="submit" class="btn-small" title="Send this event to the target again">Replay</button>
|
<button type="submit" class="btn-small" title="Send this event to the target again">Replay</button>
|
||||||
</form>
|
</form>
|
||||||
{{end}}
|
{{end}}
|
||||||
@@ -106,7 +89,7 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{{else}}
|
{{else}}
|
||||||
<div class="p-12 text-center text-sm text-gray-500">{{if eq $.Show "failed"}}No event has a failed delivery.{{else if eq $.Show "pending"}}No event has a delivery pending or retrying.{{else}}No events recorded yet.{{end}}</div>
|
<div class="p-12 text-center text-sm text-gray-500">No events recorded yet.</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -38,32 +38,23 @@
|
|||||||
<p class="text-xs text-gray-500 mt-1">Optional. When filled in, the webhook is created with an HTTP target that delivers each event to this URL.</p>
|
<p class="text-xs text-gray-500 mt-1">Optional. When filled in, the webhook is created with an HTTP target that delivers each event to this URL.</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- The checkbox shows the pruning and rotation choices while
|
<!-- The checkbox shows the pruning choice while checked. With
|
||||||
checked. With autocomplete="off", going back to the page
|
autocomplete="off", going back to the page does not
|
||||||
does not check the box again with the choices hidden. -->
|
check the box again with the choice hidden. -->
|
||||||
<div class="form-group" x-data="collapsible"{{if .Form.Archive}} data-open{{end}}>
|
<div class="form-group" x-data="collapsible"{{if .Form.Archive}} data-open{{end}}>
|
||||||
<label class="flex items-center gap-2 text-sm font-medium text-gray-700">
|
<label class="flex items-center gap-2 text-sm font-medium text-gray-700">
|
||||||
<input type="checkbox" name="archive" value="on"{{if .Form.Archive}} checked{{end}} autocomplete="off" @change="toggle" class="h-4 w-4">
|
<input type="checkbox" name="archive" value="on"{{if .Form.Archive}} checked{{end}} autocomplete="off" @change="toggle" class="h-4 w-4">
|
||||||
Archive
|
Archive
|
||||||
</label>
|
</label>
|
||||||
<p class="text-xs text-gray-500 mt-1">When checked, the webhook is created with an archive target that keeps a copy of every event.</p>
|
<p class="text-xs text-gray-500 mt-1">When checked, the webhook is created with a database target that keeps a copy of every event.</p>
|
||||||
<div x-show="open" x-cloak class="mt-3">
|
<div x-show="open" x-cloak class="mt-3">
|
||||||
<label for="archive_expiry" class="label">Archive expiry</label>
|
<label for="archive_expiry" class="label">Archive pruning</label>
|
||||||
<select id="archive_expiry" name="archive_expiry" class="input">
|
<select id="archive_expiry" name="archive_expiry" class="input">
|
||||||
{{range .ArchiveExpiryChoices}}
|
{{range .ArchiveExpiryChoices}}
|
||||||
<option value="{{.Value}}"{{if .Selected}} selected{{end}}>{{.Label}}</option>
|
<option value="{{.Value}}"{{if .Selected}} selected{{end}}>{{.Label}}</option>
|
||||||
{{end}}
|
{{end}}
|
||||||
</select>
|
</select>
|
||||||
<p class="text-xs text-gray-500 mt-1">Archived events older than this are deleted from the archive; never keeps them all.</p>
|
<p class="text-xs text-gray-500 mt-1">Archived events older than this are deleted from the archive; never keeps them all.</p>
|
||||||
<div class="mt-3">
|
|
||||||
<label for="archive_rotation" class="label">Archive rotation</label>
|
|
||||||
<select id="archive_rotation" name="archive_rotation" class="input">
|
|
||||||
{{range .ArchiveRotationChoices}}
|
|
||||||
<option value="{{.Value}}"{{if .Selected}} selected{{end}}>{{.Label}}</option>
|
|
||||||
{{end}}
|
|
||||||
</select>
|
|
||||||
<p class="text-xs text-gray-500 mt-1">Starts a new archive file each month, day or hour (UTC), named for it; none keeps one file.</p>
|
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,7 @@
|
|||||||
<div class="mb-6">
|
<div class="mb-6">
|
||||||
<a href="/hook/{{.Webhook.ID}}" class="btn-small">← Back to {{.Webhook.Name}}</a>
|
<a href="/hook/{{.Webhook.ID}}" class="btn-small">← Back to {{.Webhook.Name}}</a>
|
||||||
<h1 class="text-2xl font-medium text-gray-900 mt-2">Edit Target</h1>
|
<h1 class="text-2xl font-medium text-gray-900 mt-2">Edit Target</h1>
|
||||||
<p class="text-sm text-gray-500 mt-1">Type: {{if eq .Target.Type "database"}}archive{{else}}{{.Target.Type}}{{end}}. A target's type cannot be changed; create a new target to deliver a different way.</p>
|
<p class="text-sm text-gray-500 mt-1">Type: {{.Target.Type}}. A target's type cannot be changed; create a new target to deliver a different way.</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="card p-6">
|
<div class="card p-6">
|
||||||
@@ -59,7 +59,7 @@
|
|||||||
|
|
||||||
{{if eq .Target.Type "database"}}
|
{{if eq .Target.Type "database"}}
|
||||||
<div class="form-group">
|
<div class="form-group">
|
||||||
<label for="expiry" class="label">Archive expiry</label>
|
<label for="expiry" class="label">Archive Expiry</label>
|
||||||
<select id="expiry" name="expiry" class="input">
|
<select id="expiry" name="expiry" class="input">
|
||||||
{{range .ArchiveExpiryChoices}}
|
{{range .ArchiveExpiryChoices}}
|
||||||
<option value="{{.Value}}"{{if .Selected}} selected{{end}}>{{.Label}}</option>
|
<option value="{{.Value}}"{{if .Selected}} selected{{end}}>{{.Label}}</option>
|
||||||
@@ -67,23 +67,13 @@
|
|||||||
</select>
|
</select>
|
||||||
<p class="text-xs text-gray-500 mt-1">Archived events older than this are deleted from the archive; never keeps them all.</p>
|
<p class="text-xs text-gray-500 mt-1">Archived events older than this are deleted from the archive; never keeps them all.</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="form-group">
|
|
||||||
<label for="rotation" class="label">Archive rotation</label>
|
|
||||||
<select id="rotation" name="rotation" class="input">
|
|
||||||
{{range .ArchiveRotationChoices}}
|
|
||||||
<option value="{{.Value}}"{{if .Selected}} selected{{end}}>{{.Label}}</option>
|
|
||||||
{{end}}
|
|
||||||
</select>
|
|
||||||
<p class="text-xs text-gray-500 mt-1">Starts a new archive file each month, day or hour (UTC), named for it; none keeps one file. A change applies from the next event; existing files keep their names.</p>
|
|
||||||
</div>
|
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
||||||
{{if or (eq .Target.Type "http") (eq .Target.Type "slack")}}
|
{{if or (eq .Target.Type "http") (eq .Target.Type "slack")}}
|
||||||
<div class="form-group">
|
<div class="form-group">
|
||||||
<label for="max_retries" class="label">Delivery attempts</label>
|
<label for="max_retries" class="label">Max retries</label>
|
||||||
<input type="number" id="max_retries" name="max_retries" value="{{.TargetForm.MaxRetries}}" min="0" max="20" class="input">
|
<input type="number" id="max_retries" name="max_retries" value="{{.TargetForm.MaxRetries}}" min="0" max="20" class="input">
|
||||||
<p class="text-xs text-gray-500 mt-1">How many times each delivery is attempted in all, the first attempt included. 0 means a single attempt with no retries and no circuit breaker.</p>
|
<p class="text-xs text-gray-500 mt-1">This is the total number of delivery attempts, not retries on top of the first: a value of 3 makes three attempts in all. 0 means a single attempt with no retries and no circuit breaker.</p>
|
||||||
</div>
|
</div>
|
||||||
{{end}}
|
{{end}}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user