Compare commits
1
Commits
next
..
840c795c66
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
840c795c66 |
+4
-10
@@ -1,20 +1,14 @@
|
|||||||
# .git is sent so the build can derive the version it stamps into the binary
|
|
||||||
# (script/version). Its config, which can hold a remote URL carrying a
|
|
||||||
# credential and which `git describe` does not need, is left out of a
|
|
||||||
# directory context. A context sent as a tar is not filtered by this file, so
|
|
||||||
# it carries .git/config unless its sender leaves it out.
|
|
||||||
.git/config
|
|
||||||
|
|
||||||
# No tracked file may be listed here: git in the build would see it as
|
|
||||||
# deleted and mark the version -dirty.
|
|
||||||
#
|
|
||||||
# .ci-fingerprint is deliberately NOT excluded: it is the CI cache barrier
|
# .ci-fingerprint is deliberately NOT excluded: it is the CI cache barrier
|
||||||
# that keeps the check stages from replaying a cached pass. See the lint
|
# that keeps the check stages from replaying a cached pass. See the lint
|
||||||
# stage of the Dockerfile.
|
# stage of the Dockerfile.
|
||||||
|
.git/
|
||||||
bin/
|
bin/
|
||||||
# Extracted from 3p/ by `make assets` inside the build; a host copy is not
|
# Extracted from 3p/ by `make assets` inside the build; a host copy is not
|
||||||
# needed. The tarball in 3p/ must stay in the context.
|
# needed. The tarball in 3p/ must stay in the context.
|
||||||
static/js/alpine.min.js
|
static/js/alpine.min.js
|
||||||
|
*.md
|
||||||
|
LICENSE
|
||||||
|
.editorconfig
|
||||||
.env
|
.env
|
||||||
.env.*
|
.env.*
|
||||||
*.db
|
*.db
|
||||||
|
|||||||
@@ -12,8 +12,9 @@ jobs:
|
|||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 2024-10-23
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 2024-10-23
|
||||||
with:
|
with:
|
||||||
# The superseded-status step needs history to walk ancestors (it
|
# The fingerprint step below needs history to find the last commit
|
||||||
# aborts on a shallow clone).
|
# that touched the Docker build context, and the superseded-status
|
||||||
|
# step needs it to walk ancestors (it aborts on a shallow clone).
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
|
|
||||||
- name: Mark superseded run statuses
|
- name: Mark superseded run statuses
|
||||||
@@ -27,11 +28,16 @@ jobs:
|
|||||||
run: script/ci-mark-superseded
|
run: script/ci-mark-superseded
|
||||||
|
|
||||||
- name: Fingerprint the build context
|
- name: Fingerprint the build context
|
||||||
# Writes the hash of the commit being checked into the context, which
|
# `.dockerignore` keeps docs out of the build context, so a docs-only
|
||||||
# invalidates the `COPY . .` layer of both check stages: a commit
|
# commit legitimately replays the whole image from cache and stays
|
||||||
# that was never linted, format-checked, tested and built cannot
|
# cheap. Every other commit writes a new fingerprint into the context,
|
||||||
# report success from cache.
|
# which invalidates the `COPY . .` layer of both check stages: a
|
||||||
run: git rev-parse HEAD > .ci-fingerprint
|
# commit that was never linted, formatted-checked, tested and built
|
||||||
|
# cannot report success from cache.
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
fp="$(git log -1 --format=%H -- . ':!*.md' ':!LICENSE' ':!.editorconfig')"
|
||||||
|
printf '%s\n' "${fp:-$GITHUB_SHA}" > .ci-fingerprint
|
||||||
|
|
||||||
- name: Build Docker image (runs make fmt-check, golangci-lint, make test, make build)
|
- name: Build Docker image (runs make check)
|
||||||
run: script/cibuild
|
run: script/cibuild
|
||||||
|
|||||||
Binary file not shown.
Binary file not shown.
+10
-24
@@ -12,8 +12,8 @@ WORKDIR /src
|
|||||||
COPY go.mod go.sum ./
|
COPY go.mod go.sum ./
|
||||||
RUN go mod download
|
RUN go mod download
|
||||||
|
|
||||||
# Copy source code. In CI the context also carries .ci-fingerprint, which
|
# Copy source code. In CI the context also carries .ci-fingerprint, whose
|
||||||
# holds the hash of the commit being checked (see
|
# value changes with every commit that touches the build context (see
|
||||||
# .gitea/workflows/check.yml). That invalidates this layer, so the checks
|
# .gitea/workflows/check.yml). That invalidates this layer, so the checks
|
||||||
# below cannot report success by replaying a cached pass. Do not add it to
|
# below cannot report success by replaying a cached pass. Do not add it to
|
||||||
# .dockerignore.
|
# .dockerignore.
|
||||||
@@ -25,9 +25,8 @@ COPY . .
|
|||||||
# would need a docker daemon inside the build. Keep these steps in step with
|
# would need a docker daemon inside the build. Keep these steps in step with
|
||||||
# Dockerfile.lint, including --network=none (see its header for why).
|
# Dockerfile.lint, including --network=none (see its header for why).
|
||||||
RUN make fmt-check
|
RUN make fmt-check
|
||||||
RUN script/assets
|
|
||||||
RUN --network=none golangci-lint config verify --config .golangci.yml
|
RUN --network=none golangci-lint config verify --config .golangci.yml
|
||||||
RUN --network=none golangci-lint run --config .golangci.yml --build-tags browser ./...
|
RUN --network=none golangci-lint run --config .golangci.yml ./...
|
||||||
|
|
||||||
# Build stage
|
# Build stage
|
||||||
# golang:1.26.1-bookworm (Debian-based), 2026-03-17
|
# golang:1.26.1-bookworm (Debian-based), 2026-03-17
|
||||||
@@ -39,13 +38,8 @@ FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a349228
|
|||||||
COPY --from=lint /src/go.sum /dev/null
|
COPY --from=lint /src/go.sum /dev/null
|
||||||
|
|
||||||
# jq is a runtime dependency of script/ci-mark-superseded, which the test
|
# jq is a runtime dependency of script/ci-mark-superseded, which the test
|
||||||
# suite executes. git is what script/version derives the version with.
|
# suite executes.
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends make curl ca-certificates jq git && rm -rf /var/lib/apt/lists/*
|
RUN apt-get update && apt-get install -y --no-install-recommends make curl ca-certificates jq && rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
# A build context sent as a tar archive keeps its files' owners, and git
|
|
||||||
# refuses to read a checkout owned by another user. Trust this one
|
|
||||||
# whoever owns it.
|
|
||||||
RUN git config --system --add safe.directory /build
|
|
||||||
|
|
||||||
WORKDIR /build
|
WORKDIR /build
|
||||||
|
|
||||||
@@ -61,22 +55,14 @@ COPY . .
|
|||||||
# from its tarball in 3p/.
|
# from its tarball in 3p/.
|
||||||
RUN make test
|
RUN make test
|
||||||
|
|
||||||
# Version stamped into the binary: the VERSION build arg when one is
|
# Version stamped into the binary. .dockerignore excludes .git/, so
|
||||||
# given, otherwise what script/version derives from the .git the build
|
# nothing in this stage can derive it: script/docker resolves it on the
|
||||||
# context carries, so any `docker build .` of a clone stamps its commit.
|
# host and passes it in. The default is what a bare `docker build .`
|
||||||
# With neither, as from a source tarball, it is "unknown".
|
# with no --build-arg gets, and it names no tag the tree may not be at.
|
||||||
#
|
#
|
||||||
# Declared here, below the test step, so a changed version does not
|
# Declared here, below the test step, so a changed version does not
|
||||||
# invalidate its cached layer.
|
# invalidate its cached layer.
|
||||||
ARG VERSION
|
ARG VERSION=unknown
|
||||||
|
|
||||||
# A context that carries .git must not stamp "unknown": that means git is
|
|
||||||
# missing here or could not read the checkout, and the image could not be
|
|
||||||
# traced back to its commit.
|
|
||||||
RUN if [ -d .git ] && [ "$(make version VERSION="$VERSION")" = unknown ]; then \
|
|
||||||
echo "version is unknown although the build context carries .git" >&2; \
|
|
||||||
exit 1; \
|
|
||||||
fi
|
|
||||||
|
|
||||||
RUN make build VERSION="$VERSION"
|
RUN make build VERSION="$VERSION"
|
||||||
|
|
||||||
|
|||||||
@@ -1,29 +0,0 @@
|
|||||||
# Browser test image, built by script/test-browser (make test-browser). It
|
|
||||||
# runs the test in internal/server that loads the pages in a headless
|
|
||||||
# browser under the real Content-Security-Policy. That test is built only
|
|
||||||
# with the browser build tag, so make test leaves it out. Here the browser
|
|
||||||
# comes from a digest-pinned image, and if it is missing the test fails.
|
|
||||||
|
|
||||||
# golang:1.26.1-bookworm, 2026-03-17: the builder stage's image in Dockerfile.
|
|
||||||
FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a3492282a6c820bf4755fd64a4 AS build
|
|
||||||
|
|
||||||
WORKDIR /src
|
|
||||||
|
|
||||||
COPY go.mod go.sum ./
|
|
||||||
RUN go mod download
|
|
||||||
|
|
||||||
COPY . .
|
|
||||||
|
|
||||||
# The test binary embeds the templates and static files, so the browser
|
|
||||||
# stage needs nothing else. -p 4 keeps the compile's memory down, as in
|
|
||||||
# script/test.
|
|
||||||
RUN make assets && go test -c -p 4 -tags browser -o /browser.test ./internal/server
|
|
||||||
|
|
||||||
# chromedp/headless-shell:151.0.7922.109 (Debian trixie), 2026-08-11. The
|
|
||||||
# browser is on PATH as headless-shell, where the test's browser library
|
|
||||||
# looks for it.
|
|
||||||
FROM chromedp/headless-shell:151.0.7922.109@sha256:2d349b544a1ea6b5b5fd7c0fe99215ff662339c57407ee2e8c0a11af93516b04 AS browser
|
|
||||||
|
|
||||||
COPY --from=build /browser.test /browser.test
|
|
||||||
|
|
||||||
RUN /browser.test -test.v -test.timeout 90s -test.run '^TestAlpineRunsUnderTheSecurityPolicy$'
|
|
||||||
+1
-7
@@ -31,13 +31,7 @@ FROM deps AS lint
|
|||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# static/static.go embeds the Alpine.js file this extracts from 3p/; without
|
|
||||||
# it the static package does not compile and cannot be linted.
|
|
||||||
RUN script/assets
|
|
||||||
|
|
||||||
# `run` silently ignores config keys it does not recognize, so a typo would
|
# `run` silently ignores config keys it does not recognize, so a typo would
|
||||||
# disable a setting without a word. `config verify` is what catches that.
|
# disable a setting without a word. `config verify` is what catches that.
|
||||||
RUN --network=none golangci-lint config verify --config .golangci.yml
|
RUN --network=none golangci-lint config verify --config .golangci.yml
|
||||||
# --build-tags browser also lints the browser test, which is built only with
|
RUN --network=none golangci-lint run --config .golangci.yml ./...
|
||||||
# that tag (make test-browser).
|
|
||||||
RUN --network=none golangci-lint run --config .golangci.yml --build-tags browser ./...
|
|
||||||
|
|||||||
@@ -1,15 +1,15 @@
|
|||||||
.PHONY: bootstrap setup assets test test-browser lint fmt fmt-check check build run dev deps docker clean hooks css version
|
.PHONY: bootstrap setup assets test lint fmt fmt-check check build run dev deps docker clean hooks css version
|
||||||
|
|
||||||
# Default target
|
# Default target
|
||||||
.DEFAULT_GOAL := check
|
.DEFAULT_GOAL := check
|
||||||
|
|
||||||
# Version stamped into the binary. Derived from git by script/version;
|
# Version stamped into the binary. Derived from git by script/version;
|
||||||
# override it (`make build VERSION=v1.2.3`) to stamp a given value, which is
|
# override it (`make build VERSION=v1.2.3`) where git metadata is
|
||||||
# how the Dockerfile passes its build arg in.
|
# unavailable, which is how the Dockerfile passes its build arg in.
|
||||||
VERSION ?= $(shell script/version)
|
VERSION ?= $(shell script/version)
|
||||||
|
|
||||||
# An empty override (`make build VERSION=`, or the Dockerfile's `make build
|
# An empty override (`make build VERSION=`, or a `--build-arg VERSION=`
|
||||||
# VERSION="$VERSION"` when no VERSION build arg was given) means unset,
|
# landing on the Dockerfile's `make build VERSION="$VERSION"`) means unset,
|
||||||
# exactly as it does in script/version -- stamping "" would leave the binary
|
# exactly as it does in script/version -- stamping "" would leave the binary
|
||||||
# reporting no version and the footer back on its "dev" fallback. `override`
|
# reporting no version and the footer back on its "dev" fallback. `override`
|
||||||
# is required: a plain assignment loses to the command-line definition it
|
# is required: a plain assignment loses to the command-line definition it
|
||||||
@@ -33,9 +33,6 @@ assets:
|
|||||||
test:
|
test:
|
||||||
@script/test
|
@script/test
|
||||||
|
|
||||||
test-browser:
|
|
||||||
@script/test-browser
|
|
||||||
|
|
||||||
lint:
|
lint:
|
||||||
@script/lint
|
@script/lint
|
||||||
|
|
||||||
|
|||||||
@@ -40,6 +40,12 @@ duplicate. That is deliberate — the alternative is a silent lost
|
|||||||
delivery — and the README says so under Rationale. It is not a defect
|
delivery — and the README says so under Rationale. It is not a defect
|
||||||
to re-file.
|
to re-file.
|
||||||
|
|
||||||
|
One caveat on reading a green check: a docs-only commit deliberately
|
||||||
|
replays from the layer cache
|
||||||
|
(https://git.eeqj.de/sneak/webhooker/issues/119), so a green status on
|
||||||
|
such a commit evidences a replay rather than an executed run. A code
|
||||||
|
commit invalidates the `COPY` layer and genuinely executes.
|
||||||
|
|
||||||
# Next Step
|
# Next Step
|
||||||
|
|
||||||
Clear the rest of the open 1.0.0 milestone
|
Clear the rest of the open 1.0.0 milestone
|
||||||
@@ -381,7 +387,7 @@ point of the branch.
|
|||||||
- 2026-03-05 security headers middleware, session regeneration on
|
- 2026-03-05 security headers middleware, session regeneration on
|
||||||
login, request body size limits (#41)
|
login, request body size limits (#41)
|
||||||
- 2026-03-04 tests for delivery, middleware, and session packages
|
- 2026-03-04 tests for delivery, middleware, and session packages
|
||||||
(#32); removed the build-architecture global (#31)
|
(#32); removed globals.Buildarch (#31)
|
||||||
- 2026-03-04 1.0 MVP merge: Webhook/Entrypoint/Target rename, core
|
- 2026-03-04 1.0 MVP merge: Webhook/Entrypoint/Target rename, core
|
||||||
delivery engine with bounded worker pool and circuit breaker,
|
delivery engine with bounded worker pool and circuit breaker,
|
||||||
parallel fan-out, per-webhook event databases, management UI (#16)
|
parallel fan-out, per-webhook event databases, management UI (#16)
|
||||||
|
|||||||
+12
-32
@@ -8,7 +8,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"go.uber.org/fx/fxevent"
|
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/datadir"
|
"sneak.berlin/go/webhooker/internal/datadir"
|
||||||
@@ -17,7 +16,6 @@ import (
|
|||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
"sneak.berlin/go/webhooker/internal/handlers"
|
||||||
"sneak.berlin/go/webhooker/internal/healthcheck"
|
"sneak.berlin/go/webhooker/internal/healthcheck"
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
"sneak.berlin/go/webhooker/internal/metrics"
|
|
||||||
"sneak.berlin/go/webhooker/internal/middleware"
|
"sneak.berlin/go/webhooker/internal/middleware"
|
||||||
"sneak.berlin/go/webhooker/internal/resetpw"
|
"sneak.berlin/go/webhooker/internal/resetpw"
|
||||||
"sneak.berlin/go/webhooker/internal/server"
|
"sneak.berlin/go/webhooker/internal/server"
|
||||||
@@ -38,19 +36,17 @@ import (
|
|||||||
// hook that used the whole budget would exhaust it at that instant,
|
// hook that used the whole budget would exhaust it at that instant,
|
||||||
// and fx would skip every hook after the server — the delivery
|
// and fx would skip every hook after the server — the delivery
|
||||||
// engine, the healthcheck, the webhook DB manager and the database
|
// engine, the healthcheck, the webhook DB manager and the database
|
||||||
// close. That hook is the HTTP drain plus the Sentry flush that
|
// close. That hook is the 3s HTTP drain plus the Sentry flush that
|
||||||
// follows it in the same hook, and each is clamped to the stop
|
// follows it in the same hook, so the flush is clamped to the stop
|
||||||
// context's remaining time less server.TailHookReserve rather than
|
// context's remaining time less server.TailHookReserve rather than
|
||||||
// running for its own fixed 3s and 2s; the reserve is what the tail
|
// running for its own fixed 2s; the reserve is what the tail hooks
|
||||||
// hooks live on, and they are microsecond-scale in normal operation.
|
// live on, and they are microsecond-scale in normal operation.
|
||||||
// TestStopTimeout_LeavesHeadroomForTailHooks pins the arithmetic
|
// TestStopTimeout_LeavesHeadroomForTailHooks pins the arithmetic
|
||||||
// across every drain length and every amount of budget the hooks
|
// across every drain length.
|
||||||
// before the server may already have spent.
|
|
||||||
//
|
//
|
||||||
// This does not make the database close unconditional: the
|
// This does not make the database close unconditional: the
|
||||||
// ArchiveSweeper and RetentionReaper hooks run before the server.
|
// ArchiveSweeper and RetentionReaper hooks run before the server
|
||||||
// What they spend comes out of the drain first, but past 3s it comes
|
// and can still consume the whole budget on their own.
|
||||||
// out of the reserve, and they can consume the whole budget.
|
|
||||||
const stopTimeout = 5 * time.Second
|
const stopTimeout = 5 * time.Second
|
||||||
|
|
||||||
// exitUsage is the status for a command line this binary cannot make
|
// exitUsage is the status for a command line this binary cannot make
|
||||||
@@ -171,19 +167,6 @@ func run(stderr io.Writer) int {
|
|||||||
func newApp() *fx.App {
|
func newApp() *fx.App {
|
||||||
return fx.New(
|
return fx.New(
|
||||||
fx.StopTimeout(stopTimeout),
|
fx.StopTimeout(stopTimeout),
|
||||||
// fx's own events go through the service's logger, not fx's
|
|
||||||
// console logger on standard error. The exception is a failure
|
|
||||||
// before this logger is built, such as an invalid configuration
|
|
||||||
// value, which fx's console logger still prints there. fx holds
|
|
||||||
// its events back until this logger is built and then replays
|
|
||||||
// them, so it takes the configuration, which sets the level
|
|
||||||
// DEBUG=true asks for: without it the replay would run at INFO
|
|
||||||
// and drop every record of how the graph was built.
|
|
||||||
fx.WithLogger(
|
|
||||||
func(l *logger.Logger, _ *config.Config) fxevent.Logger {
|
|
||||||
return logger.NewFxLogger(l.Get())
|
|
||||||
},
|
|
||||||
),
|
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
@@ -194,10 +177,6 @@ func newApp() *fx.App {
|
|||||||
healthcheck.New,
|
healthcheck.New,
|
||||||
session.New,
|
session.New,
|
||||||
handlers.New,
|
handlers.New,
|
||||||
// The registry /metrics serves, and the delivery
|
|
||||||
// collectors registered on it.
|
|
||||||
metrics.NewRegistry,
|
|
||||||
metrics.New,
|
|
||||||
middleware.New,
|
middleware.New,
|
||||||
// The one SSRF guard both target-creation validation
|
// The one SSRF guard both target-creation validation
|
||||||
// and the delivery dialer consult, so they cannot
|
// and the delivery dialer consult, so they cannot
|
||||||
@@ -208,10 +187,11 @@ func newApp() *fx.App {
|
|||||||
// Wire *delivery.Engine as delivery.Notifier so the
|
// Wire *delivery.Engine as delivery.Notifier so the
|
||||||
// webhook handler can notify the engine of new deliveries.
|
// webhook handler can notify the engine of new deliveries.
|
||||||
func(e *delivery.Engine) delivery.Notifier { return e },
|
func(e *delivery.Engine) delivery.Notifier { return e },
|
||||||
// Wire *delivery.Engine as delivery.Archives so deleting
|
// Wire *delivery.Engine as delivery.WebhookEvictor so
|
||||||
// or renaming a webhook or target reaches its archive
|
// deleting a webhook releases its archive writer.
|
||||||
// files.
|
func(e *delivery.Engine) delivery.WebhookEvictor {
|
||||||
func(e *delivery.Engine) delivery.Archives { return e },
|
return e
|
||||||
|
},
|
||||||
server.New,
|
server.New,
|
||||||
),
|
),
|
||||||
fx.Invoke(
|
fx.Invoke(
|
||||||
|
|||||||
+9
-129
@@ -2,19 +2,12 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
|
||||||
"io"
|
|
||||||
"log/slog"
|
|
||||||
"net"
|
|
||||||
"os"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
|
||||||
"sneak.berlin/go/webhooker/internal/datadir"
|
"sneak.berlin/go/webhooker/internal/datadir"
|
||||||
"sneak.berlin/go/webhooker/internal/resetpw"
|
"sneak.berlin/go/webhooker/internal/resetpw"
|
||||||
"sneak.berlin/go/webhooker/internal/server"
|
"sneak.berlin/go/webhooker/internal/server"
|
||||||
@@ -37,7 +30,6 @@ const dockerStopGrace = 10 * time.Second
|
|||||||
// fx.New applies options before it executes invokes, so the timeout
|
// fx.New applies options before it executes invokes, so the timeout
|
||||||
// is set whether or not the graph itself can be constructed here.
|
// is set whether or not the graph itself can be constructed here.
|
||||||
func TestNewApp_StopTimeout(t *testing.T) {
|
func TestNewApp_StopTimeout(t *testing.T) {
|
||||||
config.ClearEnvForTest(t)
|
|
||||||
t.Setenv("DATA_DIR", t.TempDir())
|
t.Setenv("DATA_DIR", t.TempDir())
|
||||||
|
|
||||||
got := newApp().StopTimeout()
|
got := newApp().StopTimeout()
|
||||||
@@ -46,100 +38,6 @@ func TestNewApp_StopTimeout(t *testing.T) {
|
|||||||
require.Less(t, got, dockerStopGrace)
|
require.Less(t, got, dockerStopGrace)
|
||||||
}
|
}
|
||||||
|
|
||||||
// freePort returns a loopback TCP port that was free a moment ago, by
|
|
||||||
// taking one and releasing it.
|
|
||||||
func freePort(t *testing.T) int {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var listenCfg net.ListenConfig
|
|
||||||
|
|
||||||
l, err := listenCfg.Listen(t.Context(), "tcp", "127.0.0.1:0")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
addr, ok := l.Addr().(*net.TCPAddr)
|
|
||||||
require.True(t, ok, "listener is not TCP")
|
|
||||||
require.NoError(t, l.Close())
|
|
||||||
|
|
||||||
return addr.Port
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestNewApp_SendsFxEventsToTheLogger starts and stops the app main
|
|
||||||
// runs, with DEBUG=true, and reads back what reached the service's
|
|
||||||
// logger. fx's own events must arrive there as structured records:
|
|
||||||
// the start at INFO, and at DEBUG the records of how the graph was
|
|
||||||
// built.
|
|
||||||
//
|
|
||||||
// fx holds its events back until its logger is built and then replays
|
|
||||||
// them all at once, so the earliest of them arriving shows the replay
|
|
||||||
// ran at DEBUG: that globals.New was provided, which fx records before
|
|
||||||
// anything is built, and the run of logger.New, which happens before
|
|
||||||
// the configuration sets the level.
|
|
||||||
func TestNewApp_SendsFxEventsToTheLogger(t *testing.T) {
|
|
||||||
config.ClearEnvForTest(t)
|
|
||||||
t.Setenv("DATA_DIR", t.TempDir())
|
|
||||||
t.Setenv("PORT", strconv.Itoa(freePort(t)))
|
|
||||||
t.Setenv("DEBUG", "true")
|
|
||||||
|
|
||||||
// internal/logger writes to whatever os.Stdout is when it builds
|
|
||||||
// its handler. A file is not a terminal, so that handler is the
|
|
||||||
// JSON one the service uses in production.
|
|
||||||
out, err := os.CreateTemp(t.TempDir(), "stdout")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
stdout := os.Stdout
|
|
||||||
os.Stdout = out
|
|
||||||
|
|
||||||
t.Cleanup(func() {
|
|
||||||
os.Stdout = stdout
|
|
||||||
_ = out.Close()
|
|
||||||
})
|
|
||||||
|
|
||||||
app := newApp()
|
|
||||||
require.NoError(t, app.Start(t.Context()))
|
|
||||||
require.NoError(t, app.Stop(t.Context()))
|
|
||||||
|
|
||||||
_, err = out.Seek(0, io.SeekStart)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
written, err := io.ReadAll(out)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
type record struct {
|
|
||||||
Level string `json:"level"`
|
|
||||||
Msg string `json:"msg"`
|
|
||||||
Name string `json:"name"`
|
|
||||||
Constructor string `json:"constructor"`
|
|
||||||
}
|
|
||||||
|
|
||||||
var records []record
|
|
||||||
|
|
||||||
for line := range strings.Lines(string(written)) {
|
|
||||||
var r record
|
|
||||||
|
|
||||||
// The first-boot banner is plain text, not a record.
|
|
||||||
if json.Unmarshal([]byte(line), &r) == nil {
|
|
||||||
records = append(records, r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const pkg = "sneak.berlin/go/webhooker/internal/"
|
|
||||||
|
|
||||||
info := slog.LevelInfo.String()
|
|
||||||
debug := slog.LevelDebug.String()
|
|
||||||
|
|
||||||
assert.Contains(t, records, record{Level: info, Msg: "started"})
|
|
||||||
assert.Contains(t, records, record{
|
|
||||||
Level: debug, Msg: "provided", Constructor: pkg + "globals.New()",
|
|
||||||
})
|
|
||||||
assert.Contains(t, records, record{
|
|
||||||
Level: debug, Msg: "run", Name: pkg + "logger.New()",
|
|
||||||
})
|
|
||||||
assert.Contains(t, records, record{Level: debug, Msg: "invoking"})
|
|
||||||
assert.Contains(t, records, record{
|
|
||||||
Level: debug, Msg: "initialized custom fxevent.Logger",
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRunRefusesLockedDataDir pins what an operator's second start
|
// TestRunRefusesLockedDataDir pins what an operator's second start
|
||||||
// does. The entry point must refuse before it builds the fx graph —
|
// does. The entry point must refuse before it builds the fx graph —
|
||||||
// nothing may open a database in a DATA_DIR another process holds —
|
// nothing may open a database in a DATA_DIR another process holds —
|
||||||
@@ -252,40 +150,22 @@ const tailHeadroom = 2 * time.Second
|
|||||||
// can produce, since a shorter drain leaves the flush more room and
|
// can produce, since a shorter drain leaves the flush more room and
|
||||||
// the worst case is not necessarily at either extreme.
|
// the worst case is not necessarily at either extreme.
|
||||||
//
|
//
|
||||||
// Nor does the hook start on a full budget: the ArchiveSweeper and
|
// Shrinking either budget, or unbounding the flush again, must fail
|
||||||
// RetentionReaper hooks run before it, and whatever they spent is
|
// here rather than silently recreating a hook that swallows the
|
||||||
// gone. The outer sweep walks every amount they can spend. Once they
|
// whole sequence.
|
||||||
// have eaten into the headroom themselves, the hook must spend
|
|
||||||
// nothing of what is left. A drain that starts on the full budget
|
|
||||||
// must still get all of ShutdownTimeout, so a smaller stopTimeout
|
|
||||||
// cannot silently shorten every drain.
|
|
||||||
//
|
|
||||||
// Shrinking either budget, or unbounding the drain or the flush
|
|
||||||
// again, must fail here rather than silently recreating a hook that
|
|
||||||
// swallows the whole sequence.
|
|
||||||
func TestStopTimeout_LeavesHeadroomForTailHooks(t *testing.T) {
|
func TestStopTimeout_LeavesHeadroomForTailHooks(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
require.Less(t, server.ShutdownTimeout, stopTimeout)
|
require.Less(t, server.ShutdownTimeout, stopTimeout)
|
||||||
require.Equal(
|
|
||||||
t, server.ShutdownTimeout, server.DrainBudget(stopTimeout),
|
|
||||||
"a drain that starts on the full stop budget is cut short",
|
|
||||||
)
|
|
||||||
|
|
||||||
const step = 10 * time.Millisecond
|
const step = 10 * time.Millisecond
|
||||||
|
|
||||||
for spent := time.Duration(0); spent <= stopTimeout; spent += step {
|
for drain := time.Duration(0); drain <= server.ShutdownTimeout; drain += step {
|
||||||
remaining := stopTimeout - spent
|
hook := drain + server.SentryFlushBudget(stopTimeout-drain)
|
||||||
longest := max(server.DrainBudget(remaining), 0)
|
|
||||||
|
|
||||||
for drain := time.Duration(0); drain <= longest; drain += step {
|
require.LessOrEqual(
|
||||||
hook := drain + server.SentryFlushBudget(remaining-drain)
|
t, hook+tailHeadroom, stopTimeout,
|
||||||
|
"a %s drain leaves the tail hooks short", drain,
|
||||||
require.GreaterOrEqual(
|
)
|
||||||
t, remaining-hook, min(remaining, tailHeadroom),
|
|
||||||
"a %s drain after %s of earlier hooks leaves "+
|
|
||||||
"the tail hooks short", drain, spent,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,9 +4,6 @@ go 1.26.1
|
|||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8
|
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8
|
||||||
github.com/chromedp/cdproto v0.0.0-20260922220944-a19bff23514f
|
|
||||||
github.com/chromedp/chromedp v0.16.0
|
|
||||||
github.com/dustin/go-humanize v1.0.1
|
|
||||||
github.com/getsentry/sentry-go v0.25.0
|
github.com/getsentry/sentry-go v0.25.0
|
||||||
github.com/go-chi/chi v1.5.5
|
github.com/go-chi/chi v1.5.5
|
||||||
github.com/go-chi/cors v1.2.1
|
github.com/go-chi/cors v1.2.1
|
||||||
@@ -20,7 +17,7 @@ require (
|
|||||||
github.com/prometheus/client_model v0.5.0
|
github.com/prometheus/client_model v0.5.0
|
||||||
github.com/slok/go-http-metrics v0.11.0
|
github.com/slok/go-http-metrics v0.11.0
|
||||||
github.com/stretchr/testify v1.11.1
|
github.com/stretchr/testify v1.11.1
|
||||||
go.uber.org/fx v1.24.0
|
go.uber.org/fx v1.20.1
|
||||||
golang.org/x/crypto v0.38.0
|
golang.org/x/crypto v0.38.0
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
gorm.io/driver/sqlite v1.5.4
|
gorm.io/driver/sqlite v1.5.4
|
||||||
@@ -31,17 +28,14 @@ require (
|
|||||||
require (
|
require (
|
||||||
github.com/beorn7/perks v1.0.1 // indirect
|
github.com/beorn7/perks v1.0.1 // indirect
|
||||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||||
github.com/chromedp/sysutil v1.1.0 // indirect
|
|
||||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||||
github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3 // indirect
|
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||||
github.com/gobwas/httphead v0.1.0 // indirect
|
|
||||||
github.com/gobwas/pool v0.2.1 // indirect
|
|
||||||
github.com/gobwas/ws v1.4.0 // indirect
|
|
||||||
github.com/gorilla/securecookie v1.1.2 // indirect
|
github.com/gorilla/securecookie v1.1.2 // indirect
|
||||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||||
github.com/jinzhu/now v1.1.5 // indirect
|
github.com/jinzhu/now v1.1.5 // indirect
|
||||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||||
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
||||||
|
github.com/kr/text v0.2.0 // indirect
|
||||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||||
github.com/mattn/go-sqlite3 v1.14.17 // indirect
|
github.com/mattn/go-sqlite3 v1.14.17 // indirect
|
||||||
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
|
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
|
||||||
@@ -50,12 +44,13 @@ require (
|
|||||||
github.com/prometheus/procfs v0.12.0 // indirect
|
github.com/prometheus/procfs v0.12.0 // indirect
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
github.com/zeebo/xxh3 v1.0.2 // indirect
|
||||||
go.uber.org/dig v1.19.0 // indirect
|
go.uber.org/atomic v1.9.0 // indirect
|
||||||
go.uber.org/multierr v1.10.0 // indirect
|
go.uber.org/dig v1.17.0 // indirect
|
||||||
go.uber.org/zap v1.26.0 // indirect
|
go.uber.org/multierr v1.9.0 // indirect
|
||||||
|
go.uber.org/zap v1.23.0 // indirect
|
||||||
golang.org/x/mod v0.17.0 // indirect
|
golang.org/x/mod v0.17.0 // indirect
|
||||||
golang.org/x/sync v0.14.0 // indirect
|
golang.org/x/sync v0.14.0 // indirect
|
||||||
golang.org/x/sys v0.47.0 // indirect
|
golang.org/x/sys v0.37.0 // indirect
|
||||||
golang.org/x/text v0.25.0 // indirect
|
golang.org/x/text v0.25.0 // indirect
|
||||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
|
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
|
||||||
google.golang.org/protobuf v1.31.0 // indirect
|
google.golang.org/protobuf v1.31.0 // indirect
|
||||||
|
|||||||
@@ -1,15 +1,14 @@
|
|||||||
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8 h1:nMpu1t4amK3vJWBibQ5X/Nv0aXL+b69TQf2uK5PH7Go=
|
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8 h1:nMpu1t4amK3vJWBibQ5X/Nv0aXL+b69TQf2uK5PH7Go=
|
||||||
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8/go.mod h1:3cARGAK9CfW3HoxCy1a0G4TKrdiKke8ftOMEOHyySYs=
|
github.com/99designs/basicauth-go v0.0.0-20230316000542-bf6f9cbbf0f8/go.mod h1:3cARGAK9CfW3HoxCy1a0G4TKrdiKke8ftOMEOHyySYs=
|
||||||
|
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
|
||||||
|
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||||
github.com/chromedp/cdproto v0.0.0-20260922220944-a19bff23514f h1:8PK9FM4bE0C8GMoWBW5lVsef3U7sPICjDg6JqngyYhk=
|
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||||
github.com/chromedp/cdproto v0.0.0-20260922220944-a19bff23514f/go.mod h1:3v4FIp5njIUyPDvqXsxEOxnB34lijG0up98/5kM1KaE=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/chromedp/chromedp v0.16.0 h1:rOO4deOm4CbZgBCa8mD9g2rDyIoNs0BkgvNrlbp5ouk=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/chromedp/chromedp v0.16.0/go.mod h1:rbuGKFT1vMcFcFqKfPIO1GpX/N+2s8onm2qMxZLbU5U=
|
|
||||||
github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
|
|
||||||
github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
|
|
||||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||||
@@ -24,14 +23,6 @@ github.com/go-chi/httprate v0.15.0 h1:j54xcWV9KGmPf/X4H32/aTH+wBlrvxL7P+SdnRqxh5
|
|||||||
github.com/go-chi/httprate v0.15.0/go.mod h1:rzGHhVrsBn3IMLYDOZQsSU4fJNWcjui4fWKJcCId1R4=
|
github.com/go-chi/httprate v0.15.0/go.mod h1:rzGHhVrsBn3IMLYDOZQsSU4fJNWcjui4fWKJcCId1R4=
|
||||||
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
||||||
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||||
github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3 h1:UADEEmDKgfXbtnGJZ97beY5XLo9ZechG1nlU4KnRrkE=
|
|
||||||
github.com/go-json-experiment/json v0.0.0-20260820222146-c27c302e5fc3/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg=
|
|
||||||
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
|
|
||||||
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
|
|
||||||
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
|
|
||||||
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
|
||||||
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
|
|
||||||
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
|
|
||||||
github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw=
|
github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw=
|
||||||
github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0=
|
github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0=
|
||||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||||
@@ -64,20 +55,17 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
|||||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
|
|
||||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
|
|
||||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||||
github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
|
github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
|
||||||
github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||||
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
|
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
|
||||||
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
|
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
|
||||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
|
|
||||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
|
|
||||||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
||||||
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
||||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
|
github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
|
||||||
@@ -94,24 +82,28 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR
|
|||||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||||
github.com/slok/go-http-metrics v0.11.0 h1:ABJUpekCZSkQT1wQrFvS4kGbhea/w6ndFJaWJeh3zL0=
|
github.com/slok/go-http-metrics v0.11.0 h1:ABJUpekCZSkQT1wQrFvS4kGbhea/w6ndFJaWJeh3zL0=
|
||||||
github.com/slok/go-http-metrics v0.11.0/go.mod h1:ZGKeYG1ET6TEJpQx18BqAJAvxw9jBAZXCHU7bWQqqAc=
|
github.com/slok/go-http-metrics v0.11.0/go.mod h1:ZGKeYG1ET6TEJpQx18BqAJAvxw9jBAZXCHU7bWQqqAc=
|
||||||
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
||||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||||
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
||||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
||||||
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
||||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
||||||
go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4=
|
go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=
|
||||||
go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE=
|
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||||
go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg=
|
go.uber.org/dig v1.17.0 h1:5Chju+tUvcC+N7N6EV08BJz41UZuO3BmHcN4A287ZLI=
|
||||||
go.uber.org/fx v1.24.0/go.mod h1:AmDeGyS+ZARGKM4tlH4FY2Jr63VjbEDJHtqXTGP5hbo=
|
go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU=
|
||||||
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
|
go.uber.org/fx v1.20.1 h1:zVwVQGS8zYvhh9Xxcu4w1M6ESyeMzebzj2NbSayZ4Mk=
|
||||||
go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
|
go.uber.org/fx v1.20.1/go.mod h1:iSYNbHf2y55acNCwCXKx7LbWb5WG1Bnue5RDXz1OREg=
|
||||||
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
|
go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
|
||||||
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
|
||||||
go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo=
|
go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
|
||||||
go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so=
|
go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
|
||||||
|
go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
|
||||||
|
go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
|
||||||
golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
|
golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
|
||||||
golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
|
golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
|
||||||
golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
|
golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
|
||||||
@@ -119,8 +111,8 @@ golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
|||||||
golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
|
golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
|
||||||
golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
|
||||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||||
golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
|
golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
|
||||||
golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
|
golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
|
||||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
|
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
|
||||||
|
|||||||
+84
-48
@@ -75,12 +75,8 @@ const (
|
|||||||
// internet-exposed endpoint.
|
// internet-exposed endpoint.
|
||||||
defaultReceiverRateLimit = 120
|
defaultReceiverRateLimit = 120
|
||||||
|
|
||||||
// defaultTrustedProxies is TRUSTED_PROXIES when it is unset: the
|
// maxPort is the highest valid TCP port number. The lower
|
||||||
// RFC 1918 private ranges, which a reverse proxy reaching the
|
// bound (at least 1) is enforced by envPositiveInt.
|
||||||
// process over a Docker network or a private LAN connects from.
|
|
||||||
defaultTrustedProxies = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
|
|
||||||
|
|
||||||
// maxPort is the highest valid TCP port number.
|
|
||||||
maxPort = 65535
|
maxPort = 65535
|
||||||
|
|
||||||
// mappedV4Offset is the number of leading bits an IPv4-mapped
|
// mappedV4Offset is the number of leading bits an IPv4-mapped
|
||||||
@@ -104,7 +100,7 @@ var ErrInvalidEnvironment = errors.New("invalid environment")
|
|||||||
var ErrNonPositiveValue = errors.New("value must be positive")
|
var ErrNonPositiveValue = errors.New("value must be positive")
|
||||||
|
|
||||||
// ErrInvalidPort is returned when an environment variable holding a
|
// ErrInvalidPort is returned when an environment variable holding a
|
||||||
// TCP port number is set to a number outside 1 to 65535.
|
// TCP port number is set above the valid port range.
|
||||||
var ErrInvalidPort = errors.New("invalid port")
|
var ErrInvalidPort = errors.New("invalid port")
|
||||||
|
|
||||||
// ErrInvalidCIDR is returned when an environment variable holding a
|
// ErrInvalidCIDR is returned when an environment variable holding a
|
||||||
@@ -148,6 +144,7 @@ type ConfigParams struct {
|
|||||||
type Config struct {
|
type Config struct {
|
||||||
DataDir string
|
DataDir string
|
||||||
Debug bool
|
Debug bool
|
||||||
|
MaintenanceMode bool
|
||||||
Environment string
|
Environment string
|
||||||
MetricsPassword string
|
MetricsPassword string
|
||||||
MetricsUsername string
|
MetricsUsername string
|
||||||
@@ -175,14 +172,13 @@ type Config struct {
|
|||||||
|
|
||||||
// TrustedProxies is the set of networks whose members are
|
// TrustedProxies is the set of networks whose members are
|
||||||
// allowed to speak for the client with X-Forwarded-For, the
|
// allowed to speak for the client with X-Forwarded-For, the
|
||||||
// only forwarded header read. Unless TRUSTED_PROXIES is set it
|
// only forwarded header read. It is empty unless
|
||||||
// is the RFC 1918 private ranges (defaultTrustedProxies); a set
|
// TRUSTED_PROXIES is set, and empty means no peer is
|
||||||
// value replaces them. If any client can reach the process, or
|
// trusted: forwarded headers are then ignored entirely and
|
||||||
// the proxy in front of it, from an RFC 1918 source address
|
// clients are identified by the connection's own address.
|
||||||
// (directly, or through anything that can rewrite source
|
// Members can choose their own rate-limit key, so this must
|
||||||
// addresses, such as NAT or a published container port), it
|
// name proxy hosts only, never a block that also covers
|
||||||
// must be set to the proxy's address alone, or every rate limit
|
// clients.
|
||||||
// can be bypassed by those clients.
|
|
||||||
TrustedProxies []netip.Prefix
|
TrustedProxies []netip.Prefix
|
||||||
|
|
||||||
// AllowedEgressCIDRs is the set of networks a delivery target
|
// AllowedEgressCIDRs is the set of networks a delivery target
|
||||||
@@ -194,13 +190,12 @@ type Config struct {
|
|||||||
// otherwise refuse. The guard itself is always on: there is no
|
// otherwise refuse. The guard itself is always on: there is no
|
||||||
// setting that disables SSRF protection, and delivery's
|
// setting that disables SSRF protection, and delivery's
|
||||||
// alwaysBlockedNetworks stays blocked no matter what is listed
|
// alwaysBlockedNetworks stays blocked no matter what is listed
|
||||||
// here. That set is link-local, the unspecified addresses
|
// here. That set is link-local plus the cloud metadata
|
||||||
// 0.0.0.0 and ::, and the cloud metadata endpoints outside
|
// endpoints outside it that disclose credentials or user data
|
||||||
// link-local that disclose credentials or user data at a
|
// at a provider-fixed, non-public address; it is not
|
||||||
// provider-fixed, non-public address; it is not exhaustive of
|
// exhaustive of every cloud's metadata address. See
|
||||||
// every cloud's metadata address. See
|
// alwaysBlockedNetworks for the authoritative list and the
|
||||||
// alwaysBlockedNetworks for the authoritative list and why
|
// criterion it is built from.
|
||||||
// each entry is on it.
|
|
||||||
AllowedEgressCIDRs []netip.Prefix
|
AllowedEgressCIDRs []netip.Prefix
|
||||||
|
|
||||||
params *ConfigParams
|
params *ConfigParams
|
||||||
@@ -362,27 +357,17 @@ func envPositiveInt(
|
|||||||
// envPort returns the value of the named environment variable parsed
|
// envPort returns the value of the named environment variable parsed
|
||||||
// as a TCP port number. Returns defaultValue if not set. A set value
|
// as a TCP port number. Returns defaultValue if not set. A set value
|
||||||
// that is unparseable, below 1, or above maxPort is a hard error
|
// that is unparseable, below 1, or above maxPort is a hard error
|
||||||
// naming the key and the bad value; every out-of-range value wraps
|
// naming the key and the bad value.
|
||||||
// ErrInvalidPort, including one too large or too small for an int.
|
|
||||||
func envPort(key string, defaultValue int) (int, error) {
|
func envPort(key string, defaultValue int) (int, error) {
|
||||||
v := os.Getenv(key)
|
port, err := envPositiveInt(key, defaultValue)
|
||||||
if v == "" {
|
if err != nil {
|
||||||
return defaultValue, nil
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// strconv.ErrRange means a number too large or too small for an
|
if port > maxPort {
|
||||||
// int, which is outside the port range as well.
|
|
||||||
port, err := strconv.Atoi(v)
|
|
||||||
if err != nil && !errors.Is(err, strconv.ErrRange) {
|
|
||||||
return 0, fmt.Errorf(
|
return 0, fmt.Errorf(
|
||||||
"invalid integer for %s: %q: %w", key, v, err,
|
"%w: %s must be at most %d, got %d",
|
||||||
)
|
ErrInvalidPort, key, maxPort, port,
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil || port < 1 || port > maxPort {
|
|
||||||
return 0, fmt.Errorf(
|
|
||||||
"%w: %s must be from 1 to %d, got %q",
|
|
||||||
ErrInvalidPort, key, maxPort, v,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -475,15 +460,14 @@ func parseCIDR(entry string) (netip.Prefix, error) {
|
|||||||
|
|
||||||
// envPrefixList returns the value of the named environment variable
|
// envPrefixList returns the value of the named environment variable
|
||||||
// parsed as a comma-separated list of CIDR blocks (bare addresses
|
// parsed as a comma-separated list of CIDR blocks (bare addresses
|
||||||
// allowed). An unset, empty, or blank value is read as defaultValue
|
// allowed). An unset, empty, or blank value yields an empty list. A
|
||||||
// instead. A set value containing an unparseable entry is a hard
|
// set value containing an unparseable entry is a hard error naming
|
||||||
// error naming the key and the bad entry, so startup fails loudly
|
// the key and the bad entry, so startup fails loudly rather than
|
||||||
// rather than silently running with a list the operator did not
|
// silently running with a list the operator did not intend.
|
||||||
// intend.
|
func envPrefixList(key string) ([]netip.Prefix, error) {
|
||||||
func envPrefixList(key, defaultValue string) ([]netip.Prefix, error) {
|
|
||||||
v := strings.TrimSpace(os.Getenv(key))
|
v := strings.TrimSpace(os.Getenv(key))
|
||||||
if v == "" {
|
if v == "" {
|
||||||
v = defaultValue
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var prefixes []netip.Prefix
|
var prefixes []netip.Prefix
|
||||||
@@ -666,6 +650,11 @@ func loadFromEnv() (*Config, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
maintenanceMode, err := envBool("MAINTENANCE_MODE", false)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
retentionSweepInterval, err := envPositiveDuration(
|
retentionSweepInterval, err := envPositiveDuration(
|
||||||
"RETENTION_SWEEP_INTERVAL",
|
"RETENTION_SWEEP_INTERVAL",
|
||||||
defaultRetentionSweepInterval,
|
defaultRetentionSweepInterval,
|
||||||
@@ -692,12 +681,12 @@ func loadFromEnv() (*Config, error) {
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
trustedProxies, err := envPrefixList("TRUSTED_PROXIES", defaultTrustedProxies)
|
trustedProxies, err := envPrefixList("TRUSTED_PROXIES")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
allowedEgressCIDRs, err := envPrefixList("ALLOWED_EGRESS_CIDRS", "")
|
allowedEgressCIDRs, err := envPrefixList("ALLOWED_EGRESS_CIDRS")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -715,6 +704,7 @@ func loadFromEnv() (*Config, error) {
|
|||||||
return &Config{
|
return &Config{
|
||||||
DataDir: DataDir(),
|
DataDir: DataDir(),
|
||||||
Debug: debug,
|
Debug: debug,
|
||||||
|
MaintenanceMode: maintenanceMode,
|
||||||
Environment: environment,
|
Environment: environment,
|
||||||
MetricsUsername: metricsUsername,
|
MetricsUsername: metricsUsername,
|
||||||
MetricsPassword: metricsPassword,
|
MetricsPassword: metricsPassword,
|
||||||
@@ -770,6 +760,50 @@ func (c *Config) warnEgressAllowlist(log *slog.Logger) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// warnSharedRateLimitBucket logs a startup warning whenever
|
||||||
|
// TRUSTED_PROXIES is empty, in any environment.
|
||||||
|
//
|
||||||
|
// With no trusted proxies every rate limiter keys on the connecting
|
||||||
|
// peer's address. Whether that is harmless or dangerous depends on
|
||||||
|
// what is in front of the process, which this code cannot observe:
|
||||||
|
// with nothing in front, the peer is the client and the limits are
|
||||||
|
// per-client as intended; behind a reverse proxy the peer is the proxy
|
||||||
|
// for every request, so all clients share one bucket per limiter.
|
||||||
|
//
|
||||||
|
// The login endpoint no longer spends budget on arrival — it verifies
|
||||||
|
// credentials first and charges only failures — so a shared bucket
|
||||||
|
// cannot deny the operator a correct password. What it does collapse
|
||||||
|
// is the failure counting: one client's wrong passwords throttle
|
||||||
|
// everyone else's wrong passwords, and the receiver's limits become
|
||||||
|
// service-wide ceilings.
|
||||||
|
//
|
||||||
|
// The warning is deliberately not gated on WEBHOOKER_ENVIRONMENT:
|
||||||
|
// behind a proxy every client shares one bucket in dev and prod alike.
|
||||||
|
//
|
||||||
|
// The default of trusting nobody is deliberate — trusting forwarded
|
||||||
|
// headers from arbitrary peers lets any client choose its own bucket —
|
||||||
|
// so this warns rather than failing startup or changing the key.
|
||||||
|
func (c *Config) warnSharedRateLimitBucket(log *slog.Logger) {
|
||||||
|
if len(c.TrustedProxies) > 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Warn(
|
||||||
|
"TRUSTED_PROXIES is empty: every rate limit keys on the "+
|
||||||
|
"connecting peer's address. With nothing proxying to "+
|
||||||
|
"this process that is the client itself and the limits "+
|
||||||
|
"are per-client as intended. Behind a reverse proxy the "+
|
||||||
|
"peer is the proxy on every request, so all clients "+
|
||||||
|
"share one bucket per limit: the receiver limits become "+
|
||||||
|
"service-wide ceilings, and one client's failed logins "+
|
||||||
|
"throttle every other client's failed logins — a "+
|
||||||
|
"correct password still gets in. If anything proxies to "+
|
||||||
|
"this process, set TRUSTED_PROXIES to its address.",
|
||||||
|
"environment", c.Environment,
|
||||||
|
"trustedProxies", len(c.TrustedProxies),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
// New creates a Config by reading environment variables.
|
// New creates a Config by reading environment variables.
|
||||||
//
|
//
|
||||||
//nolint:revive // lc parameter is required by fx even if unused.
|
//nolint:revive // lc parameter is required by fx even if unused.
|
||||||
@@ -801,6 +835,7 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) {
|
|||||||
// host can reach the admin UI.
|
// host can reach the admin UI.
|
||||||
"bindAddress", s.BindAddress,
|
"bindAddress", s.BindAddress,
|
||||||
"debug", s.Debug,
|
"debug", s.Debug,
|
||||||
|
"maintenanceMode", s.MaintenanceMode,
|
||||||
"dataDir", s.DataDir,
|
"dataDir", s.DataDir,
|
||||||
"retentionSweepInterval", s.RetentionSweepInterval.String(),
|
"retentionSweepInterval", s.RetentionSweepInterval.String(),
|
||||||
// Logged because a perfectly valid non-positive value here
|
// Logged because a perfectly valid non-positive value here
|
||||||
@@ -814,6 +849,7 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) {
|
|||||||
"hasMetricsAuth", s.MetricsAuthEnabled(),
|
"hasMetricsAuth", s.MetricsAuthEnabled(),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
s.warnSharedRateLimitBucket(log)
|
||||||
s.warnEgressAllowlist(log)
|
s.warnEgressAllowlist(log)
|
||||||
|
|
||||||
return s, nil
|
return s, nil
|
||||||
|
|||||||
+146
-42
@@ -3,6 +3,7 @@ package config_test
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -70,12 +71,14 @@ func TestEnvironmentConfig(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
config.ClearEnvForTest(t)
|
|
||||||
|
|
||||||
if tt.envValue != "" {
|
if tt.envValue != "" {
|
||||||
t.Setenv(
|
t.Setenv(
|
||||||
"WEBHOOKER_ENVIRONMENT", tt.envValue,
|
"WEBHOOKER_ENVIRONMENT", tt.envValue,
|
||||||
)
|
)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv(
|
||||||
|
"WEBHOOKER_ENVIRONMENT",
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
for k, v := range tt.envVars {
|
for k, v := range tt.envVars {
|
||||||
@@ -121,11 +124,6 @@ func testEnvironmentConfigSuccess(
|
|||||||
|
|
||||||
app := fxtest.New(
|
app := fxtest.New(
|
||||||
t,
|
t,
|
||||||
// fx's own log is discarded, not sent to t.Logf: a hook still
|
|
||||||
// running after a start or stop timeout would write there after
|
|
||||||
// the test has returned. The same holds for every fxtest.New
|
|
||||||
// below.
|
|
||||||
fx.NopLogger,
|
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
@@ -196,11 +194,14 @@ func TestRetentionSweepInterval(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
config.ClearEnvForTest(t)
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
t.Setenv("RETENTION_SWEEP_INTERVAL", tt.value)
|
t.Setenv("RETENTION_SWEEP_INTERVAL", tt.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv(
|
||||||
|
"RETENTION_SWEEP_INTERVAL",
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
if tt.expectError {
|
if tt.expectError {
|
||||||
@@ -271,7 +272,6 @@ func testRetentionSweepIntervalSuccess(
|
|||||||
|
|
||||||
app := fxtest.New(
|
app := fxtest.New(
|
||||||
t,
|
t,
|
||||||
fx.NopLogger,
|
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
@@ -335,11 +335,14 @@ func TestSessionIdleTimeout(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
config.ClearEnvForTest(t)
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
t.Setenv("SESSION_IDLE_TIMEOUT", tt.value)
|
t.Setenv("SESSION_IDLE_TIMEOUT", tt.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv(
|
||||||
|
"SESSION_IDLE_TIMEOUT",
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
if tt.expectError {
|
if tt.expectError {
|
||||||
@@ -361,7 +364,6 @@ func testSessionIdleTimeoutSuccess(
|
|||||||
|
|
||||||
app := fxtest.New(
|
app := fxtest.New(
|
||||||
t,
|
t,
|
||||||
fx.NopLogger,
|
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
@@ -388,17 +390,20 @@ func TestDefaultDataDir(t *testing.T) {
|
|||||||
t.Run("env="+name, func(t *testing.T) {
|
t.Run("env="+name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
config.ClearEnvForTest(t)
|
|
||||||
|
|
||||||
if env != "" {
|
if env != "" {
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", env)
|
t.Setenv("WEBHOOKER_ENVIRONMENT", env)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv(
|
||||||
|
"WEBHOOKER_ENVIRONMENT",
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
require.NoError(t, os.Unsetenv("DATA_DIR"))
|
||||||
|
|
||||||
var cfg *config.Config
|
var cfg *config.Config
|
||||||
|
|
||||||
app := fxtest.New(
|
app := fxtest.New(
|
||||||
t,
|
t,
|
||||||
fx.NopLogger,
|
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
@@ -433,9 +438,9 @@ func TestDataDirHelper(t *testing.T) {
|
|||||||
t.Run(name, func(t *testing.T) {
|
t.Run(name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
config.ClearEnvForTest(t)
|
if set == "" {
|
||||||
|
require.NoError(t, os.Unsetenv("DATA_DIR"))
|
||||||
if set != "" {
|
} else {
|
||||||
t.Setenv("DATA_DIR", set)
|
t.Setenv("DATA_DIR", set)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -498,11 +503,14 @@ func TestReceiverRateLimit(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
config.ClearEnvForTest(t)
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
t.Setenv("RECEIVER_RATE_LIMIT", tt.value)
|
t.Setenv("RECEIVER_RATE_LIMIT", tt.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv(
|
||||||
|
"RECEIVER_RATE_LIMIT",
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
if tt.expectError {
|
if tt.expectError {
|
||||||
@@ -526,7 +534,6 @@ func testReceiverRateLimitSuccess(
|
|||||||
|
|
||||||
app := fxtest.New(
|
app := fxtest.New(
|
||||||
t,
|
t,
|
||||||
fx.NopLogger,
|
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
@@ -544,11 +551,6 @@ func testReceiverRateLimitSuccess(
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestTrustedProxies(t *testing.T) {
|
func TestTrustedProxies(t *testing.T) {
|
||||||
// Unset, the RFC 1918 private ranges are trusted, so a reverse
|
|
||||||
// proxy on a Docker network or a private LAN is covered without
|
|
||||||
// configuration.
|
|
||||||
defaultProxies := []string{cidrPrivateV4, "172.16.0.0/12", "192.168.0.0/16"}
|
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
set bool
|
set bool
|
||||||
@@ -557,21 +559,18 @@ func TestTrustedProxies(t *testing.T) {
|
|||||||
expected []string
|
expected []string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
|
// The default must be "trust nobody": an empty list
|
||||||
|
// means forwarded headers are ignored, never that
|
||||||
|
// every peer may speak for the client.
|
||||||
name: caseUnsetUsesDefault,
|
name: caseUnsetUsesDefault,
|
||||||
set: false,
|
set: false,
|
||||||
expected: defaultProxies,
|
expected: []string{},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "blank value uses default",
|
name: "blank value trusts nothing",
|
||||||
set: true,
|
set: true,
|
||||||
value: " ",
|
value: " ",
|
||||||
expected: defaultProxies,
|
expected: []string{},
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "set value replaces the default entirely",
|
|
||||||
set: true,
|
|
||||||
value: "203.0.113.7",
|
|
||||||
expected: []string{"203.0.113.7/32"},
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: caseValidValueParsed,
|
name: caseValidValueParsed,
|
||||||
@@ -614,11 +613,12 @@ func TestTrustedProxies(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
config.ClearEnvForTest(t)
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
t.Setenv("TRUSTED_PROXIES", tt.value)
|
t.Setenv("TRUSTED_PROXIES", tt.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv("TRUSTED_PROXIES"))
|
||||||
}
|
}
|
||||||
|
|
||||||
if tt.expectError {
|
if tt.expectError {
|
||||||
@@ -642,7 +642,6 @@ func testTrustedProxiesSuccess(
|
|||||||
|
|
||||||
app := fxtest.New(
|
app := fxtest.New(
|
||||||
t,
|
t,
|
||||||
fx.NopLogger,
|
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
@@ -725,11 +724,14 @@ func TestAllowedEgressCIDRs(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
config.ClearEnvForTest(t)
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
t.Setenv("ALLOWED_EGRESS_CIDRS", tt.value)
|
t.Setenv("ALLOWED_EGRESS_CIDRS", tt.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(
|
||||||
|
t, os.Unsetenv("ALLOWED_EGRESS_CIDRS"),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if tt.expectError {
|
if tt.expectError {
|
||||||
@@ -753,7 +755,6 @@ func testAllowedEgressCIDRsSuccess(
|
|||||||
|
|
||||||
app := fxtest.New(
|
app := fxtest.New(
|
||||||
t,
|
t,
|
||||||
fx.NopLogger,
|
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
@@ -797,10 +798,13 @@ func TestEgressAllowlistWarning(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
config.ClearEnvForTest(t)
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", config.EnvironmentDev)
|
t.Setenv("WEBHOOKER_ENVIRONMENT", config.EnvironmentDev)
|
||||||
|
|
||||||
if tt.allowed != "" {
|
if tt.allowed == "" {
|
||||||
|
require.NoError(
|
||||||
|
t, os.Unsetenv("ALLOWED_EGRESS_CIDRS"),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
t.Setenv("ALLOWED_EGRESS_CIDRS", tt.allowed)
|
t.Setenv("ALLOWED_EGRESS_CIDRS", tt.allowed)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -841,6 +845,101 @@ func TestEgressAllowlistWarning(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestSharedRateLimitBucketWarning covers the startup warning that
|
||||||
|
// tells an operator a deployment behind a reverse proxy shares one
|
||||||
|
// rate-limit bucket between every client, which turns the receiver
|
||||||
|
// limits into service-wide ceilings and collapses login failure
|
||||||
|
// counting. It must fire whenever TRUSTED_PROXIES is empty, in any
|
||||||
|
// environment, because behind a proxy every client shares one bucket
|
||||||
|
// in dev and prod alike. It stays quiet once proxies are named.
|
||||||
|
func TestSharedRateLimitBucketWarning(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
environment string
|
||||||
|
trustedProxies string
|
||||||
|
expectWarning bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "prod without trusted proxies warns",
|
||||||
|
environment: config.EnvironmentProd,
|
||||||
|
expectWarning: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "prod with trusted proxies is quiet",
|
||||||
|
environment: config.EnvironmentProd,
|
||||||
|
trustedProxies: cidrPrivateV4,
|
||||||
|
expectWarning: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "dev without trusted proxies warns",
|
||||||
|
environment: config.EnvironmentDev,
|
||||||
|
expectWarning: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "dev with trusted proxies is quiet",
|
||||||
|
environment: config.EnvironmentDev,
|
||||||
|
trustedProxies: cidrPrivateV4,
|
||||||
|
expectWarning: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
|
// is incompatible with parallel subtests.
|
||||||
|
t.Setenv("WEBHOOKER_ENVIRONMENT", tt.environment)
|
||||||
|
|
||||||
|
if tt.trustedProxies == "" {
|
||||||
|
require.NoError(
|
||||||
|
t, os.Unsetenv("TRUSTED_PROXIES"),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
t.Setenv("TRUSTED_PROXIES", tt.trustedProxies)
|
||||||
|
}
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
|
||||||
|
log := slog.New(slog.NewJSONHandler(
|
||||||
|
&buf, &slog.HandlerOptions{
|
||||||
|
Level: slog.LevelDebug,
|
||||||
|
},
|
||||||
|
))
|
||||||
|
|
||||||
|
require.NoError(
|
||||||
|
t,
|
||||||
|
config.WarnSharedRateLimitBucketForTest(log),
|
||||||
|
)
|
||||||
|
|
||||||
|
if !tt.expectWarning {
|
||||||
|
assert.Empty(t, buf.String())
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
logged := buf.String()
|
||||||
|
|
||||||
|
assert.Contains(t, logged, `"level":"WARN"`)
|
||||||
|
assert.Contains(t, logged, "TRUSTED_PROXIES")
|
||||||
|
assert.Contains(t, logged, "share one bucket")
|
||||||
|
assert.Contains(
|
||||||
|
t, logged, "throttle every other client's failed logins",
|
||||||
|
)
|
||||||
|
// The warning must not claim a lockout the login
|
||||||
|
// endpoint no longer permits: credentials are verified
|
||||||
|
// before any budget is spent.
|
||||||
|
assert.Contains(
|
||||||
|
t, logged, "a correct password still gets in",
|
||||||
|
)
|
||||||
|
// The text must stay accurate for a developer with
|
||||||
|
// nothing in front of the process, where an empty
|
||||||
|
// list costs nothing.
|
||||||
|
assert.Contains(
|
||||||
|
t, logged, "nothing proxying to this process",
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// metricsEnv describes what one subtest below puts in the
|
// metricsEnv describes what one subtest below puts in the
|
||||||
// environment for a single METRICS_ variable. A variable that is
|
// environment for a single METRICS_ variable. A variable that is
|
||||||
// set to the empty string and one that is not set at all are
|
// set to the empty string and one that is not set at all are
|
||||||
@@ -933,14 +1032,20 @@ func TestMetricsAuthConfig(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
config.ClearEnvForTest(t)
|
|
||||||
|
|
||||||
if tt.username.set {
|
if tt.username.set {
|
||||||
t.Setenv("METRICS_USERNAME", tt.username.value)
|
t.Setenv("METRICS_USERNAME", tt.username.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(
|
||||||
|
t, os.Unsetenv("METRICS_USERNAME"),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if tt.password.set {
|
if tt.password.set {
|
||||||
t.Setenv("METRICS_PASSWORD", tt.password.value)
|
t.Setenv("METRICS_PASSWORD", tt.password.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(
|
||||||
|
t, os.Unsetenv("METRICS_PASSWORD"),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if tt.expectError {
|
if tt.expectError {
|
||||||
@@ -988,7 +1093,6 @@ func assertMetricsAuthAccepted(t *testing.T, expectAuth bool) {
|
|||||||
|
|
||||||
app := fxtest.New(
|
app := fxtest.New(
|
||||||
t,
|
t,
|
||||||
fx.NopLogger,
|
|
||||||
fx.Provide(globals.New, logger.New, config.New),
|
fx.Provide(globals.New, logger.New, config.New),
|
||||||
fx.Populate(&cfg),
|
fx.Populate(&cfg),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -22,6 +22,17 @@ const malformedDotEnv = "PORT 19615\n" +
|
|||||||
"this is not = valid ! syntax\n" +
|
"this is not = valid ! syntax\n" +
|
||||||
"\"unclosed\n"
|
"\"unclosed\n"
|
||||||
|
|
||||||
|
// unsetDotEnvKey makes dotEnvKey genuinely absent for the duration of
|
||||||
|
// the test and restores it afterwards. t.Setenv registers the restore;
|
||||||
|
// the Unsetenv that follows is what the test actually needs, because a
|
||||||
|
// variable set to the empty string is still present in os.Environ and
|
||||||
|
// godotenv would refuse to overwrite it.
|
||||||
|
func unsetDotEnvKey(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
t.Setenv(dotEnvKey, "placeholder")
|
||||||
|
require.NoError(t, os.Unsetenv(dotEnvKey))
|
||||||
|
}
|
||||||
|
|
||||||
// writeDotEnv writes contents to a .env file in a fresh temporary
|
// writeDotEnv writes contents to a .env file in a fresh temporary
|
||||||
// directory and returns its path.
|
// directory and returns its path.
|
||||||
func writeDotEnv(t *testing.T, contents string) string {
|
func writeDotEnv(t *testing.T, contents string) string {
|
||||||
@@ -39,9 +50,9 @@ func writeDotEnv(t *testing.T, contents string) string {
|
|||||||
// normally rather than be refused for a file it was never meant to
|
// normally rather than be refused for a file it was never meant to
|
||||||
// have.
|
// have.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // ClearEnvForTest uses t.Setenv.
|
//nolint:paralleltest // unsetDotEnvKey uses t.Setenv.
|
||||||
func TestLoadDotEnv_MissingFileIsFine(t *testing.T) {
|
func TestLoadDotEnv_MissingFileIsFine(t *testing.T) {
|
||||||
config.ClearEnvForTest(t)
|
unsetDotEnvKey(t)
|
||||||
|
|
||||||
absent := filepath.Join(t.TempDir(), config.DotEnvPath)
|
absent := filepath.Join(t.TempDir(), config.DotEnvPath)
|
||||||
require.NoError(t, config.LoadDotEnvFileForTest(absent))
|
require.NoError(t, config.LoadDotEnvFileForTest(absent))
|
||||||
@@ -54,9 +65,9 @@ func TestLoadDotEnv_MissingFileIsFine(t *testing.T) {
|
|||||||
// reaches the environment, which is the whole reason the file is read
|
// reaches the environment, which is the whole reason the file is read
|
||||||
// at all.
|
// at all.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // ClearEnvForTest uses t.Setenv.
|
//nolint:paralleltest // unsetDotEnvKey uses t.Setenv.
|
||||||
func TestLoadDotEnv_AppliesValues(t *testing.T) {
|
func TestLoadDotEnv_AppliesValues(t *testing.T) {
|
||||||
config.ClearEnvForTest(t)
|
unsetDotEnvKey(t)
|
||||||
|
|
||||||
path := writeDotEnv(t, "# a comment\n"+dotEnvKey+"=from-dot-env\n")
|
path := writeDotEnv(t, "# a comment\n"+dotEnvKey+"=from-dot-env\n")
|
||||||
|
|
||||||
@@ -82,9 +93,9 @@ func TestLoadDotEnv_RealEnvironmentWins(t *testing.T) {
|
|||||||
// reverts to its default; the process used to start that way with no
|
// reverts to its default; the process used to start that way with no
|
||||||
// log line naming the file at all.
|
// log line naming the file at all.
|
||||||
//
|
//
|
||||||
//nolint:paralleltest // ClearEnvForTest uses t.Setenv.
|
//nolint:paralleltest // unsetDotEnvKey uses t.Setenv.
|
||||||
func TestLoadDotEnv_MalformedFileAborts(t *testing.T) {
|
func TestLoadDotEnv_MalformedFileAborts(t *testing.T) {
|
||||||
config.ClearEnvForTest(t)
|
unsetDotEnvKey(t)
|
||||||
|
|
||||||
path := writeDotEnv(
|
path := writeDotEnv(
|
||||||
t, malformedDotEnv+dotEnvKey+"=from-dot-env\n",
|
t, malformedDotEnv+dotEnvKey+"=from-dot-env\n",
|
||||||
@@ -132,7 +143,7 @@ func TestLoadDotEnv_UnreadableFileAborts(t *testing.T) {
|
|||||||
//
|
//
|
||||||
//nolint:paralleltest // t.Chdir moves the whole process.
|
//nolint:paralleltest // t.Chdir moves the whole process.
|
||||||
func TestLoadDotEnv_ReadsTheWorkingDirectory(t *testing.T) {
|
func TestLoadDotEnv_ReadsTheWorkingDirectory(t *testing.T) {
|
||||||
config.ClearEnvForTest(t)
|
unsetDotEnvKey(t)
|
||||||
|
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
require.NoError(t, os.WriteFile(
|
require.NoError(t, os.WriteFile(
|
||||||
|
|||||||
+103
-81
@@ -1,6 +1,7 @@
|
|||||||
package config_test
|
package config_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
@@ -17,9 +18,10 @@ const testEnvKey = "WEBHOOKER_TEST_VALUE"
|
|||||||
|
|
||||||
// Real configuration variables exercised by the config.New tests.
|
// Real configuration variables exercised by the config.New tests.
|
||||||
const (
|
const (
|
||||||
envKeyPort = "PORT"
|
envKeyPort = "PORT"
|
||||||
envKeyDebug = "DEBUG"
|
envKeyDebug = "DEBUG"
|
||||||
envKeyBindAddress = "BIND_ADDRESS"
|
envKeyMaintenanceMode = "MAINTENANCE_MODE"
|
||||||
|
envKeyBindAddress = "BIND_ADDRESS"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Sample BIND_ADDRESS values used by the tables below.
|
// Sample BIND_ADDRESS values used by the tables below.
|
||||||
@@ -120,10 +122,10 @@ func TestEnvBool(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
config.ClearEnvForTest(t)
|
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
t.Setenv(testEnvKey, tt.value)
|
t.Setenv(testEnvKey, tt.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv(testEnvKey))
|
||||||
}
|
}
|
||||||
|
|
||||||
got, err := config.EnvBoolForTest(
|
got, err := config.EnvBoolForTest(
|
||||||
@@ -144,62 +146,17 @@ func TestEnvBool(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// envIntCase is one row of the envPositiveInt and envPort tables.
|
|
||||||
type envIntCase struct {
|
|
||||||
name string
|
|
||||||
set bool
|
|
||||||
value string
|
|
||||||
expectError bool
|
|
||||||
errIs error
|
|
||||||
expected int
|
|
||||||
}
|
|
||||||
|
|
||||||
// runEnvIntCases runs each row through parse, which is
|
|
||||||
// envPositiveInt or envPort, with testEnvKey set to the row's value
|
|
||||||
// or left unset.
|
|
||||||
func runEnvIntCases(
|
|
||||||
t *testing.T,
|
|
||||||
parse func(key string, defaultValue int) (int, error),
|
|
||||||
defaultValue int,
|
|
||||||
tests []envIntCase,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
|
||||||
// is incompatible with parallel subtests.
|
|
||||||
config.ClearEnvForTest(t)
|
|
||||||
|
|
||||||
if tt.set {
|
|
||||||
t.Setenv(testEnvKey, tt.value)
|
|
||||||
}
|
|
||||||
|
|
||||||
got, err := parse(testEnvKey, defaultValue)
|
|
||||||
|
|
||||||
if tt.expectError {
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Contains(t, err.Error(), testEnvKey)
|
|
||||||
assert.Contains(t, err.Error(), tt.value)
|
|
||||||
|
|
||||||
if tt.errIs != nil {
|
|
||||||
require.ErrorIs(t, err, tt.errIs)
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, tt.expected, got)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
//nolint:paralleltest // runEnvIntCases uses t.Setenv.
|
|
||||||
func TestEnvPositiveInt(t *testing.T) {
|
func TestEnvPositiveInt(t *testing.T) {
|
||||||
const defaultValue = 7
|
const defaultValue = 7
|
||||||
|
|
||||||
runEnvIntCases(t, config.EnvPositiveIntForTest, defaultValue, []envIntCase{
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
set bool
|
||||||
|
value string
|
||||||
|
expectError bool
|
||||||
|
errIs error
|
||||||
|
expected int
|
||||||
|
}{
|
||||||
{
|
{
|
||||||
name: "unset returns the default integer",
|
name: "unset returns the default integer",
|
||||||
expected: defaultValue,
|
expected: defaultValue,
|
||||||
@@ -236,14 +193,51 @@ func TestEnvPositiveInt(t *testing.T) {
|
|||||||
expectError: true,
|
expectError: true,
|
||||||
errIs: config.ErrNonPositiveValue,
|
errIs: config.ErrNonPositiveValue,
|
||||||
},
|
},
|
||||||
})
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
|
// is incompatible with parallel subtests.
|
||||||
|
if tt.set {
|
||||||
|
t.Setenv(testEnvKey, tt.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv(testEnvKey))
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := config.EnvPositiveIntForTest(
|
||||||
|
testEnvKey, defaultValue,
|
||||||
|
)
|
||||||
|
|
||||||
|
if tt.expectError {
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), testEnvKey)
|
||||||
|
assert.Contains(t, err.Error(), tt.value)
|
||||||
|
|
||||||
|
if tt.errIs != nil {
|
||||||
|
require.ErrorIs(t, err, tt.errIs)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, tt.expected, got)
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//nolint:paralleltest // runEnvIntCases uses t.Setenv.
|
|
||||||
func TestEnvPort(t *testing.T) {
|
func TestEnvPort(t *testing.T) {
|
||||||
const defaultValue = 8080
|
const defaultValue = 8080
|
||||||
|
|
||||||
runEnvIntCases(t, config.EnvPortForTest, defaultValue, []envIntCase{
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
set bool
|
||||||
|
value string
|
||||||
|
expectError bool
|
||||||
|
errIs error
|
||||||
|
expected int
|
||||||
|
}{
|
||||||
{
|
{
|
||||||
name: "unset returns the default port",
|
name: "unset returns the default port",
|
||||||
expected: defaultValue,
|
expected: defaultValue,
|
||||||
@@ -271,14 +265,7 @@ func TestEnvPort(t *testing.T) {
|
|||||||
set: true,
|
set: true,
|
||||||
value: "0",
|
value: "0",
|
||||||
expectError: true,
|
expectError: true,
|
||||||
errIs: config.ErrInvalidPort,
|
errIs: config.ErrNonPositiveValue,
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "negative is rejected",
|
|
||||||
set: true,
|
|
||||||
value: "-1",
|
|
||||||
expectError: true,
|
|
||||||
errIs: config.ErrInvalidPort,
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "above the port range is rejected",
|
name: "above the port range is rejected",
|
||||||
@@ -287,14 +274,37 @@ func TestEnvPort(t *testing.T) {
|
|||||||
expectError: true,
|
expectError: true,
|
||||||
errIs: config.ErrInvalidPort,
|
errIs: config.ErrInvalidPort,
|
||||||
},
|
},
|
||||||
{
|
}
|
||||||
name: "too large for an int is rejected",
|
|
||||||
set: true,
|
for _, tt := range tests {
|
||||||
value: "99999999999999999999",
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
expectError: true,
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
errIs: config.ErrInvalidPort,
|
// is incompatible with parallel subtests.
|
||||||
},
|
if tt.set {
|
||||||
})
|
t.Setenv(testEnvKey, tt.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv(testEnvKey))
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := config.EnvPortForTest(
|
||||||
|
testEnvKey, defaultValue,
|
||||||
|
)
|
||||||
|
|
||||||
|
if tt.expectError {
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), testEnvKey)
|
||||||
|
|
||||||
|
if tt.errIs != nil {
|
||||||
|
require.ErrorIs(t, err, tt.errIs)
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, tt.expected, got)
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestEnvBindAddress covers BIND_ADDRESS parsing.
|
// TestEnvBindAddress covers BIND_ADDRESS parsing.
|
||||||
@@ -310,10 +320,10 @@ func TestEnvBindAddress(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
config.ClearEnvForTest(t)
|
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
t.Setenv(testEnvKey, tt.value)
|
t.Setenv(testEnvKey, tt.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv(testEnvKey))
|
||||||
}
|
}
|
||||||
|
|
||||||
got, err := config.EnvBindAddressForTest(
|
got, err := config.EnvBindAddressForTest(
|
||||||
@@ -476,7 +486,6 @@ func TestNewRejectsBadEnvValues(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
config.ClearEnvForTest(t)
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
t.Setenv(tt.key, tt.value)
|
t.Setenv(tt.key, tt.value)
|
||||||
|
|
||||||
@@ -595,6 +604,12 @@ func flagEnvValueCases() []badEnvValueCase {
|
|||||||
value: "ture",
|
value: "ture",
|
||||||
expectError: true,
|
expectError: true,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "unparseable MAINTENANCE_MODE aborts startup",
|
||||||
|
key: envKeyMaintenanceMode,
|
||||||
|
value: "sometimes",
|
||||||
|
expectError: true,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -638,15 +653,22 @@ func sentryEnvValueCases() []badEnvValueCase {
|
|||||||
// break the legitimate unset case: absent variables still get their
|
// break the legitimate unset case: absent variables still get their
|
||||||
// documented defaults.
|
// documented defaults.
|
||||||
func TestNewUsesDefaultsWhenUnset(t *testing.T) {
|
func TestNewUsesDefaultsWhenUnset(t *testing.T) {
|
||||||
config.ClearEnvForTest(t)
|
|
||||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||||
|
|
||||||
|
for _, key := range []string{
|
||||||
|
envKeyPort, envKeyDebug, envKeyMaintenanceMode,
|
||||||
|
envKeyBindAddress, envKeySentryDSN,
|
||||||
|
} {
|
||||||
|
require.NoError(t, os.Unsetenv(key))
|
||||||
|
}
|
||||||
|
|
||||||
cfg, err := buildConfig(t)
|
cfg, err := buildConfig(t)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NotNil(t, cfg)
|
require.NotNil(t, cfg)
|
||||||
|
|
||||||
assert.Equal(t, 8080, cfg.Port)
|
assert.Equal(t, 8080, cfg.Port)
|
||||||
assert.False(t, cfg.Debug)
|
assert.False(t, cfg.Debug)
|
||||||
|
assert.False(t, cfg.MaintenanceMode)
|
||||||
|
|
||||||
// Loopback, not the wildcard: the default must not publish the
|
// Loopback, not the wildcard: the default must not publish the
|
||||||
// cleartext admin UI and the unauthenticated receiver on every
|
// cleartext admin UI and the unauthenticated receiver on every
|
||||||
|
|||||||
@@ -6,6 +6,21 @@ import "log/slog"
|
|||||||
// the external config_test package so each helper can be covered by
|
// the external config_test package so each helper can be covered by
|
||||||
// its own table-driven test without weakening the package API.
|
// its own table-driven test without weakening the package API.
|
||||||
|
|
||||||
|
// WarnSharedRateLimitBucketForTest loads a Config from the current
|
||||||
|
// environment and emits its startup warnings to log. The real logger
|
||||||
|
// writes to stdout, so this lets the warning's firing condition be
|
||||||
|
// asserted against a handler the test controls.
|
||||||
|
func WarnSharedRateLimitBucketForTest(log *slog.Logger) error {
|
||||||
|
c, err := loadFromEnv()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
c.warnSharedRateLimitBucket(log)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// WarnEgressAllowlistForTest loads a Config from the current
|
// WarnEgressAllowlistForTest loads a Config from the current
|
||||||
// environment and emits its egress-allowlist startup warning to
|
// environment and emits its egress-allowlist startup warning to
|
||||||
// log, so a test can assert both that the warning fires only when
|
// log, so a test can assert both that the warning fires only when
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package config_test
|
package config_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
@@ -100,10 +101,10 @@ func TestEnvSentryDSN(t *testing.T) {
|
|||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
// Cannot use t.Parallel() here because t.Setenv
|
// Cannot use t.Parallel() here because t.Setenv
|
||||||
// is incompatible with parallel subtests.
|
// is incompatible with parallel subtests.
|
||||||
config.ClearEnvForTest(t)
|
|
||||||
|
|
||||||
if tt.set {
|
if tt.set {
|
||||||
t.Setenv(envKeySentryDSN, tt.value)
|
t.Setenv(envKeySentryDSN, tt.value)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, os.Unsetenv(envKeySentryDSN))
|
||||||
}
|
}
|
||||||
|
|
||||||
got, err := config.EnvSentryDSNForTest(envKeySentryDSN)
|
got, err := config.EnvSentryDSNForTest(envKeySentryDSN)
|
||||||
|
|||||||
@@ -1,50 +0,0 @@
|
|||||||
package config
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ClearEnvForTest unsets every variable in the process environment
|
|
||||||
// for the rest of the test, so a test sees only the variables it sets
|
|
||||||
// itself, not whatever the developer's shell exports. When the test
|
|
||||||
// ends it leaves the environment exactly as it found it: each variable
|
|
||||||
// it unset is put back, and any variable added since is removed.
|
|
||||||
func ClearEnvForTest(t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
present := make(map[string]bool)
|
|
||||||
|
|
||||||
for _, entry := range os.Environ() {
|
|
||||||
key, _, _ := strings.Cut(entry, "=")
|
|
||||||
present[key] = true
|
|
||||||
|
|
||||||
// t.Setenv registers the restore; the Unsetenv after it is
|
|
||||||
// what makes the key absent, since a key set to the empty
|
|
||||||
// string is still present, and godotenv will not overwrite a
|
|
||||||
// present key.
|
|
||||||
t.Setenv(key, "")
|
|
||||||
|
|
||||||
err := os.Unsetenv(key)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("unsetting %s: %v", key, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A variable the test adds other than through t.Setenv, as loading
|
|
||||||
// a .env file does, has no restore of its own.
|
|
||||||
t.Cleanup(func() {
|
|
||||||
for _, entry := range os.Environ() {
|
|
||||||
key, _, _ := strings.Cut(entry, "=")
|
|
||||||
if present[key] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
err := os.Unsetenv(key)
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("unsetting %s: %v", key, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -1,36 +0,0 @@
|
|||||||
package config_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestClearEnvForTest_RemovesAddedVariables pins that a variable set
|
|
||||||
// after the clear other than through t.Setenv, as a test's .env file
|
|
||||||
// sets one, is gone once the test ends, so it cannot reach the tests
|
|
||||||
// that run after it.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // ClearEnvForTest uses t.Setenv.
|
|
||||||
func TestClearEnvForTest_RemovesAddedVariables(t *testing.T) {
|
|
||||||
// The outer clear keeps a value of the key exported in the shell
|
|
||||||
// from making it a variable the inner clear has to put back.
|
|
||||||
config.ClearEnvForTest(t)
|
|
||||||
|
|
||||||
t.Run("loads a .env file after the clear", func(t *testing.T) {
|
|
||||||
config.ClearEnvForTest(t)
|
|
||||||
|
|
||||||
path := writeDotEnv(t, dotEnvKey+"=from-dot-env\n")
|
|
||||||
require.NoError(t, config.LoadDotEnvFileForTest(path))
|
|
||||||
require.Equal(t, "from-dot-env", os.Getenv(dotEnvKey))
|
|
||||||
})
|
|
||||||
|
|
||||||
_, present := os.LookupEnv(dotEnvKey)
|
|
||||||
assert.False(
|
|
||||||
t, present,
|
|
||||||
"a variable set after the clear must not outlive the test",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -152,7 +152,9 @@ func TestEventTierQueriesUseTheirIndexes(t *testing.T) {
|
|||||||
// TestStatisticsQueriesUseTheirIndexes does the same for the webhook
|
// TestStatisticsQueriesUseTheirIndexes does the same for the webhook
|
||||||
// page's statistics (readEventStats in the handlers): deliveries in
|
// page's statistics (readEventStats in the handlers): deliveries in
|
||||||
// progress, each target's deliveries finished since a time, which must
|
// progress, each target's deliveries finished since a time, which must
|
||||||
// come from the index alone, and events received since a time.
|
// come from the index alone, events received since a time, and the
|
||||||
|
// newest event, which must come straight off an index rather than from
|
||||||
|
// sorting every event.
|
||||||
func TestStatisticsQueriesUseTheirIndexes(t *testing.T) {
|
func TestStatisticsQueriesUseTheirIndexes(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -170,6 +172,7 @@ func TestStatisticsQueriesUseTheirIndexes(t *testing.T) {
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
count int64
|
count int64
|
||||||
|
newest []time.Time
|
||||||
byTarget []struct{ TargetID string }
|
byTarget []struct{ TargetID string }
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -197,6 +200,12 @@ func TestStatisticsQueriesUseTheirIndexes(t *testing.T) {
|
|||||||
Where("created_at >= ?", since).Count(&count),
|
Where("created_at >= ?", since).Count(&count),
|
||||||
"idx_events_deleted_at_created_at "+
|
"idx_events_deleted_at_created_at "+
|
||||||
"(deleted_at=? AND created_at>?)")
|
"(deleted_at=? AND created_at>?)")
|
||||||
|
|
||||||
|
newestEvent := dry.Model(&database.Event{}).
|
||||||
|
Order("created_at DESC").Limit(1).Pluck("created_at", &newest)
|
||||||
|
assertPlanUses(t, db, newestEvent,
|
||||||
|
"idx_events_deleted_at_created_at (deleted_at=?)")
|
||||||
|
assert.NotContains(t, queryPlan(t, db, newestEvent), "TEMP B-TREE")
|
||||||
}
|
}
|
||||||
|
|
||||||
// assertPlanUses asserts that SQLite's plan for a statement GORM built
|
// assertPlanUses asserts that SQLite's plan for a statement GORM built
|
||||||
@@ -207,6 +216,18 @@ func assertPlanUses(
|
|||||||
) {
|
) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
plan := queryPlan(t, db, built)
|
||||||
|
|
||||||
|
for _, index := range indexes {
|
||||||
|
assert.Contains(t, plan, index, built.Statement.SQL.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// queryPlan returns SQLite's plan for a statement GORM built in a dry
|
||||||
|
// run, run with the same SQL and arguments GORM would send.
|
||||||
|
func queryPlan(t *testing.T, db, built *gorm.DB) string {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
var plan []struct{ Detail string }
|
var plan []struct{ Detail string }
|
||||||
|
|
||||||
require.NoError(t, db.Raw(
|
require.NoError(t, db.Raw(
|
||||||
@@ -214,8 +235,5 @@ func assertPlanUses(
|
|||||||
built.Statement.Vars...,
|
built.Statement.Vars...,
|
||||||
).Scan(&plan).Error)
|
).Scan(&plan).Error)
|
||||||
|
|
||||||
for _, index := range indexes {
|
return fmt.Sprint(plan)
|
||||||
assert.Contains(t, fmt.Sprint(plan), index,
|
|
||||||
built.Statement.SQL.String())
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"testing"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
@@ -84,14 +83,3 @@ func (d *Database) ExportSetBannerOut(w io.Writer) {
|
|||||||
func DummyPasswordHashForTest() string {
|
func DummyPasswordHashForTest() string {
|
||||||
return dummyPasswordHash()
|
return dummyPasswordHash()
|
||||||
}
|
}
|
||||||
|
|
||||||
// HashAtShippedCostForTest makes HashPassword hash at the shipped
|
|
||||||
// memory cost until t ends. t must not run in parallel with other
|
|
||||||
// tests, which would hash at that cost alongside it.
|
|
||||||
func HashAtShippedCostForTest(t *testing.T) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
hashAtShippedCostInTest = true
|
|
||||||
|
|
||||||
t.Cleanup(func() { hashAtShippedCostInTest = false })
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ type APIKey struct {
|
|||||||
Description string `json:"description"`
|
Description string `json:"description"`
|
||||||
LastUsedAt *time.Time `json:"lastUsedAt,omitempty"`
|
LastUsedAt *time.Time `json:"lastUsedAt,omitempty"`
|
||||||
|
|
||||||
// Relations. No model marshals the record it belongs to:
|
// Relations
|
||||||
// User.APIKeys leads back here, and the JSON could loop.
|
User User `json:"user,omitzero"`
|
||||||
User User `json:"-"`
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -56,10 +56,8 @@ type Delivery struct {
|
|||||||
// the index.
|
// the index.
|
||||||
FinishedAt *time.Time `gorm:"index:idx_deliveries_status,priority:3" json:"finishedAt,omitempty"`
|
FinishedAt *time.Time `gorm:"index:idx_deliveries_status,priority:3" json:"finishedAt,omitempty"`
|
||||||
|
|
||||||
// Relations. No model marshals the record it belongs to:
|
// Relations
|
||||||
// Event.Deliveries and Target.Deliveries lead back here, and the
|
Event Event `json:"event,omitzero"`
|
||||||
// JSON could loop.
|
Target Target `json:"target,omitzero"`
|
||||||
Event Event `json:"-"`
|
|
||||||
Target Target `json:"-"`
|
|
||||||
DeliveryResults []DeliveryResult `json:"deliveryResults,omitempty"`
|
DeliveryResults []DeliveryResult `json:"deliveryResults,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,7 +23,6 @@ type DeliveryResult struct {
|
|||||||
Error string `json:"error,omitempty"`
|
Error string `json:"error,omitempty"`
|
||||||
Duration int64 `json:"durationMs"` // Duration in milliseconds
|
Duration int64 `json:"durationMs"` // Duration in milliseconds
|
||||||
|
|
||||||
// Relations. No model marshals the record it belongs to:
|
// Relations
|
||||||
// Delivery.DeliveryResults leads back here, and the JSON could loop.
|
Delivery Delivery `json:"delivery,omitzero"`
|
||||||
Delivery Delivery `json:"-"`
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ type Entrypoint struct {
|
|||||||
Description string `json:"description"`
|
Description string `json:"description"`
|
||||||
Active bool `gorm:"default:true" json:"active"`
|
Active bool `gorm:"default:true" json:"active"`
|
||||||
|
|
||||||
// Relations. No model marshals the record it belongs to:
|
// Relations
|
||||||
// Webhook.Entrypoints leads back here, and the JSON could loop.
|
Webhook Webhook `json:"webhook,omitzero"`
|
||||||
Webhook Webhook `json:"-"`
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -31,11 +31,6 @@ type Event struct {
|
|||||||
Body string `gorm:"type:text" json:"body"`
|
Body string `gorm:"type:text" json:"body"`
|
||||||
ContentType string `json:"contentType"`
|
ContentType string `json:"contentType"`
|
||||||
|
|
||||||
// BodyBytes is the size of Body in bytes, recorded when the event
|
|
||||||
// is stored so the recent events list can show it without reading
|
|
||||||
// the body.
|
|
||||||
BodyBytes int64 `gorm:"not null" json:"bodyBytes"`
|
|
||||||
|
|
||||||
// ResubmittedFromID names the event this one was copied from by
|
// ResubmittedFromID names the event this one was copied from by
|
||||||
// an operator resubmit. It is nil for an event that arrived on
|
// an operator resubmit. It is nil for an event that arrived on
|
||||||
// the receiver, which is every event created before the column
|
// the receiver, which is every event created before the column
|
||||||
@@ -44,9 +39,8 @@ type Event struct {
|
|||||||
// kept as the record of where the copy came from either way.
|
// kept as the record of where the copy came from either way.
|
||||||
ResubmittedFromID *string `gorm:"type:uuid;index" json:"resubmittedFromId,omitempty"`
|
ResubmittedFromID *string `gorm:"type:uuid;index" json:"resubmittedFromId,omitempty"`
|
||||||
|
|
||||||
// Relations. No model marshals the record it belongs to, so
|
// Relations
|
||||||
// Webhook and Entrypoint are left out of the JSON.
|
Webhook Webhook `json:"webhook,omitzero"`
|
||||||
Webhook Webhook `json:"-"`
|
Entrypoint Entrypoint `json:"entrypoint,omitzero"`
|
||||||
Entrypoint Entrypoint `json:"-"`
|
|
||||||
Deliveries []Delivery `json:"deliveries,omitempty"`
|
Deliveries []Delivery `json:"deliveries,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,126 +0,0 @@
|
|||||||
package database_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestPreloadedModelsMarshalWithoutTheirParent pins that a child's
|
|
||||||
// reference to the record it belongs to is left out of the JSON, so a
|
|
||||||
// webhook and its targets cannot marshal each other in a loop, and that
|
|
||||||
// GORM still preloads that reference, since it ignores json tags.
|
|
||||||
func TestPreloadedModelsMarshalWithoutTheirParent(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := startedTestDB(t)
|
|
||||||
|
|
||||||
stored := database.Webhook{
|
|
||||||
UserID: uuid.New().String(),
|
|
||||||
Name: testWebhookName,
|
|
||||||
Entrypoints: []database.Entrypoint{{Path: uuid.New().String()}},
|
|
||||||
Targets: []database.Target{{
|
|
||||||
Name: "log",
|
|
||||||
Type: database.TargetTypeLog,
|
|
||||||
}},
|
|
||||||
}
|
|
||||||
require.NoError(t, db.Create(&stored).Error)
|
|
||||||
|
|
||||||
entrypointID := stored.Entrypoints[0].ID
|
|
||||||
targetID := stored.Targets[0].ID
|
|
||||||
|
|
||||||
var webhook database.Webhook
|
|
||||||
|
|
||||||
require.NoError(t, db.
|
|
||||||
Preload("Entrypoints.Webhook").
|
|
||||||
Preload("Targets.Webhook").
|
|
||||||
First(&webhook, "id = ?", stored.ID).Error)
|
|
||||||
|
|
||||||
require.Len(t, webhook.Entrypoints, 1)
|
|
||||||
require.Len(t, webhook.Targets, 1)
|
|
||||||
assert.Equal(t, stored.ID, webhook.Entrypoints[0].Webhook.ID)
|
|
||||||
assert.Equal(t, stored.ID, webhook.Targets[0].Webhook.ID)
|
|
||||||
|
|
||||||
encoded := marshalModel(t, webhook)
|
|
||||||
|
|
||||||
assert.Contains(t, encoded, entrypointID)
|
|
||||||
assert.Contains(t, encoded, targetID)
|
|
||||||
|
|
||||||
// Each child holds the parent's id as its webhookId, so the parent
|
|
||||||
// is looked for by its own id field.
|
|
||||||
parentIDField := `"id":"` + stored.ID + `"`
|
|
||||||
|
|
||||||
assert.NotContains(t, marshalModel(t, webhook.Entrypoints[0]), parentIDField)
|
|
||||||
assert.NotContains(t, marshalModel(t, webhook.Targets[0]), parentIDField)
|
|
||||||
|
|
||||||
var target database.Target
|
|
||||||
|
|
||||||
require.NoError(t, db.
|
|
||||||
Preload("Webhook").
|
|
||||||
First(&target, "id = ?", targetID).Error)
|
|
||||||
|
|
||||||
assert.Equal(t, stored.ID, target.Webhook.ID)
|
|
||||||
|
|
||||||
encoded = marshalModel(t, target)
|
|
||||||
|
|
||||||
assert.Contains(t, encoded, stored.ID)
|
|
||||||
assert.NotContains(t, encoded, parentIDField)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestModelsMarshalWithoutTheirParent covers the other references to a
|
|
||||||
// parent: each model is built with its parent set, and the parent's id
|
|
||||||
// must not appear in the JSON.
|
|
||||||
func TestModelsMarshalWithoutTheirParent(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
parent := database.BaseModel{ID: uuid.New().String()}
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
model any
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "Webhook.User",
|
|
||||||
model: database.Webhook{User: database.User{BaseModel: parent}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "APIKey.User",
|
|
||||||
model: database.APIKey{User: database.User{BaseModel: parent}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Delivery.Event",
|
|
||||||
model: database.Delivery{Event: database.Event{BaseModel: parent}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Delivery.Target",
|
|
||||||
model: database.Delivery{Target: database.Target{BaseModel: parent}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "DeliveryResult.Delivery",
|
|
||||||
model: database.DeliveryResult{
|
|
||||||
Delivery: database.Delivery{BaseModel: parent},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Event.Webhook",
|
|
||||||
model: database.Event{Webhook: database.Webhook{BaseModel: parent}},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Event.Entrypoint",
|
|
||||||
model: database.Event{
|
|
||||||
Entrypoint: database.Entrypoint{BaseModel: parent},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
assert.NotContains(t, marshalModel(t, tc.model), parent.ID)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -34,8 +34,7 @@ type Target struct {
|
|||||||
MaxRetries int `json:"maxRetries,omitempty"`
|
MaxRetries int `json:"maxRetries,omitempty"`
|
||||||
MaxQueueSize int `json:"maxQueueSize,omitempty"`
|
MaxQueueSize int `json:"maxQueueSize,omitempty"`
|
||||||
|
|
||||||
// Relations. No model marshals the record it belongs to:
|
// Relations
|
||||||
// Webhook.Targets leads back here, and the JSON could loop.
|
Webhook Webhook `json:"webhook,omitzero"`
|
||||||
Webhook Webhook `json:"-"`
|
|
||||||
Deliveries []Delivery `json:"deliveries,omitempty"`
|
Deliveries []Delivery `json:"deliveries,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package database
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
|
||||||
|
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
)
|
)
|
||||||
@@ -14,17 +13,12 @@ import (
|
|||||||
// rows it counts.
|
// rows it counts.
|
||||||
|
|
||||||
// EventTotals is the single row counting a webhook's events: every
|
// EventTotals is the single row counting a webhook's events: every
|
||||||
// event ever stored, how many of them retention has deleted, and when
|
// event ever stored, and how many of them retention has deleted.
|
||||||
// the newest arrived, which retention leaves as it is.
|
|
||||||
type EventTotals struct {
|
type EventTotals struct {
|
||||||
ID int64 `gorm:"primaryKey"`
|
ID int64 `gorm:"primaryKey"`
|
||||||
|
|
||||||
Events int64 `gorm:"not null"`
|
Events int64 `gorm:"not null"`
|
||||||
EventsRemoved int64 `gorm:"not null"`
|
EventsRemoved int64 `gorm:"not null"`
|
||||||
|
|
||||||
// LastEventAt is when the newest event arrived, or nil before the
|
|
||||||
// first.
|
|
||||||
LastEventAt *time.Time
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TableName names the table AddEventTotals updates.
|
// TableName names the table AddEventTotals updates.
|
||||||
@@ -52,17 +46,15 @@ func (TargetTotals) TableName() string {
|
|||||||
return "target_totals"
|
return "target_totals"
|
||||||
}
|
}
|
||||||
|
|
||||||
// AddEventTotals adds each count in add to the webhook's event totals,
|
// AddEventTotals adds each count in add to the webhook's event totals.
|
||||||
// and records add.LastEventAt as when the newest event arrived if it is
|
// Call it on the transaction that writes or deletes the events it
|
||||||
// set. Call it on the transaction that writes or deletes the events it
|
|
||||||
// counts.
|
// counts.
|
||||||
func AddEventTotals(tx *gorm.DB, add EventTotals) error {
|
func AddEventTotals(tx *gorm.DB, add EventTotals) error {
|
||||||
err := tx.Exec(
|
err := tx.Exec(
|
||||||
`UPDATE event_totals SET
|
`UPDATE event_totals SET
|
||||||
events = events + ?,
|
events = events + ?,
|
||||||
events_removed = events_removed + ?,
|
events_removed = events_removed + ?`,
|
||||||
last_event_at = coalesce(?, last_event_at)`,
|
add.Events, add.EventsRemoved,
|
||||||
add.Events, add.EventsRemoved, add.LastEventAt,
|
|
||||||
).Error
|
).Error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("adding to event totals: %w", err)
|
return fmt.Errorf("adding to event totals: %w", err)
|
||||||
|
|||||||
@@ -66,9 +66,8 @@ type Webhook struct {
|
|||||||
// must equal DefaultRetentionDays.
|
// must equal DefaultRetentionDays.
|
||||||
RetentionDays int `gorm:"default:30" json:"retentionDays"`
|
RetentionDays int `gorm:"default:30" json:"retentionDays"`
|
||||||
|
|
||||||
// Relations. No model marshals the record it belongs to:
|
// Relations
|
||||||
// User.Webhooks leads back here, and the JSON could loop.
|
User User `json:"user,omitzero"`
|
||||||
User User `json:"-"`
|
|
||||||
Entrypoints []Entrypoint `json:"entrypoints,omitempty"`
|
Entrypoints []Entrypoint `json:"entrypoints,omitempty"`
|
||||||
Targets []Target `json:"targets,omitempty"`
|
Targets []Target `json:"targets,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ import (
|
|||||||
"math/big"
|
"math/big"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"testing"
|
|
||||||
|
|
||||||
"golang.org/x/crypto/argon2"
|
"golang.org/x/crypto/argon2"
|
||||||
)
|
)
|
||||||
@@ -64,30 +63,10 @@ func DefaultPasswordConfig() *PasswordConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// testArgon2Memory is the Argon2id memory cost, in KiB, that a test
|
// HashPassword generates an Argon2id hash of the password
|
||||||
// binary hashes with: 1 MB instead of the shipped 64 MB. Every test
|
|
||||||
// that starts a database hashes the bootstrap admin password, dozens
|
|
||||||
// of them run in parallel, and under the race detector each 64 MB hash
|
|
||||||
// holds about 150 MB. VerifyPassword reads the cost from the hash it
|
|
||||||
// checks, so verification follows.
|
|
||||||
const testArgon2Memory = 1024
|
|
||||||
|
|
||||||
// hashAtShippedCostInTest makes a test binary hash at the shipped
|
|
||||||
// memory cost. Only TestHashPassword_ShippedParameters sets it.
|
|
||||||
//
|
|
||||||
//nolint:gochecknoglobals // set by one test, see above
|
|
||||||
var hashAtShippedCostInTest bool
|
|
||||||
|
|
||||||
// HashPassword generates an Argon2id hash of the password. A binary
|
|
||||||
// built by go test hashes at testArgon2Memory; one built by go build
|
|
||||||
// always hashes at the defaults.
|
|
||||||
func HashPassword(password string) (string, error) {
|
func HashPassword(password string) (string, error) {
|
||||||
config := DefaultPasswordConfig()
|
config := DefaultPasswordConfig()
|
||||||
|
|
||||||
if testing.Testing() && !hashAtShippedCostInTest {
|
|
||||||
config.Memory = testArgon2Memory
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate a salt
|
// Generate a salt
|
||||||
salt := make([]byte, config.SaltLen)
|
salt := make([]byte, config.SaltLen)
|
||||||
|
|
||||||
|
|||||||
@@ -192,39 +192,6 @@ func TestHashPasswordUniqueness(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHashPassword_ShippedParameters hashes and verifies through
|
|
||||||
// HashPassword at the shipped Argon2id parameters. Every other test
|
|
||||||
// hashes at the lower memory cost a test binary uses, so this is the
|
|
||||||
// one that keeps production hashing covered. One hash and one
|
|
||||||
// verification: each costs 64 MB.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // changes the hashing cost for the whole binary
|
|
||||||
func TestHashPassword_ShippedParameters(t *testing.T) {
|
|
||||||
database.HashAtShippedCostForTest(t)
|
|
||||||
|
|
||||||
password := "correct horse battery staple"
|
|
||||||
|
|
||||||
hash, err := database.HashPassword(password)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("hashing with the shipped parameters: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
const shipped = "$argon2id$v=19$m=65536,t=1,p=4$"
|
|
||||||
|
|
||||||
if !strings.HasPrefix(hash, shipped) {
|
|
||||||
t.Errorf("hash = %q, want prefix %q", hash, shipped)
|
|
||||||
}
|
|
||||||
|
|
||||||
valid, err := database.VerifyPassword(password, hash)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("VerifyPassword() error = %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !valid {
|
|
||||||
t.Error("VerifyPassword() returned false for correct password")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestVerifyDummyPassword_DoesRealWork covers the anti-enumeration
|
// TestVerifyDummyPassword_DoesRealWork covers the anti-enumeration
|
||||||
// path. Login charges an unknown username a verification against a
|
// path. Login charges an unknown username a verification against a
|
||||||
// dummy hash so that a nonexistent account is not answered in
|
// dummy hash so that a nonexistent account is not answered in
|
||||||
|
|||||||
@@ -25,12 +25,6 @@ const hoursPerDay = 24
|
|||||||
// busy timeout.
|
// busy timeout.
|
||||||
const reapBatchSize = 1000
|
const reapBatchSize = 1000
|
||||||
|
|
||||||
// reapBatchPause is how long retention waits after one batch before
|
|
||||||
// starting the next. A writer waiting for the write lock checks for it
|
|
||||||
// again after at most 100 ms, so a longer pause lets it in between two
|
|
||||||
// batches instead of only after the whole prune.
|
|
||||||
const reapBatchPause = 200 * time.Millisecond
|
|
||||||
|
|
||||||
// RetentionReaperParams holds the fx dependencies for the
|
// RetentionReaperParams holds the fx dependencies for the
|
||||||
// RetentionReaper.
|
// RetentionReaper.
|
||||||
type RetentionReaperParams struct {
|
type RetentionReaperParams struct {
|
||||||
@@ -184,31 +178,32 @@ func (r *RetentionReaper) sweep(ctx context.Context) {
|
|||||||
|
|
||||||
wh := webhooks[i]
|
wh := webhooks[i]
|
||||||
|
|
||||||
|
// Skip retain-forever webhooks before building any query.
|
||||||
|
// RetainsForever covers both the RetentionForeverDays
|
||||||
|
// sentinel and the non-positive values that predate it: the
|
||||||
|
// sentinel is a positive number, so without this the reaper
|
||||||
|
// would compute a cutoff a thousand years in the past and
|
||||||
|
// issue a DELETE matching nothing on every single sweep.
|
||||||
|
if wh.RetainsForever() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
// Nothing to reap if the per-webhook database has never
|
// Nothing to reap if the per-webhook database has never
|
||||||
// been created.
|
// been created.
|
||||||
if !r.dbManager.DBExists(wh.ID) {
|
if !r.dbManager.DBExists(wh.ID) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
r.reapWebhook(ctx, wh.ID, wh.RetentionDays)
|
r.reapWebhook(wh.ID, wh.RetentionDays)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// reapWebhook removes every expired event (and its dependents) from a
|
// reapWebhook removes every expired event (and its dependents) from a
|
||||||
// single webhook's database, or as many as it reaches before ctx is
|
// single webhook's database.
|
||||||
// cancelled.
|
|
||||||
func (r *RetentionReaper) reapWebhook(
|
func (r *RetentionReaper) reapWebhook(
|
||||||
ctx context.Context,
|
|
||||||
webhookID string,
|
webhookID string,
|
||||||
retentionDays int,
|
retentionDays int,
|
||||||
) {
|
) {
|
||||||
// A retain-forever webhook has no cutoff, so its database is not
|
|
||||||
// even opened.
|
|
||||||
cutoff, ok := retentionCutoff(time.Now(), retentionDays)
|
|
||||||
if !ok {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
db, err := r.dbManager.GetDB(webhookID)
|
db, err := r.dbManager.GetDB(webhookID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
r.log.Error(
|
r.log.Error(
|
||||||
@@ -220,7 +215,12 @@ func (r *RetentionReaper) reapWebhook(
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
deleted, err := reapExpired(ctx, db, cutoff)
|
cutoff, ok := retentionCutoff(time.Now(), retentionDays)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
deleted, err := reapExpired(db, cutoff)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
r.log.Error(
|
r.log.Error(
|
||||||
"retention sweep: failed to reap expired events",
|
"retention sweep: failed to reap expired events",
|
||||||
@@ -274,13 +274,9 @@ func retentionCutoff(
|
|||||||
|
|
||||||
// reapExpired hard-deletes the events older than cutoff, with their
|
// reapExpired hard-deletes the events older than cutoff, with their
|
||||||
// deliveries and delivery results, reapBatchSize events per
|
// deliveries and delivery results, reapBatchSize events per
|
||||||
// transaction with reapBatchPause between transactions, until none is
|
// transaction until none is left. It returns the number of events
|
||||||
// left. Once ctx is cancelled it returns after the batch in hand,
|
// deleted.
|
||||||
// leaving the rest to the next sweep, so stopping the app does not
|
func reapExpired(db *gorm.DB, cutoff time.Time) (int64, error) {
|
||||||
// wait for a long prune. It returns the number of events deleted.
|
|
||||||
func reapExpired(
|
|
||||||
ctx context.Context, db *gorm.DB, cutoff time.Time,
|
|
||||||
) (int64, error) {
|
|
||||||
var total int64
|
var total int64
|
||||||
|
|
||||||
for {
|
for {
|
||||||
@@ -310,12 +306,6 @@ func reapExpired(
|
|||||||
if len(eventIDs) < reapBatchSize {
|
if len(eventIDs) < reapBatchSize {
|
||||||
return total, nil
|
return total, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return total, nil
|
|
||||||
case <-time.After(reapBatchPause):
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -362,7 +362,7 @@ func TestRetentionReaper_HugeFiniteRetentionRetainsRecentEvents(
|
|||||||
t,
|
t,
|
||||||
overflowingRetentionDays,
|
overflowingRetentionDays,
|
||||||
database.RetentionForeverDays,
|
database.RetentionForeverDays,
|
||||||
"the test value must not be treated as retain-forever",
|
"the test value must not be rescued by the forever skip",
|
||||||
)
|
)
|
||||||
|
|
||||||
webhookID := createWebhook(
|
webhookID := createWebhook(
|
||||||
|
|||||||
@@ -102,13 +102,6 @@ func TestWebhookDBManager_TotalsSurviveReopen(t *testing.T) {
|
|||||||
// seedExpiredEvents stores count events created at the given time,
|
// seedExpiredEvents stores count events created at the given time,
|
||||||
// each with a delivered delivery to one target and a failed delivery
|
// each with a delivered delivery to one target and a failed delivery
|
||||||
// to the other, and one attempt for each delivery.
|
// to the other, and one attempt for each delivery.
|
||||||
//
|
|
||||||
// It and seedBareEvents insert 50 rows per statement, not more. The
|
|
||||||
// SQLite driver looks up each parameter's value by scanning the
|
|
||||||
// statement's arguments from the first until it reaches that
|
|
||||||
// parameter's, so the time to bind a statement grows with the square of
|
|
||||||
// its parameter count: at 500 rows, several thousand parameters, the
|
|
||||||
// seeding took most of these tests' time under -race.
|
|
||||||
func seedExpiredEvents(
|
func seedExpiredEvents(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
db *gorm.DB,
|
db *gorm.DB,
|
||||||
@@ -145,8 +138,8 @@ func seedExpiredEvents(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
require.NoError(t, db.CreateInBatches(events, 50).Error)
|
require.NoError(t, db.CreateInBatches(events, 500).Error)
|
||||||
require.NoError(t, db.CreateInBatches(deliveries, 50).Error)
|
require.NoError(t, db.CreateInBatches(deliveries, 500).Error)
|
||||||
|
|
||||||
results := make([]database.DeliveryResult, len(deliveries))
|
results := make([]database.DeliveryResult, len(deliveries))
|
||||||
for i := range deliveries {
|
for i := range deliveries {
|
||||||
@@ -155,31 +148,7 @@ func seedExpiredEvents(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
require.NoError(t, db.CreateInBatches(results, 50).Error)
|
require.NoError(t, db.CreateInBatches(results, 500).Error)
|
||||||
}
|
|
||||||
|
|
||||||
// seedBareEvents stores count events created at the given time, with
|
|
||||||
// no deliveries.
|
|
||||||
func seedBareEvents(
|
|
||||||
t *testing.T,
|
|
||||||
db *gorm.DB,
|
|
||||||
webhookID string,
|
|
||||||
count int,
|
|
||||||
createdAt time.Time,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
events := make([]database.Event, count)
|
|
||||||
for i := range events {
|
|
||||||
events[i] = database.Event{
|
|
||||||
WebhookID: webhookID,
|
|
||||||
EntrypointID: uuid.New().String(),
|
|
||||||
Method: http.MethodPost,
|
|
||||||
}
|
|
||||||
events[i].CreatedAt = createdAt
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NoError(t, db.CreateInBatches(events, 50).Error)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRetentionReaper_PrunesMoreThanOneBatch verifies that a prune
|
// TestRetentionReaper_PrunesMoreThanOneBatch verifies that a prune
|
||||||
@@ -257,161 +226,3 @@ func TestRetentionReaper_PrunesMoreThanOneBatch(t *testing.T) {
|
|||||||
assert.Equal(t, eventTotals, readEventTotals(t, db))
|
assert.Equal(t, eventTotals, readEventTotals(t, db))
|
||||||
assert.Equal(t, targetTotals, readTargetTotals(t, db))
|
assert.Equal(t, targetTotals, readTargetTotals(t, db))
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestRetentionReaper_WriteDuringPruneSucceeds verifies that a prune
|
|
||||||
// of several batches lets other writers in between its batches: an
|
|
||||||
// event stored once the first batch is deleted is stored while expired
|
|
||||||
// events are still left, not only after the prune has finished.
|
|
||||||
func TestRetentionReaper_WriteDuringPruneSucceeds(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupRetentionTest(t)
|
|
||||||
|
|
||||||
webhookID := createWebhook(t, env.mainDB.DB(), 30)
|
|
||||||
|
|
||||||
db, err := env.mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Three batches of expired events, with nothing else stored: only
|
|
||||||
// the number of batches matters here.
|
|
||||||
expired := 3 * database.ExportReapBatchSize
|
|
||||||
seedBareEvents(t, db, webhookID, expired,
|
|
||||||
time.Now().Add(-40*24*time.Hour))
|
|
||||||
|
|
||||||
cutoff := time.Now().Add(-30 * 24 * time.Hour)
|
|
||||||
countExpired := func() int64 {
|
|
||||||
var count int64
|
|
||||||
|
|
||||||
require.NoError(t, db.Model(&database.Event{}).
|
|
||||||
Where("created_at < ?", cutoff).
|
|
||||||
Count(&count).Error)
|
|
||||||
|
|
||||||
return count
|
|
||||||
}
|
|
||||||
|
|
||||||
pruned := make(chan struct{})
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
defer close(pruned)
|
|
||||||
|
|
||||||
env.reaper.ExportSweep(context.Background())
|
|
||||||
}()
|
|
||||||
|
|
||||||
t.Cleanup(func() { <-pruned })
|
|
||||||
|
|
||||||
// Every stored event is expired until the write below.
|
|
||||||
require.Eventually(t, func() bool {
|
|
||||||
var count int64
|
|
||||||
|
|
||||||
err := db.Model(&database.Event{}).Count(&count).Error
|
|
||||||
|
|
||||||
return err == nil && count < int64(expired)
|
|
||||||
}, 10*time.Second, 10*time.Millisecond)
|
|
||||||
|
|
||||||
event := &database.Event{
|
|
||||||
WebhookID: webhookID,
|
|
||||||
EntrypointID: uuid.New().String(),
|
|
||||||
Method: http.MethodPost,
|
|
||||||
}
|
|
||||||
require.NoError(t, db.Create(event).Error)
|
|
||||||
|
|
||||||
assert.Positive(t, countExpired(),
|
|
||||||
"the event was stored only after the whole prune")
|
|
||||||
|
|
||||||
<-pruned
|
|
||||||
|
|
||||||
assert.Zero(t, countExpired())
|
|
||||||
|
|
||||||
var stored database.Event
|
|
||||||
|
|
||||||
require.NoError(t, db.First(&stored, "id = ?", event.ID).Error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRetentionReaper_StopDuringPruneLeavesTheRest verifies that
|
|
||||||
// stopping the reaper during a prune of several batches returns
|
|
||||||
// between two batches, well inside the stop timeout, leaving the
|
|
||||||
// remaining expired events for the next sweep, and that the totals
|
|
||||||
// match the rows left.
|
|
||||||
func TestRetentionReaper_StopDuringPruneLeavesTheRest(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupRetentionTest(t)
|
|
||||||
|
|
||||||
webhookID := createWebhook(t, env.mainDB.DB(), 30)
|
|
||||||
|
|
||||||
db, err := env.mgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
// Two batches and one more of expired events, a few of them with a
|
|
||||||
// delivered and a failed delivery for the target totals to count.
|
|
||||||
// Most carry nothing else, to keep the test quick.
|
|
||||||
const withDeliveries = 10
|
|
||||||
|
|
||||||
expiredAt := time.Now().Add(-40 * 24 * time.Hour)
|
|
||||||
delivered, failed := uuid.New().String(), uuid.New().String()
|
|
||||||
seedExpiredEvents(t, db, webhookID, withDeliveries, expiredAt,
|
|
||||||
delivered, failed)
|
|
||||||
seedBareEvents(t, db, webhookID,
|
|
||||||
2*database.ExportReapBatchSize+1-withDeliveries, expiredAt)
|
|
||||||
|
|
||||||
n := int64(2*database.ExportReapBatchSize + 1)
|
|
||||||
require.NoError(t, database.AddEventTotals(db, database.EventTotals{
|
|
||||||
Events: n,
|
|
||||||
}))
|
|
||||||
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
|
||||||
TargetID: delivered, Deliveries: withDeliveries,
|
|
||||||
Delivered: withDeliveries,
|
|
||||||
}))
|
|
||||||
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
|
||||||
TargetID: failed, Deliveries: withDeliveries,
|
|
||||||
Failed: withDeliveries,
|
|
||||||
}))
|
|
||||||
|
|
||||||
env.reaper.ExportSetInterval(time.Millisecond)
|
|
||||||
env.reaper.ExportStart()
|
|
||||||
|
|
||||||
// Stop once the first batch is deleted. The stop lands in the pause
|
|
||||||
// after it, or at worst during the second batch, so at least the
|
|
||||||
// last event is left.
|
|
||||||
require.Eventually(t, func() bool {
|
|
||||||
var count int64
|
|
||||||
|
|
||||||
err := db.Model(&database.Event{}).Count(&count).Error
|
|
||||||
|
|
||||||
return err == nil && count < n
|
|
||||||
}, 10*time.Second, 10*time.Millisecond)
|
|
||||||
|
|
||||||
// The app's stop timeout.
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
require.NoError(t, env.reaper.ExportStop(ctx))
|
|
||||||
|
|
||||||
var events int64
|
|
||||||
|
|
||||||
require.NoError(t, db.Model(&database.Event{}).Count(&events).Error)
|
|
||||||
assert.Positive(t, events, "the stop waited for the whole prune")
|
|
||||||
|
|
||||||
eventTotals := readEventTotals(t, db)
|
|
||||||
assert.Equal(t, events, eventTotals.Events-eventTotals.EventsRemoved)
|
|
||||||
|
|
||||||
targetTotals := readTargetTotals(t, db)
|
|
||||||
require.Len(t, targetTotals, 2)
|
|
||||||
|
|
||||||
for target, totals := range targetTotals {
|
|
||||||
var deliveries, failures int64
|
|
||||||
|
|
||||||
require.NoError(t, db.Model(&database.Delivery{}).
|
|
||||||
Where("target_id = ?", target).
|
|
||||||
Count(&deliveries).Error)
|
|
||||||
require.NoError(t, db.Model(&database.Delivery{}).
|
|
||||||
Where("target_id = ? AND status = ?",
|
|
||||||
target, database.DeliveryStatusFailed).
|
|
||||||
Count(&failures).Error)
|
|
||||||
|
|
||||||
assert.Equal(t, deliveries,
|
|
||||||
totals.Deliveries-totals.DeliveriesRemoved, target)
|
|
||||||
assert.Equal(t, failures, totals.Failed-totals.FailedRemoved,
|
|
||||||
target)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -33,19 +33,6 @@ var errInvalidCachedDBType = errors.New(
|
|||||||
"invalid cached database type",
|
"invalid cached database type",
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrEventDBNotRemoved is in DeleteDB's error when the event
|
|
||||||
// database file itself could not be removed: it is still on disk.
|
|
||||||
var ErrEventDBNotRemoved = errors.New(
|
|
||||||
"event database file not removed",
|
|
||||||
)
|
|
||||||
|
|
||||||
// ErrSidecarNotRemoved is in DeleteDB's error when the event
|
|
||||||
// database file was removed, so its events are gone, but its -wal
|
|
||||||
// or -shm sidecar could not be.
|
|
||||||
var ErrSidecarNotRemoved = errors.New(
|
|
||||||
"event database file removed, but a -wal or -shm sidecar was not",
|
|
||||||
)
|
|
||||||
|
|
||||||
// WebhookDBManager manages per-webhook SQLite database files
|
// WebhookDBManager manages per-webhook SQLite database files
|
||||||
// for event storage. Each webhook gets its own dedicated
|
// for event storage. Each webhook gets its own dedicated
|
||||||
// database containing Events, Deliveries, DeliveryResults and the
|
// database containing Events, Deliveries, DeliveryResults and the
|
||||||
@@ -164,10 +151,7 @@ func (m *WebhookDBManager) DBExists(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// DeleteDB closes the connection and deletes the database file
|
// DeleteDB closes the connection and deletes the database file
|
||||||
// for a webhook, with its -wal and -shm sidecars. The files are
|
// for a webhook. The file is permanently removed.
|
||||||
// permanently removed. Each file is tried even when another could
|
|
||||||
// not be removed, and the error wraps ErrEventDBNotRemoved or
|
|
||||||
// ErrSidecarNotRemoved to say which was left, naming each file.
|
|
||||||
func (m *WebhookDBManager) DeleteDB(
|
func (m *WebhookDBManager) DeleteDB(
|
||||||
webhookID string,
|
webhookID string,
|
||||||
) error {
|
) error {
|
||||||
@@ -186,23 +170,16 @@ func (m *WebhookDBManager) DeleteDB(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Delete the main DB file and WAL/SHM files
|
||||||
path := m.dbPath(webhookID)
|
path := m.dbPath(webhookID)
|
||||||
|
for _, suffix := range []string{"", "-wal", "-shm"} {
|
||||||
dbErr := removeFile(path)
|
err := os.Remove(path + suffix)
|
||||||
sidecarErr := errors.Join(
|
if err != nil && !os.IsNotExist(err) {
|
||||||
removeFile(path+"-wal"),
|
return fmt.Errorf(
|
||||||
removeFile(path+"-shm"),
|
"deleting webhook database file %s%s: %w",
|
||||||
)
|
path, suffix, err,
|
||||||
|
)
|
||||||
if dbErr != nil {
|
}
|
||||||
return fmt.Errorf(
|
|
||||||
"%w: %w",
|
|
||||||
ErrEventDBNotRemoved, errors.Join(dbErr, sidecarErr),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if sidecarErr != nil {
|
|
||||||
return fmt.Errorf("%w: %w", ErrSidecarNotRemoved, sidecarErr)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
m.log.Info(
|
m.log.Info(
|
||||||
@@ -213,17 +190,6 @@ func (m *WebhookDBManager) DeleteDB(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// removeFile removes path. A file that is already gone counts as
|
|
||||||
// removed; the error from any other failure names the file.
|
|
||||||
func removeFile(path string) error {
|
|
||||||
err := os.Remove(path)
|
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// CloseAll closes all open per-webhook database connections.
|
// CloseAll closes all open per-webhook database connections.
|
||||||
// Called during application shutdown.
|
// Called during application shutdown.
|
||||||
func (m *WebhookDBManager) CloseAll() error {
|
func (m *WebhookDBManager) CloseAll() error {
|
||||||
|
|||||||
@@ -182,91 +182,17 @@ func TestWebhookDBManager_DeleteDB(t *testing.T) {
|
|||||||
}
|
}
|
||||||
require.NoError(t, db.Create(event).Error)
|
require.NoError(t, db.Create(event).Error)
|
||||||
|
|
||||||
// Under WAL, an open database that has been written to has both
|
|
||||||
// sidecars beside it.
|
|
||||||
dbPath := mgr.DBPath(webhookID)
|
|
||||||
require.FileExists(t, dbPath+"-wal")
|
|
||||||
require.FileExists(t, dbPath+"-shm")
|
|
||||||
|
|
||||||
// Delete the DB
|
// Delete the DB
|
||||||
require.NoError(t, mgr.DeleteDB(webhookID))
|
require.NoError(t, mgr.DeleteDB(webhookID))
|
||||||
|
|
||||||
// File should no longer exist
|
// File should no longer exist
|
||||||
assert.False(t, mgr.DBExists(webhookID))
|
assert.False(t, mgr.DBExists(webhookID))
|
||||||
|
|
||||||
// Verify the files are actually gone from disk
|
// Verify the file is actually gone from disk
|
||||||
assert.NoFileExists(t, dbPath)
|
|
||||||
assert.NoFileExists(t, dbPath+"-wal")
|
|
||||||
assert.NoFileExists(t, dbPath+"-shm")
|
|
||||||
}
|
|
||||||
|
|
||||||
// blockRemoval puts a non-empty directory at path, which os.Remove
|
|
||||||
// cannot remove whoever runs the test, root included.
|
|
||||||
func blockRemoval(t *testing.T, path string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
require.NoError(t, os.MkdirAll(filepath.Join(path, "keep"), 0o700))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWebhookDBManager_DeleteDBKeepsDatabaseFile proves that when the
|
|
||||||
// event database file cannot be removed, the error says so, and both
|
|
||||||
// sidecars are still removed.
|
|
||||||
func TestWebhookDBManager_DeleteDBKeepsDatabaseFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
mgr, lc := setupTestWebhookDBManager(t)
|
|
||||||
ctx := context.Background()
|
|
||||||
require.NoError(t, lc.Start(ctx))
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
|
||||||
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
dbPath := mgr.DBPath(webhookID)
|
dbPath := mgr.DBPath(webhookID)
|
||||||
|
|
||||||
blockRemoval(t, dbPath)
|
_, err = os.Stat(dbPath)
|
||||||
require.NoError(t, os.WriteFile(dbPath+"-wal", nil, 0o600))
|
assert.True(t, os.IsNotExist(err))
|
||||||
require.NoError(t, os.WriteFile(dbPath+"-shm", nil, 0o600))
|
|
||||||
|
|
||||||
err := mgr.DeleteDB(webhookID)
|
|
||||||
|
|
||||||
require.ErrorIs(t, err, database.ErrEventDBNotRemoved)
|
|
||||||
require.NotErrorIs(t, err, database.ErrSidecarNotRemoved)
|
|
||||||
assert.Contains(t, err.Error(), dbPath)
|
|
||||||
assert.NoFileExists(t, dbPath+"-wal")
|
|
||||||
assert.NoFileExists(t, dbPath+"-shm")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestWebhookDBManager_DeleteDBKeepsSidecar proves that when the
|
|
||||||
// event database file is removed but a sidecar is not, the error
|
|
||||||
// says the database file is gone, and the other sidecar is still
|
|
||||||
// removed.
|
|
||||||
func TestWebhookDBManager_DeleteDBKeepsSidecar(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
mgr, lc := setupTestWebhookDBManager(t)
|
|
||||||
ctx := context.Background()
|
|
||||||
require.NoError(t, lc.Start(ctx))
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
|
||||||
|
|
||||||
webhookID := uuid.New().String()
|
|
||||||
dbPath := mgr.DBPath(webhookID)
|
|
||||||
|
|
||||||
require.NoError(t, mgr.CreateDB(webhookID))
|
|
||||||
// Closing removes the sidecars, so the ones below are the only
|
|
||||||
// ones there.
|
|
||||||
require.NoError(t, mgr.CloseAll())
|
|
||||||
|
|
||||||
blockRemoval(t, dbPath+"-wal")
|
|
||||||
require.NoError(t, os.WriteFile(dbPath+"-shm", nil, 0o600))
|
|
||||||
|
|
||||||
err := mgr.DeleteDB(webhookID)
|
|
||||||
|
|
||||||
require.ErrorIs(t, err, database.ErrSidecarNotRemoved)
|
|
||||||
require.NotErrorIs(t, err, database.ErrEventDBNotRemoved)
|
|
||||||
assert.Contains(t, err.Error(), dbPath+"-wal")
|
|
||||||
assert.NoFileExists(t, dbPath)
|
|
||||||
assert.NoFileExists(t, dbPath+"-shm")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWebhookDBManager_LazyCreation(t *testing.T) {
|
func TestWebhookDBManager_LazyCreation(t *testing.T) {
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"gorm.io/gorm"
|
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
"sneak.berlin/go/webhooker/internal/config"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/lifecycle"
|
"sneak.berlin/go/webhooker/internal/lifecycle"
|
||||||
@@ -26,14 +25,14 @@ type ArchiveSweeperParams struct {
|
|||||||
Logger *logger.Logger
|
Logger *logger.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArchiveSweeper periodically prunes expired rows from the
|
// ArchiveSweeper periodically prunes expired rows from
|
||||||
// archive databases of database targets that carry a positive
|
// per-webhook archive databases whose database target carries a
|
||||||
// expiry.
|
// positive expiry.
|
||||||
//
|
//
|
||||||
// Without it, pruning happens only when an archive is
|
// Without it, pruning happens only when an archive is
|
||||||
// (re)opened, and archives are only ever reopened by writes: an
|
// (re)opened, and archives are only ever reopened by writes: an
|
||||||
// archive whose target has stopped receiving events would keep
|
// archive belonging to a webhook that has stopped receiving
|
||||||
// its expired rows forever. The sweep closes
|
// events would keep its expired rows forever. The sweep closes
|
||||||
// that gap without changing anything for archives whose expiry
|
// that gap without changing anything for archives whose expiry
|
||||||
// is unset or "never".
|
// is unset or "never".
|
||||||
//
|
//
|
||||||
@@ -45,13 +44,8 @@ type ArchiveSweeper struct {
|
|||||||
eng *Engine
|
eng *Engine
|
||||||
log *slog.Logger
|
log *slog.Logger
|
||||||
interval time.Duration
|
interval time.Duration
|
||||||
|
cancel context.CancelFunc
|
||||||
// cancel needs no lock: fx calls the stop hook only after the
|
wg sync.WaitGroup
|
||||||
// start hook has returned, so stop never reads it while start
|
|
||||||
// is still setting it.
|
|
||||||
cancel context.CancelFunc
|
|
||||||
|
|
||||||
wg sync.WaitGroup
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewArchiveSweeper creates the archive sweeper and registers
|
// NewArchiveSweeper creates the archive sweeper and registers
|
||||||
@@ -161,25 +155,17 @@ func (s *ArchiveSweeper) run(ctx context.Context) {
|
|||||||
// soft-deleted along with it, so GORM's default scope already
|
// soft-deleted along with it, so GORM's default scope already
|
||||||
// excludes them.
|
// excludes them.
|
||||||
//
|
//
|
||||||
// A failure for one target is logged and the sweep continues,
|
// A failure for one webhook is logged and the sweep continues,
|
||||||
// matching how the write path already treats a prune error as
|
// matching how the write path already treats a prune error as
|
||||||
// non-fatal.
|
// non-fatal.
|
||||||
func (s *ArchiveSweeper) sweep(ctx context.Context) {
|
func (s *ArchiveSweeper) sweep(ctx context.Context) {
|
||||||
var targets []database.Target
|
var targets []database.Target
|
||||||
|
|
||||||
err := s.db.DB().
|
err := s.db.DB().
|
||||||
WithContext(ctx).
|
|
||||||
Model(&database.Target{}).
|
Model(&database.Target{}).
|
||||||
Where("type = ?", database.TargetTypeDatabase).
|
Where("type = ?", database.TargetTypeDatabase).
|
||||||
Find(&targets).Error
|
Find(&targets).Error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// The app stopping as a sweep starts cancels the listing.
|
|
||||||
// Stopping is not a failure, so it must not produce an
|
|
||||||
// error line.
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
s.log.Error(
|
s.log.Error(
|
||||||
"archive sweep: failed to list database targets",
|
"archive sweep: failed to list database targets",
|
||||||
"error", err,
|
"error", err,
|
||||||
@@ -224,20 +210,19 @@ func (s *ArchiveSweeper) sweepTarget(target *database.Target) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
err = s.eng.dbTarget.sweepArchive(target.ID, expiry)
|
err = s.eng.dbTarget.sweepWebhook(target.WebhookID, expiry)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// A writer evicted, or a target row gone, underneath the sweep
|
// A writer evicted underneath the sweep means the operator
|
||||||
// means the operator deleted the target or its webhook while
|
// deleted the webhook (or its last database target) while the
|
||||||
// the sweep was walking the target list. That is an ordinary
|
// sweep was walking the target list. That is an ordinary
|
||||||
// interleaving, not a failure, so it must not produce an
|
// interleaving, not a failure, so it must not produce an
|
||||||
// error line.
|
// error line.
|
||||||
if errors.Is(err, errArchiveWriterEvicted) ||
|
if errors.Is(err, errArchiveWriterEvicted) {
|
||||||
errors.Is(err, gorm.ErrRecordNotFound) {
|
|
||||||
s.log.Debug(
|
s.log.Debug(
|
||||||
"archive sweep: target deleted mid-sweep",
|
"archive sweep: writer evicted mid-sweep",
|
||||||
"webhook_id", target.WebhookID,
|
"webhook_id", target.WebhookID,
|
||||||
"target_id", target.ID,
|
"target_id", target.ID,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,11 +1,9 @@
|
|||||||
package delivery_test
|
package delivery_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -22,7 +20,6 @@ import (
|
|||||||
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -37,23 +34,18 @@ const (
|
|||||||
sweepConcurrentWrites = 20
|
sweepConcurrentWrites = 20
|
||||||
)
|
)
|
||||||
|
|
||||||
// archiveTestWebhookName is the name of every webhook
|
// sweeperEnv bundles the pieces an archive sweep test drives:
|
||||||
// seedDatabaseTarget creates. It is not safe in a file name as it
|
// a main configuration database holding webhooks and targets, a
|
||||||
// stands, so every archive test goes through archiveNamePart.
|
// delivery engine owning the archive writer registry, and the
|
||||||
const archiveTestWebhookName = "Sweep Test!"
|
// data directory the archive files live in.
|
||||||
|
type sweeperEnv struct {
|
||||||
// archiveEnv bundles the pieces an archive test drives: a main
|
|
||||||
// configuration database holding webhooks and targets, a delivery
|
|
||||||
// engine owning the archive writer registry, the archive sweeper,
|
|
||||||
// and the data directory the archive files live in.
|
|
||||||
type archiveEnv struct {
|
|
||||||
sweeper *delivery.ArchiveSweeper
|
sweeper *delivery.ArchiveSweeper
|
||||||
eng *delivery.Engine
|
eng *delivery.Engine
|
||||||
mainDB *database.Database
|
mainDB *database.Database
|
||||||
dataDir string
|
dataDir string
|
||||||
}
|
}
|
||||||
|
|
||||||
func setupArchiveTest(t *testing.T) *archiveEnv {
|
func setupSweeperTest(t *testing.T) *sweeperEnv {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dataDir := t.TempDir()
|
dataDir := t.TempDir()
|
||||||
@@ -71,8 +63,7 @@ func setupArchiveTest(t *testing.T) *archiveEnv {
|
|||||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||||
|
|
||||||
gdb, err := gorm.Open(
|
gdb, err := gorm.Open(
|
||||||
sqlite.Dialector{Conn: sqlDB},
|
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
|
||||||
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
|
||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
@@ -87,7 +78,7 @@ func setupArchiveTest(t *testing.T) *archiveEnv {
|
|||||||
1,
|
1,
|
||||||
)
|
)
|
||||||
|
|
||||||
return &archiveEnv{
|
return &sweeperEnv{
|
||||||
sweeper: delivery.NewTestArchiveSweeper(
|
sweeper: delivery.NewTestArchiveSweeper(
|
||||||
mainDB, eng, log,
|
mainDB, eng, log,
|
||||||
),
|
),
|
||||||
@@ -97,27 +88,25 @@ func setupArchiveTest(t *testing.T) *archiveEnv {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// archivePath returns where the engine keeps a database target's
|
// archivePath returns where the engine keeps a webhook's
|
||||||
// archive file, for the names seedDatabaseTarget gave it.
|
// archive file.
|
||||||
func (env *archiveEnv) archivePath(tgt *database.Target) string {
|
func (env *sweeperEnv) archivePath(webhookID string) string {
|
||||||
return filepath.Join(
|
return filepath.Join(
|
||||||
env.dataDir,
|
env.dataDir, fmt.Sprintf("archive-%s.db", webhookID),
|
||||||
delivery.ArchiveFileName(
|
|
||||||
archiveTestWebhookName, tgt.Name, tgt.ID,
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// seedDatabaseTarget creates a webhook with one database target
|
// seedDatabaseTarget creates a webhook with one database target
|
||||||
// carrying the given target config JSON, and returns the target.
|
// carrying the given target config JSON, and returns the
|
||||||
func (env *archiveEnv) seedDatabaseTarget(
|
// webhook id.
|
||||||
|
func (env *sweeperEnv) seedDatabaseTarget(
|
||||||
t *testing.T, configJSON string,
|
t *testing.T, configJSON string,
|
||||||
) *database.Target {
|
) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
wh := &database.Webhook{
|
wh := &database.Webhook{
|
||||||
UserID: uuid.New().String(),
|
UserID: uuid.New().String(),
|
||||||
Name: archiveTestWebhookName,
|
Name: "sweep-test",
|
||||||
}
|
}
|
||||||
require.NoError(
|
require.NoError(
|
||||||
t,
|
t,
|
||||||
@@ -126,19 +115,9 @@ func (env *archiveEnv) seedDatabaseTarget(
|
|||||||
Create(wh).Error,
|
Create(wh).Error,
|
||||||
)
|
)
|
||||||
|
|
||||||
return env.addDatabaseTarget(t, wh.ID, configJSON)
|
|
||||||
}
|
|
||||||
|
|
||||||
// addDatabaseTarget creates one more database target on an
|
|
||||||
// existing webhook and returns it.
|
|
||||||
func (env *archiveEnv) addDatabaseTarget(
|
|
||||||
t *testing.T, webhookID, configJSON string,
|
|
||||||
) *database.Target {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
tgt := &database.Target{
|
tgt := &database.Target{
|
||||||
WebhookID: webhookID,
|
WebhookID: wh.ID,
|
||||||
Name: "Archive",
|
Name: "archive",
|
||||||
Type: database.TargetTypeDatabase,
|
Type: database.TargetTypeDatabase,
|
||||||
Active: true,
|
Active: true,
|
||||||
Config: configJSON,
|
Config: configJSON,
|
||||||
@@ -150,19 +129,19 @@ func (env *archiveEnv) addDatabaseTarget(
|
|||||||
Create(tgt).Error,
|
Create(tgt).Error,
|
||||||
)
|
)
|
||||||
|
|
||||||
return tgt
|
return wh.ID
|
||||||
}
|
}
|
||||||
|
|
||||||
// seedArchiveRows creates the archive file for a target and
|
// seedArchiveRows creates the archive file for a webhook and
|
||||||
// inserts one row per supplied archived-at timestamp, returning
|
// inserts one row per supplied archived-at timestamp, returning
|
||||||
// the archive path. The handle is closed before returning, so
|
// the archive path. The handle is closed before returning, so
|
||||||
// the archive is idle exactly as it would be with no traffic.
|
// the archive is idle exactly as it would be with no traffic.
|
||||||
func (env *archiveEnv) seedArchiveRows(
|
func (env *sweeperEnv) seedArchiveRows(
|
||||||
t *testing.T, tgt *database.Target, archivedAt ...time.Time,
|
t *testing.T, webhookID string, archivedAt ...time.Time,
|
||||||
) string {
|
) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
path := env.archivePath(tgt)
|
path := env.archivePath(webhookID)
|
||||||
|
|
||||||
sqlDB, err := sql.Open(
|
sqlDB, err := sql.Open(
|
||||||
"sqlite", fmt.Sprintf("file:%s?mode=rwc", path),
|
"sqlite", fmt.Sprintf("file:%s?mode=rwc", path),
|
||||||
@@ -170,8 +149,7 @@ func (env *archiveEnv) seedArchiveRows(
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
gdb, err := gorm.Open(
|
gdb, err := gorm.Open(
|
||||||
sqlite.Dialector{Conn: sqlDB},
|
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
|
||||||
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
|
||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
@@ -182,7 +160,7 @@ func (env *archiveEnv) seedArchiveRows(
|
|||||||
for i, at := range archivedAt {
|
for i, at := range archivedAt {
|
||||||
row := delivery.ExportArchivedEvent{
|
row := delivery.ExportArchivedEvent{
|
||||||
EventID: fmt.Sprintf("ev-%d", i),
|
EventID: fmt.Sprintf("ev-%d", i),
|
||||||
WebhookID: tgt.WebhookID,
|
WebhookID: webhookID,
|
||||||
Method: http.MethodPost,
|
Method: http.MethodPost,
|
||||||
Body: `{"seeded":true}`,
|
Body: `{"seeded":true}`,
|
||||||
ArchivedAt: at,
|
ArchivedAt: at,
|
||||||
@@ -230,8 +208,7 @@ func countArchivedRows(path string) (int64, error) {
|
|||||||
defer func() { _ = sqlDB.Close() }()
|
defer func() { _ = sqlDB.Close() }()
|
||||||
|
|
||||||
gdb, err := gorm.Open(
|
gdb, err := gorm.Open(
|
||||||
sqlite.Dialector{Conn: sqlDB},
|
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
|
||||||
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
@@ -266,13 +243,13 @@ func TestArchiveSweeper_LoopOutlivesStartHookContext(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
|
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
path := env.seedArchiveRows(
|
path := env.seedArchiveRows(
|
||||||
t, tgt,
|
t, webhookID,
|
||||||
now.Add(-48*time.Hour),
|
now.Add(-48*time.Hour),
|
||||||
now.Add(-time.Minute),
|
now.Add(-time.Minute),
|
||||||
)
|
)
|
||||||
@@ -310,60 +287,60 @@ func TestArchiveSweeper_LoopOutlivesStartHookContext(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_DoesNotResurrectEvictedWriter covers the
|
// TestArchiveSweep_DoesNotResurrectEvictedWriter covers the
|
||||||
// interleaving where a sweep tick has already listed a target
|
// interleaving where a sweep tick has already listed a webhook's
|
||||||
// when the target is deleted and its writer evicted. The sweep
|
// target when the webhook is deleted and its writer evicted. The
|
||||||
// must not put a writer back into the registry: nothing would
|
// sweep must not put a writer back into the registry: nothing
|
||||||
// ever evict it again, which is precisely the leak this change
|
// would ever evict it again, which is precisely the leak this
|
||||||
// exists to close.
|
// change exists to close.
|
||||||
func TestArchiveSweep_DoesNotResurrectEvictedWriter(
|
func TestArchiveSweep_DoesNotResurrectEvictedWriter(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
env.seedArchiveRows(
|
env.seedArchiveRows(
|
||||||
t, tgt, time.Now().Add(-48*time.Hour),
|
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
// Prime the registry the way a delivery would, then evict as
|
// Prime the registry the way a delivery would, then evict as
|
||||||
// the deletion path does. The target row is deliberately left
|
// the deletion path does. The target row is deliberately left
|
||||||
// in place: this is the tick that listed the target before
|
// in place: this is the tick that listed the webhook before
|
||||||
// the deletion committed.
|
// the deletion committed.
|
||||||
_, err := env.eng.ExportEnsureArchiveWriter(tgt.ID)
|
_, err := env.eng.ExportEnsureArchiveWriter(webhookID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
env.eng.EvictTarget(tgt.ID)
|
env.eng.EvictWebhook(webhookID)
|
||||||
require.False(t, env.eng.ExportHasArchiveWriter(tgt.ID))
|
require.False(t, env.eng.ExportHasArchiveWriter(webhookID))
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
|
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||||
"a sweep must never re-register a writer for a target "+
|
"a sweep must never re-register a writer for a webhook "+
|
||||||
"whose registry entry has already been released",
|
"whose registry entry has already been released",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_LeavesNoRegistryEntry states the same
|
// TestArchiveSweep_LeavesNoRegistryEntry states the same
|
||||||
// invariant in its general form: sweeping an archive whose
|
// invariant in its general form: sweeping an archive whose
|
||||||
// target has no cached writer must not leave one behind, so the
|
// webhook has no cached writer must not leave one behind, so the
|
||||||
// registry keeps holding only writers a delivery created and an
|
// registry keeps holding only writers a delivery created and an
|
||||||
// eviction can reach.
|
// eviction can reach.
|
||||||
func TestArchiveSweep_LeavesNoRegistryEntry(t *testing.T) {
|
func TestArchiveSweep_LeavesNoRegistryEntry(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
path := env.seedArchiveRows(
|
path := env.seedArchiveRows(
|
||||||
t, tgt,
|
t, webhookID,
|
||||||
time.Now().Add(-48*time.Hour),
|
time.Now().Add(-48*time.Hour),
|
||||||
time.Now().Add(-time.Minute),
|
time.Now().Add(-time.Minute),
|
||||||
)
|
)
|
||||||
|
|
||||||
require.False(t, env.eng.ExportHasArchiveWriter(tgt.ID))
|
require.False(t, env.eng.ExportHasArchiveWriter(webhookID))
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
|
|
||||||
@@ -372,7 +349,7 @@ func TestArchiveSweep_LeavesNoRegistryEntry(t *testing.T) {
|
|||||||
"the sweep must still prune an idle archive",
|
"the sweep must still prune an idle archive",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||||
"the sweep must release the registry entry it created",
|
"the sweep must release the registry entry it created",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -387,31 +364,34 @@ func TestArchiveSweep_KeepsWriterAdoptedByDelivery(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
env.seedArchiveRows(
|
env.seedArchiveRows(
|
||||||
t, tgt, time.Now().Add(-48*time.Hour),
|
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
webhookDB := testWebhookDB(t)
|
||||||
event := seedEvent(t, webhookDB, `{"n":1}`)
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
|
event.WebhookID = webhookID
|
||||||
|
d := seedDatabaseTargetDelivery(
|
||||||
|
t, webhookDB, event, `{"expiry":"1h"}`,
|
||||||
|
)
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
require.False(t, env.eng.ExportHasArchiveWriter(tgt.ID))
|
require.False(t, env.eng.ExportHasArchiveWriter(webhookID))
|
||||||
|
|
||||||
env.eng.ExportDeliverDatabase(webhookDB, d)
|
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
|
|
||||||
assert.True(
|
assert.True(
|
||||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||||
"a delivery's writer must stay registered",
|
"a delivery's writer must stay registered",
|
||||||
)
|
)
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
|
|
||||||
assert.True(
|
assert.True(
|
||||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||||
"a sweep must not drop a writer a delivery owns",
|
"a sweep must not drop a writer a delivery owns",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -443,15 +423,15 @@ func TestArchiveSweep_KeepsWriterAdoptedDuringSweep(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
env.seedArchiveRows(
|
env.seedArchiveRows(
|
||||||
t, tgt, time.Now().Add(-48*time.Hour),
|
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
sweepWriter, created, err := env.eng.ExportSweepWriterFor(
|
sweepWriter, created, err := env.eng.ExportSweepWriterFor(
|
||||||
tgt.ID,
|
webhookID,
|
||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.True(
|
require.True(
|
||||||
@@ -462,34 +442,37 @@ func TestArchiveSweep_KeepsWriterAdoptedDuringSweep(
|
|||||||
// The delivery lands mid-sweep and adopts the entry.
|
// The delivery lands mid-sweep and adopts the entry.
|
||||||
webhookDB := testWebhookDB(t)
|
webhookDB := testWebhookDB(t)
|
||||||
event := seedEvent(t, webhookDB, `{"n":1}`)
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
|
event.WebhookID = webhookID
|
||||||
|
d := seedDatabaseTargetDelivery(
|
||||||
|
t, webhookDB, event, `{"expiry":"1h"}`,
|
||||||
|
)
|
||||||
|
|
||||||
env.eng.ExportDeliverDatabase(webhookDB, d)
|
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
|
|
||||||
adopted := env.eng.ExportArchiveWriterFor(tgt.ID)
|
adopted := env.eng.ExportArchiveWriterFor(webhookID)
|
||||||
require.NotNil(t, adopted)
|
require.NotNil(t, adopted)
|
||||||
require.True(
|
require.True(
|
||||||
t, sweepWriter.Same(adopted),
|
t, sweepWriter.Same(adopted),
|
||||||
"the delivery must have adopted the sweep's writer",
|
"the delivery must have adopted the sweep's writer",
|
||||||
)
|
)
|
||||||
require.True(
|
require.True(
|
||||||
t, env.eng.ExportArchiveHandleOpen(tgt.ID),
|
t, env.eng.ExportArchiveHandleOpen(webhookID),
|
||||||
"the delivery leaves the archive handle open",
|
"the delivery leaves the archive handle open",
|
||||||
)
|
)
|
||||||
|
|
||||||
// The sweep finishes.
|
// The sweep finishes.
|
||||||
env.eng.ExportReleaseSweepWriter(tgt.ID, sweepWriter)
|
env.eng.ExportReleaseSweepWriter(webhookID, sweepWriter)
|
||||||
|
|
||||||
require.True(
|
require.True(
|
||||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||||
"a writer adopted by a delivery during a sweep must "+
|
"a writer adopted by a delivery during a sweep must "+
|
||||||
"stay registered, or its open handle is unreachable",
|
"stay registered, or its open handle is unreachable",
|
||||||
)
|
)
|
||||||
|
|
||||||
env.eng.EvictTarget(tgt.ID)
|
env.eng.EvictWebhook(webhookID)
|
||||||
|
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||||
"the adopted writer must still be evictable",
|
"the adopted writer must still be evictable",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
@@ -498,34 +481,34 @@ func TestArchiveSweep_KeepsWriterAdoptedDuringSweep(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_ContinuesAfterPerTargetFailure proves a
|
// TestArchiveSweep_ContinuesAfterPerWebhookFailure proves a
|
||||||
// failure for one target does not abort the sweep for the
|
// failure for one webhook does not abort the sweep for the
|
||||||
// others: an unparseable expiry and an unreadable archive both
|
// others: an unparseable expiry and an unreadable archive both
|
||||||
// have to be logged and stepped over.
|
// have to be logged and stepped over.
|
||||||
func TestArchiveSweep_ContinuesAfterPerTargetFailure(
|
func TestArchiveSweep_ContinuesAfterPerWebhookFailure(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
// Seeded first so the sweep reaches them before the healthy
|
// Seeded first so the sweep reaches them before the healthy
|
||||||
// target: targets come back in insertion order.
|
// webhook: targets come back in insertion order.
|
||||||
badConfig := env.seedDatabaseTarget(t, `{"expiry":"!!!"}`)
|
badConfigID := env.seedDatabaseTarget(t, `{"expiry":"!!!"}`)
|
||||||
env.seedArchiveRows(
|
env.seedArchiveRows(
|
||||||
t, badConfig, time.Now().Add(-48*time.Hour),
|
t, badConfigID, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
corrupt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
corruptID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
require.NoError(t, os.WriteFile(
|
require.NoError(t, os.WriteFile(
|
||||||
env.archivePath(corrupt),
|
env.archivePath(corruptID),
|
||||||
[]byte("this is not a sqlite database"),
|
[]byte("this is not a sqlite database"),
|
||||||
0o600,
|
0o600,
|
||||||
))
|
))
|
||||||
|
|
||||||
healthy := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
healthyID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
healthyPath := env.seedArchiveRows(
|
healthyPath := env.seedArchiveRows(
|
||||||
t, healthy,
|
t, healthyID,
|
||||||
time.Now().Add(-48*time.Hour),
|
time.Now().Add(-48*time.Hour),
|
||||||
time.Now().Add(-time.Minute),
|
time.Now().Add(-time.Minute),
|
||||||
)
|
)
|
||||||
@@ -535,14 +518,14 @@ func TestArchiveSweep_ContinuesAfterPerTargetFailure(
|
|||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, []string{sweepRowNew},
|
t, []string{sweepRowNew},
|
||||||
archivedEventIDs(t, healthyPath),
|
archivedEventIDs(t, healthyPath),
|
||||||
"a failure for an earlier target must not stop the "+
|
"a failure for an earlier webhook must not stop the "+
|
||||||
"sweep from pruning the ones after it",
|
"sweep from pruning the ones after it",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_OpenExistingDoesNotCreateFile pins the second
|
// TestArchiveSweep_OpenExistingDoesNotCreateFile pins the second
|
||||||
// of the two no-create guards. The first is the stat in
|
// of the two no-create guards. The first is the stat in
|
||||||
// sweepExpired; this one is the SQLite open mode, which is what
|
// sweepWebhook; this one is the SQLite open mode, which is what
|
||||||
// protects the window between that stat and the open. Flipping
|
// protects the window between that stat and the open. Flipping
|
||||||
// the sweep's mode to create-if-missing makes this fail.
|
// the sweep's mode to create-if-missing makes this fail.
|
||||||
func TestArchiveSweep_OpenExistingDoesNotCreateFile(
|
func TestArchiveSweep_OpenExistingDoesNotCreateFile(
|
||||||
@@ -578,13 +561,13 @@ func TestArchiveSweep_OpenExistingDoesNotCreateFile(
|
|||||||
func TestArchiveSweep_PrunesIdleArchive(t *testing.T) {
|
func TestArchiveSweep_PrunesIdleArchive(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
|
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
path := env.seedArchiveRows(
|
path := env.seedArchiveRows(
|
||||||
t, tgt,
|
t, webhookID,
|
||||||
now.Add(-48*time.Hour),
|
now.Add(-48*time.Hour),
|
||||||
now.Add(-time.Minute),
|
now.Add(-time.Minute),
|
||||||
)
|
)
|
||||||
@@ -617,11 +600,11 @@ func TestArchiveSweep_PrunesIdleArchive(t *testing.T) {
|
|||||||
func TestArchiveSweep_LeavesArchiveClosed(t *testing.T) {
|
func TestArchiveSweep_LeavesArchiveClosed(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
path := env.seedArchiveRows(
|
path := env.seedArchiveRows(
|
||||||
t, tgt, time.Now().Add(-48*time.Hour),
|
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
w := delivery.NewExportArchiveWriter(
|
w := delivery.NewExportArchiveWriter(
|
||||||
@@ -657,94 +640,39 @@ func TestArchiveSweep_ClosesHandleOfRegisteredWriter(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
env.seedArchiveRows(
|
env.seedArchiveRows(
|
||||||
t, tgt, time.Now().Add(-48*time.Hour),
|
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
webhookDB := testWebhookDB(t)
|
||||||
event := seedEvent(t, webhookDB, `{"n":1}`)
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
|
event.WebhookID = webhookID
|
||||||
|
d := seedDatabaseTargetDelivery(
|
||||||
|
t, webhookDB, event, `{"expiry":"1h"}`,
|
||||||
|
)
|
||||||
|
|
||||||
env.eng.ExportDeliverDatabase(webhookDB, d)
|
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
|
|
||||||
require.True(
|
require.True(
|
||||||
t, env.eng.ExportArchiveHandleOpen(tgt.ID),
|
t, env.eng.ExportArchiveHandleOpen(webhookID),
|
||||||
"the delivery must leave the archive handle open",
|
"the delivery must leave the archive handle open",
|
||||||
)
|
)
|
||||||
|
|
||||||
env.sweeper.ExportSweep(context.Background())
|
env.sweeper.ExportSweep(context.Background())
|
||||||
|
|
||||||
require.True(
|
require.True(
|
||||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||||
"the delivery's registry entry must survive the sweep",
|
"the delivery's registry entry must survive the sweep",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.eng.ExportArchiveHandleOpen(tgt.ID),
|
t, env.eng.ExportArchiveHandleOpen(webhookID),
|
||||||
"the sweep must leave the archive closed",
|
"the sweep must leave the archive closed",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_ClosesHandleBeforeReopening proves the sweep
|
|
||||||
// closes the handle it finds open before it reopens the file.
|
|
||||||
// TestArchiveSweep_LeavesArchiveClosed cannot see this: without the
|
|
||||||
// close, the reopen replaces the handle without closing it, the
|
|
||||||
// sweep then closes only the new one, and one connection leaks per
|
|
||||||
// archive per sweep.
|
|
||||||
func TestArchiveSweep_ClosesHandleBeforeReopening(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive.db")
|
|
||||||
|
|
||||||
w := delivery.NewExportArchiveWriter(
|
|
||||||
path, archiveTestLogger(), 0,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, w.Open(time.Hour))
|
|
||||||
|
|
||||||
before, err := w.DB().DB()
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
require.NoError(t, w.SweepExpired(time.Hour))
|
|
||||||
|
|
||||||
assert.Error(
|
|
||||||
t, before.PingContext(t.Context()),
|
|
||||||
"the handle open before the sweep must be closed by it",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveSweep_CancelledSweepLogsNoError proves a sweep whose
|
|
||||||
// context is already cancelled, as when the app stops just as a
|
|
||||||
// sweep starts, returns without an error line: stopping is not a
|
|
||||||
// failure.
|
|
||||||
func TestArchiveSweep_CancelledSweepLogsNoError(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
|
|
||||||
var errorLines bytes.Buffer
|
|
||||||
|
|
||||||
sweeper := delivery.NewTestArchiveSweeper(
|
|
||||||
env.mainDB, env.eng,
|
|
||||||
slog.New(slog.NewTextHandler(
|
|
||||||
&errorLines,
|
|
||||||
&slog.HandlerOptions{Level: slog.LevelError},
|
|
||||||
)),
|
|
||||||
)
|
|
||||||
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
cancel()
|
|
||||||
|
|
||||||
sweeper.ExportSweep(ctx)
|
|
||||||
|
|
||||||
assert.Empty(
|
|
||||||
t, errorLines.String(),
|
|
||||||
"a cancelled sweep must not log at error level",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveSweep_NeverExpiryUntouched proves the sweep is a
|
// TestArchiveSweep_NeverExpiryUntouched proves the sweep is a
|
||||||
// no-op for the default retention policy, so archives with no
|
// no-op for the default retention policy, so archives with no
|
||||||
// expiry (or the literal "never") behave exactly as before.
|
// expiry (or the literal "never") behave exactly as before.
|
||||||
@@ -756,11 +684,11 @@ func TestArchiveSweep_NeverExpiryUntouched(t *testing.T) {
|
|||||||
`{"expiry":""}`,
|
`{"expiry":""}`,
|
||||||
"",
|
"",
|
||||||
} {
|
} {
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, configJSON)
|
webhookID := env.seedDatabaseTarget(t, configJSON)
|
||||||
path := env.seedArchiveRows(
|
path := env.seedArchiveRows(
|
||||||
t, tgt,
|
t, webhookID,
|
||||||
time.Now().Add(-10000*time.Hour),
|
time.Now().Add(-10000*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -771,7 +699,7 @@ func TestArchiveSweep_NeverExpiryUntouched(t *testing.T) {
|
|||||||
"config %q must keep rows forever", configJSON,
|
"config %q must keep rows forever", configJSON,
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||||
"config %q must leave no registry entry behind",
|
"config %q must leave no registry entry behind",
|
||||||
configJSON,
|
configJSON,
|
||||||
)
|
)
|
||||||
@@ -794,10 +722,10 @@ func TestArchiveSweep_NeverExpirySkipsBeforeOpening(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"never"}`)
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"never"}`)
|
||||||
path := env.archivePath(tgt)
|
path := env.archivePath(webhookID)
|
||||||
|
|
||||||
seedUnmigratedArchive(t, path)
|
seedUnmigratedArchive(t, path)
|
||||||
require.False(t, archiveTableExists(t, path))
|
require.False(t, archiveTableExists(t, path))
|
||||||
@@ -840,16 +768,16 @@ func archiveTableExists(t *testing.T, path string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_DoesNotCreateArchiveFile proves the sweep
|
// TestArchiveSweep_DoesNotCreateArchiveFile proves the sweep
|
||||||
// never conjures an archive: a database target that has never
|
// never conjures an archive: a webhook with a database target
|
||||||
// received an event must still have no archive file (nor SQLite
|
// that has never received an event must still have no archive
|
||||||
// sidecar) after a sweep, and no registry entry either.
|
// file (nor SQLite sidecar) after a sweep.
|
||||||
func TestArchiveSweep_DoesNotCreateArchiveFile(t *testing.T) {
|
func TestArchiveSweep_DoesNotCreateArchiveFile(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
path := env.archivePath(tgt)
|
path := env.archivePath(webhookID)
|
||||||
|
|
||||||
require.NoFileExists(t, path)
|
require.NoFileExists(t, path)
|
||||||
|
|
||||||
@@ -861,11 +789,6 @@ func TestArchiveSweep_DoesNotCreateArchiveFile(t *testing.T) {
|
|||||||
"the sweep must not create an archive file",
|
"the sweep must not create an archive file",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
assert.False(
|
|
||||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
|
||||||
"the sweep must leave no registry entry behind",
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_DoesNotCreateAfterWriterExists covers the
|
// TestArchiveSweep_DoesNotCreateAfterWriterExists covers the
|
||||||
@@ -877,11 +800,11 @@ func TestArchiveSweep_DoesNotCreateAfterWriterExists(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
|
|
||||||
path, err := env.eng.ExportEnsureArchiveWriter(tgt.ID)
|
path, err := env.eng.ExportEnsureArchiveWriter(webhookID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.NoFileExists(t, path)
|
require.NoFileExists(t, path)
|
||||||
|
|
||||||
@@ -896,17 +819,17 @@ func TestArchiveSweep_DoesNotCreateAfterWriterExists(
|
|||||||
func TestArchiveSweep_SkipsDeletedWebhookTargets(t *testing.T) {
|
func TestArchiveSweep_SkipsDeletedWebhookTargets(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
path := env.seedArchiveRows(
|
path := env.seedArchiveRows(
|
||||||
t, tgt, time.Now().Add(-48*time.Hour),
|
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
require.NoError(
|
require.NoError(
|
||||||
t,
|
t,
|
||||||
env.mainDB.DB().
|
env.mainDB.DB().
|
||||||
Where("webhook_id = ?", tgt.WebhookID).
|
Where("webhook_id = ?", webhookID).
|
||||||
Delete(&database.Target{}).Error,
|
Delete(&database.Target{}).Error,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -919,14 +842,14 @@ func TestArchiveSweep_SkipsDeletedWebhookTargets(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweep_ConcurrentWrites proves the sweep serialises
|
// TestArchiveSweep_ConcurrentWrites proves the sweep serialises
|
||||||
// against writes through the target's writer mutex. Run under
|
// against writes through the per-webhook writer mutex. Run
|
||||||
// -race, an unsynchronised sweep would be caught here.
|
// under -race, an unsynchronised sweep would be caught here.
|
||||||
func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
webhookDB := testWebhookDB(t)
|
||||||
|
|
||||||
@@ -939,10 +862,13 @@ func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
|||||||
|
|
||||||
for range sweepConcurrentWrites {
|
for range sweepConcurrentWrites {
|
||||||
event := seedEvent(t, webhookDB, `{"n":1}`)
|
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||||
|
event.WebhookID = webhookID
|
||||||
|
|
||||||
deliveries = append(
|
deliveries = append(
|
||||||
deliveries,
|
deliveries,
|
||||||
seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
|
seedDatabaseTargetDelivery(
|
||||||
|
t, webhookDB, event, `{"expiry":"1h"}`,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -968,7 +894,7 @@ func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
|||||||
|
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
|
||||||
assert.FileExists(t, env.archivePath(tgt))
|
assert.FileExists(t, env.archivePath(webhookID))
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveSweeper_StopsCleanly proves the background loop
|
// TestArchiveSweeper_StopsCleanly proves the background loop
|
||||||
@@ -976,11 +902,11 @@ func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
|||||||
func TestArchiveSweeper_StopsCleanly(t *testing.T) {
|
func TestArchiveSweeper_StopsCleanly(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||||
env.seedArchiveRows(
|
env.seedArchiveRows(
|
||||||
t, tgt, time.Now().Add(-48*time.Hour),
|
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||||
)
|
)
|
||||||
|
|
||||||
env.sweeper.ExportSetInterval(time.Millisecond)
|
env.sweeper.ExportSetInterval(time.Millisecond)
|
||||||
@@ -1004,7 +930,7 @@ func TestArchiveSweeper_StopHookHonoursStopTimeout(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
env := setupSweeperTest(t)
|
||||||
|
|
||||||
lc := &recordingLifecycle{}
|
lc := &recordingLifecycle{}
|
||||||
env.sweeper.ExportRegisterHooks(lc)
|
env.sweeper.ExportRegisterHooks(lc)
|
||||||
|
|||||||
@@ -114,64 +114,3 @@ func TestUpdateDeliveryStatus_DeletedDeliveryIsNotCounted(t *testing.T) {
|
|||||||
assert.Equal(t, database.TargetTotals{TargetID: targetID},
|
assert.Equal(t, database.TargetTotals{TargetID: targetID},
|
||||||
targetTotals(t, db, targetID))
|
targetTotals(t, db, targetID))
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestUpdateDeliveryStatus_FinishedDeliveryIsNotSettledAgain covers a
|
|
||||||
// delivery settled a second time, as recovery can do when a worker has
|
|
||||||
// settled it since recovery read it. Neither status writes over the
|
|
||||||
// first, and the totals do not move.
|
|
||||||
func TestUpdateDeliveryStatus_FinishedDeliveryIsNotSettledAgain(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
finished := []database.DeliveryStatus{
|
|
||||||
database.DeliveryStatusDelivered,
|
|
||||||
database.DeliveryStatusFailed,
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, first := range finished {
|
|
||||||
t.Run(string(first), func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := testWebhookDB(t)
|
|
||||||
e := testEngine(t, 1)
|
|
||||||
event := seedEvent(t, db, `{}`)
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
d := seedDelivery(
|
|
||||||
t, db, event.ID, targetID,
|
|
||||||
database.DeliveryStatusRetrying,
|
|
||||||
)
|
|
||||||
|
|
||||||
// The delivery as recovery read it, before the worker
|
|
||||||
// settled it.
|
|
||||||
readBefore := d
|
|
||||||
|
|
||||||
require.NoError(t, e.ExportUpdateDeliveryStatus(
|
|
||||||
db, &d, first,
|
|
||||||
))
|
|
||||||
|
|
||||||
var settled database.Delivery
|
|
||||||
|
|
||||||
require.NoError(t, db.First(&settled, "id = ?", d.ID).Error)
|
|
||||||
require.NotNil(t, settled.FinishedAt)
|
|
||||||
|
|
||||||
totals := targetTotals(t, db, targetID)
|
|
||||||
|
|
||||||
for _, again := range finished {
|
|
||||||
stale := readBefore
|
|
||||||
|
|
||||||
require.NoError(t, e.ExportUpdateDeliveryStatus(
|
|
||||||
db, &stale, again,
|
|
||||||
))
|
|
||||||
}
|
|
||||||
|
|
||||||
var stored database.Delivery
|
|
||||||
|
|
||||||
require.NoError(t, db.First(&stored, "id = ?", d.ID).Error)
|
|
||||||
assert.Equal(t, first, stored.Status)
|
|
||||||
require.NotNil(t, stored.FinishedAt)
|
|
||||||
assert.True(t, settled.FinishedAt.Equal(*stored.FinishedAt))
|
|
||||||
assert.Equal(t, totals, targetTotals(t, db, targetID))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
+34
-90
@@ -14,7 +14,6 @@ import (
|
|||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/globals"
|
|
||||||
"sneak.berlin/go/webhooker/internal/lifecycle"
|
"sneak.berlin/go/webhooker/internal/lifecycle"
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
"sneak.berlin/go/webhooker/internal/metrics"
|
"sneak.berlin/go/webhooker/internal/metrics"
|
||||||
@@ -123,24 +122,21 @@ type Notifier interface {
|
|||||||
Notify(tasks []Task)
|
Notify(tasks []Task)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Archives is how the handlers keep the database targets' archive
|
// WebhookEvictor releases the delivery engine's per-webhook
|
||||||
// files in step with the configuration. Deleting a webhook or a
|
// state for a webhook that no longer needs it — currently the
|
||||||
// target releases the cached archive writers, whose open file
|
// cached archive writer of the database target, whose open
|
||||||
// handles would otherwise outlive them; renaming one renames the
|
// file handle would otherwise outlive the webhook.
|
||||||
// archive files, which are named for the webhook and the target
|
|
||||||
// (see ArchiveFileName).
|
|
||||||
//
|
//
|
||||||
// It is deliberately separate from Notifier: archiving lifecycle
|
// It is deliberately separate from Notifier and deliberately
|
||||||
// is not notification, and a small interface keeps the handlers
|
// one method wide: archiving lifecycle is not notification, and
|
||||||
// package free of any dependency on the engine's internals while
|
// a single-method interface keeps the handlers package free of
|
||||||
// staying trivially fakeable in tests.
|
// any dependency on the engine's internals while staying
|
||||||
|
// trivially fakeable in tests.
|
||||||
//
|
//
|
||||||
// Neither eviction deletes an archive file. Both are idempotent
|
// EvictWebhook never deletes an archive file. It is idempotent
|
||||||
// and are no-ops for a webhook or target with no engine state.
|
// and is a no-op for a webhook with no engine state.
|
||||||
type Archives interface {
|
type WebhookEvictor interface {
|
||||||
EvictWebhook(webhookID string)
|
EvictWebhook(webhookID string)
|
||||||
EvictTarget(targetID string)
|
|
||||||
Rename(targetID, webhookName, targetName string) error
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// EngineParams are the fx dependencies for the delivery
|
// EngineParams are the fx dependencies for the delivery
|
||||||
@@ -150,10 +146,8 @@ type EngineParams struct {
|
|||||||
|
|
||||||
DB *database.Database
|
DB *database.Database
|
||||||
DBManager *database.WebhookDBManager
|
DBManager *database.WebhookDBManager
|
||||||
Globals *globals.Globals
|
|
||||||
Logger *logger.Logger
|
Logger *logger.Logger
|
||||||
SSRFGuard *Guard
|
SSRFGuard *Guard
|
||||||
Metrics *metrics.Set
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Engine processes queued deliveries in the background
|
// Engine processes queued deliveries in the background
|
||||||
@@ -173,14 +167,10 @@ type Engine struct {
|
|||||||
retryCh chan Task
|
retryCh chan Task
|
||||||
workers int
|
workers int
|
||||||
|
|
||||||
// version is the running build's version, the one the web UI
|
// mtr is the delivery metric set. Production wires the
|
||||||
// footer shows. userAgent puts it on every outbound request.
|
// process-wide one; a test can substitute a set registered on
|
||||||
version string
|
// a private registry so its assertions are not disturbed by
|
||||||
|
// deliveries other tests are making at the same time.
|
||||||
// mtr is the delivery metric set. Production wires the one
|
|
||||||
// registered on the registry /metrics serves; a test can
|
|
||||||
// substitute a set registered on a registry it holds, so it can
|
|
||||||
// gather what its own deliveries recorded.
|
|
||||||
mtr *metrics.Set
|
mtr *metrics.Set
|
||||||
|
|
||||||
// targets maps each target type to its implementation.
|
// targets maps each target type to its implementation.
|
||||||
@@ -191,7 +181,7 @@ type Engine struct {
|
|||||||
httpTarget *httpTarget
|
httpTarget *httpTarget
|
||||||
|
|
||||||
// dbTarget is retained so the engine can reach the archive
|
// dbTarget is retained so the engine can reach the archive
|
||||||
// writer registry for eviction, renames and the idle sweep.
|
// writer registry for webhook eviction and the idle sweep.
|
||||||
dbTarget *databaseTarget
|
dbTarget *databaseTarget
|
||||||
|
|
||||||
// inflight is the set of deliveries this engine currently owns.
|
// inflight is the set of deliveries this engine currently owns.
|
||||||
@@ -214,8 +204,7 @@ func New(
|
|||||||
deliveryCh: make(chan Task, deliveryChannelSize),
|
deliveryCh: make(chan Task, deliveryChannelSize),
|
||||||
retryCh: make(chan Task, retryChannelSize),
|
retryCh: make(chan Task, retryChannelSize),
|
||||||
workers: defaultWorkers,
|
workers: defaultWorkers,
|
||||||
version: params.Globals.Version,
|
mtr: metrics.Default(),
|
||||||
mtr: params.Metrics,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
e.initTargets(&http.Client{
|
e.initTargets(&http.Client{
|
||||||
@@ -260,44 +249,17 @@ func (e *Engine) Notify(tasks []Task) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// EvictWebhook implements Archives. The cached archive writer of
|
// EvictWebhook implements WebhookEvictor. It releases the
|
||||||
// every database target of the webhook is dropped from the
|
// engine's per-webhook archiving state: the database target's
|
||||||
// registry and its file handle closed. The archive files
|
// cached archive writer is dropped from the registry and its
|
||||||
// themselves are left on disk — they are long-term storage the
|
// file handle closed. The archive file itself is left on disk
|
||||||
// operator owns.
|
// — it is long-term storage the operator owns.
|
||||||
func (e *Engine) EvictWebhook(webhookID string) {
|
func (e *Engine) EvictWebhook(webhookID string) {
|
||||||
if e.dbTarget == nil {
|
if e.dbTarget == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
e.dbTarget.evictWebhook(webhookID)
|
e.dbTarget.evict(webhookID)
|
||||||
}
|
|
||||||
|
|
||||||
// EvictTarget implements Archives. It is EvictWebhook for a single
|
|
||||||
// database target, and leaves the archive file on disk the same
|
|
||||||
// way.
|
|
||||||
func (e *Engine) EvictTarget(targetID string) {
|
|
||||||
if e.dbTarget == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
e.dbTarget.evict(targetID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rename implements Archives. It renames a database target's
|
|
||||||
// archive file to ArchiveFileName(webhookName, targetName,
|
|
||||||
// targetID), under the lock the target's archive writes and the
|
|
||||||
// idle sweep take. It never replaces a file: if one already has the
|
|
||||||
// new name, the error is ErrArchiveNameTaken. The caller renames
|
|
||||||
// before it saves the new name: see databaseTarget.rename.
|
|
||||||
func (e *Engine) Rename(
|
|
||||||
targetID, webhookName, targetName string,
|
|
||||||
) error {
|
|
||||||
if e.dbTarget == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return e.dbTarget.rename(targetID, webhookName, targetName)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ScheduleRetry schedules a task to be re-enqueued onto the
|
// ScheduleRetry schedules a task to be re-enqueued onto the
|
||||||
@@ -338,13 +300,6 @@ func (e *Engine) ScheduleRetry(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// userAgent is the User-Agent header of every http and slack
|
|
||||||
// delivery request: the program name and the running build's
|
|
||||||
// version.
|
|
||||||
func (e *Engine) userAgent() string {
|
|
||||||
return "webhooker/" + e.version
|
|
||||||
}
|
|
||||||
|
|
||||||
// registerHooks wires the engine's start and stop into the fx
|
// registerHooks wires the engine's start and stop into the fx
|
||||||
// lifecycle. The start hook's context is deliberately ignored
|
// lifecycle. The start hook's context is deliberately ignored
|
||||||
// (see start for why the worker pool must not inherit it); the
|
// (see start for why the worker pool must not inherit it); the
|
||||||
@@ -411,8 +366,7 @@ func (e *Engine) start() {
|
|||||||
// Once the pool has drained it closes the archive writers, so a
|
// Once the pool has drained it closes the archive writers, so a
|
||||||
// clean stop leaves no archive -wal behind. Nothing else holds a
|
// clean stop leaves no archive -wal behind. Nothing else holds a
|
||||||
// writer for long by then: the archive sweeper stops before the
|
// writer for long by then: the archive sweeper stops before the
|
||||||
// engine, and deleting or renaming a webhook or target only closes
|
// engine, and deleting a webhook only closes one. If the pool did
|
||||||
// or moves one. If the pool did
|
|
||||||
// not drain in time, the writers are left open, as a kill would
|
// not drain in time, the writers are left open, as a kill would
|
||||||
// leave them. Closing them would wait for any write in progress,
|
// leave them. Closing them would wait for any write in progress,
|
||||||
// and a worker still running would then open new writers that
|
// and a worker still running would then open new writers that
|
||||||
@@ -577,11 +531,6 @@ func (e *Engine) processRetryTask(
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set before anything below can fail the delivery: the failure is
|
|
||||||
// added to this target's totals.
|
|
||||||
d.EventID = task.EventID
|
|
||||||
d.TargetID = task.TargetID
|
|
||||||
|
|
||||||
if d.Status != database.DeliveryStatusRetrying {
|
if d.Status != database.DeliveryStatusRetrying {
|
||||||
e.log.Debug(
|
e.log.Debug(
|
||||||
"skipping retry for delivery "+
|
"skipping retry for delivery "+
|
||||||
@@ -613,6 +562,8 @@ func (e *Engine) processRetryTask(
|
|||||||
}
|
}
|
||||||
|
|
||||||
target := buildTargetFromTask(task)
|
target := buildTargetFromTask(task)
|
||||||
|
d.EventID = task.EventID
|
||||||
|
d.TargetID = task.TargetID
|
||||||
d.Event = event
|
d.Event = event
|
||||||
d.Target = target
|
d.Target = target
|
||||||
|
|
||||||
@@ -1626,11 +1577,9 @@ func (e *Engine) updateDeliveryStatus(
|
|||||||
|
|
||||||
// writeDeliveryStatus writes a delivery's new status. A delivery that
|
// writeDeliveryStatus writes a delivery's new status. A delivery that
|
||||||
// becomes delivered or failed also gets the time it finished, and is
|
// becomes delivered or failed also gets the time it finished, and is
|
||||||
// added to its target's delivered or failed total. That write changes
|
// added to its target's delivered or failed total. It is counted only
|
||||||
// only a delivery not yet delivered or failed, and the total moves
|
// if the row was still there to update: retention may have deleted it
|
||||||
// only when it changed a row: retention may have deleted the delivery
|
// while the engine was working on it.
|
||||||
// while the engine was working on it, and a recovery path may settle
|
|
||||||
// a delivery that a worker has already settled.
|
|
||||||
func writeDeliveryStatus(
|
func writeDeliveryStatus(
|
||||||
tx *gorm.DB,
|
tx *gorm.DB,
|
||||||
d *database.Delivery,
|
d *database.Delivery,
|
||||||
@@ -1640,15 +1589,10 @@ func writeDeliveryStatus(
|
|||||||
return tx.Model(d).Update("status", status).Error
|
return tx.Model(d).Update("status", status).Error
|
||||||
}
|
}
|
||||||
|
|
||||||
res := tx.Model(d).
|
res := tx.Model(d).Updates(map[string]any{
|
||||||
Where("status NOT IN ?", []database.DeliveryStatus{
|
"status": status,
|
||||||
database.DeliveryStatusDelivered,
|
"finished_at": time.Now(),
|
||||||
database.DeliveryStatusFailed,
|
})
|
||||||
}).
|
|
||||||
Updates(map[string]any{
|
|
||||||
"status": status,
|
|
||||||
"finished_at": time.Now(),
|
|
||||||
})
|
|
||||||
if res.Error != nil || res.RowsAffected == 0 {
|
if res.Error != nil || res.RowsAffected == 0 {
|
||||||
return res.Error
|
return res.Error
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,7 +23,6 @@ import (
|
|||||||
_ "modernc.org/sqlite"
|
_ "modernc.org/sqlite"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// iSetup holds common integration test dependencies.
|
// iSetup holds common integration test dependencies.
|
||||||
@@ -81,8 +80,7 @@ func iMainDB(t *testing.T) *gorm.DB {
|
|||||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||||
|
|
||||||
db, err := gorm.Open(
|
db, err := gorm.Open(
|
||||||
sqlite.Dialector{Conn: sqlDB},
|
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
|
||||||
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
|
||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
@@ -357,14 +355,9 @@ func TestProcessRetryTask_SuccessfulRetry(t *testing.T) {
|
|||||||
|
|
||||||
s := newISetup(t)
|
s := newISetup(t)
|
||||||
|
|
||||||
var receivedBody string
|
|
||||||
|
|
||||||
ts := httptest.NewServer(
|
ts := httptest.NewServer(
|
||||||
http.HandlerFunc(
|
http.HandlerFunc(
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
func(w http.ResponseWriter, _ *http.Request) {
|
||||||
body, _ := io.ReadAll(r.Body)
|
|
||||||
receivedBody = string(body)
|
|
||||||
|
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -404,8 +397,6 @@ func TestProcessRetryTask_SuccessfulRetry(t *testing.T) {
|
|||||||
context.TODO(), &task,
|
context.TODO(), &task,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Equal(t, event.Body, receivedBody)
|
|
||||||
|
|
||||||
iAssertStatus(t, s.WebhookDB, d.ID,
|
iAssertStatus(t, s.WebhookDB, d.ID,
|
||||||
database.DeliveryStatusDelivered,
|
database.DeliveryStatusDelivered,
|
||||||
)
|
)
|
||||||
@@ -452,14 +443,9 @@ func TestProcessRetryTask_LargeBody_FetchFromDB(
|
|||||||
|
|
||||||
s := newISetup(t)
|
s := newISetup(t)
|
||||||
|
|
||||||
var receivedBody string
|
|
||||||
|
|
||||||
ts := httptest.NewServer(
|
ts := httptest.NewServer(
|
||||||
http.HandlerFunc(
|
http.HandlerFunc(
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
func(w http.ResponseWriter, _ *http.Request) {
|
||||||
body, _ := io.ReadAll(r.Body)
|
|
||||||
receivedBody = string(body)
|
|
||||||
|
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -496,8 +482,6 @@ func TestProcessRetryTask_LargeBody_FetchFromDB(
|
|||||||
context.TODO(), &task,
|
context.TODO(), &task,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Equal(t, largeBody, receivedBody)
|
|
||||||
|
|
||||||
iAssertStatus(t, s.WebhookDB, d.ID,
|
iAssertStatus(t, s.WebhookDB, d.ID,
|
||||||
database.DeliveryStatusDelivered,
|
database.DeliveryStatusDelivered,
|
||||||
)
|
)
|
||||||
@@ -1427,32 +1411,6 @@ func TestDeliverHTTP_InvalidConfig(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestDeliverHTTP_InvalidConfigUnrecordedStaysPending: a delivery is
|
|
||||||
// failed for an invalid config only once the reason is recorded.
|
|
||||||
// Unrecorded, it stays pending, where the sweep finds it again.
|
|
||||||
func TestDeliverHTTP_InvalidConfigUnrecordedStaysPending(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
db := testWebhookDB(t)
|
|
||||||
e := testEngine(t, 1)
|
|
||||||
|
|
||||||
event, del := iSeedEventAndDelivery(
|
|
||||||
t, db, `{"config":"invalid"}`, "",
|
|
||||||
)
|
|
||||||
|
|
||||||
task, d := iHTTPTaskAndDelivery(
|
|
||||||
event, del, "bad-config", `not-json`, 0, 1,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, db.Exec("drop table delivery_results").Error)
|
|
||||||
|
|
||||||
e.ExportDeliverHTTP(context.TODO(), db, d, task)
|
|
||||||
|
|
||||||
iAssertStatus(t, db, del.ID,
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- Notify batching ---
|
// --- Notify batching ---
|
||||||
|
|
||||||
func TestNotify_MultipleTasks(t *testing.T) {
|
func TestNotify_MultipleTasks(t *testing.T) {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package delivery_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"fmt"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -9,7 +10,6 @@ import (
|
|||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"gorm.io/gorm/clause"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
)
|
)
|
||||||
@@ -272,35 +272,22 @@ func TestEngine_StopHookHonoursStopTimeout(t *testing.T) {
|
|||||||
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
||||||
}
|
}
|
||||||
|
|
||||||
// deliverToArchive gives the setup's webhook a database target,
|
// deliverToArchive runs one delivery to a database target through
|
||||||
// runs one delivery to it through the running engine, and returns
|
// the running engine and returns the webhook's archive file path.
|
||||||
// the target's ID and archive file path. The archive writer holds
|
// The archive writer holds the file open afterwards.
|
||||||
// the file open afterwards.
|
func deliverToArchive(t *testing.T, s iSetup) string {
|
||||||
func deliverToArchive(t *testing.T, s iSetup) (string, string) {
|
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
iCreateWebhook(t, s.MainDB, s.WebhookID, "hook")
|
|
||||||
|
|
||||||
tgt := &database.Target{
|
|
||||||
WebhookID: s.WebhookID,
|
|
||||||
Name: "archive",
|
|
||||||
Type: database.TargetTypeDatabase,
|
|
||||||
}
|
|
||||||
require.NoError(
|
|
||||||
t, s.MainDB.Omit(clause.Associations).Create(tgt).Error,
|
|
||||||
)
|
|
||||||
|
|
||||||
deliveryID, task := seedLogTask(t, s)
|
deliveryID, task := seedLogTask(t, s)
|
||||||
task.TargetID = tgt.ID
|
|
||||||
task.TargetType = database.TargetTypeDatabase
|
task.TargetType = database.TargetTypeDatabase
|
||||||
|
|
||||||
s.Engine.Notify([]delivery.Task{task})
|
s.Engine.Notify([]delivery.Task{task})
|
||||||
|
|
||||||
iWaitForDelivered(t, s.WebhookDB, deliveryID)
|
iWaitForDelivered(t, s.WebhookDB, deliveryID)
|
||||||
|
|
||||||
return tgt.ID, filepath.Join(
|
return filepath.Join(
|
||||||
filepath.Dir(s.DBMgr.DBPath(s.WebhookID)),
|
filepath.Dir(s.DBMgr.DBPath(s.WebhookID)),
|
||||||
"archive-hook-archive-"+tgt.ID+".db",
|
fmt.Sprintf("archive-%s.db", s.WebhookID),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -317,7 +304,7 @@ func TestEngine_StopHookClosesArchives(t *testing.T) {
|
|||||||
|
|
||||||
lc := startEngineViaHook(t, s.Engine)
|
lc := startEngineViaHook(t, s.Engine)
|
||||||
|
|
||||||
_, path := deliverToArchive(t, s)
|
path := deliverToArchive(t, s)
|
||||||
require.FileExists(
|
require.FileExists(
|
||||||
t, path+"-wal",
|
t, path+"-wal",
|
||||||
"an open archive should have a -wal for the stop to remove",
|
"an open archive should have a -wal for the stop to remove",
|
||||||
@@ -351,7 +338,7 @@ func TestEngine_StopHookTimeoutLeavesArchivesOpen(t *testing.T) {
|
|||||||
|
|
||||||
lc := startEngineViaHook(t, s.Engine)
|
lc := startEngineViaHook(t, s.Engine)
|
||||||
|
|
||||||
targetID, _ := deliverToArchive(t, s)
|
deliverToArchive(t, s)
|
||||||
|
|
||||||
release := make(chan struct{})
|
release := make(chan struct{})
|
||||||
|
|
||||||
@@ -365,7 +352,7 @@ func TestEngine_StopHookTimeoutLeavesArchivesOpen(t *testing.T) {
|
|||||||
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
||||||
|
|
||||||
require.True(
|
require.True(
|
||||||
t, s.Engine.ExportArchiveHandleOpen(targetID),
|
t, s.Engine.ExportArchiveHandleOpen(s.WebhookID),
|
||||||
"a stop that timed out must not close archive writers",
|
"a stop that timed out must not close archive writers",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
@@ -26,7 +25,6 @@ import (
|
|||||||
_ "modernc.org/sqlite"
|
_ "modernc.org/sqlite"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
|
||||||
"sneak.berlin/go/webhooker/internal/metrics"
|
"sneak.berlin/go/webhooker/internal/metrics"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -51,8 +49,7 @@ func testWebhookDB(t *testing.T) *gorm.DB {
|
|||||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||||
|
|
||||||
db, err := gorm.Open(
|
db, err := gorm.Open(
|
||||||
sqlite.Dialector{Conn: sqlDB},
|
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
|
||||||
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
|
||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
@@ -354,15 +351,23 @@ func TestDeliverDatabase_ImmediateSuccess(
|
|||||||
|
|
||||||
db := testWebhookDB(t)
|
db := testWebhookDB(t)
|
||||||
|
|
||||||
// The database target archives for real, so the engine needs
|
// The database target archives for real now, so the engine
|
||||||
// the target in the main database and a data directory.
|
// needs a webhook DB manager to locate the data directory.
|
||||||
env := setupArchiveTest(t)
|
e := delivery.NewTestEngineWithDB(
|
||||||
tgt := env.seedDatabaseTarget(t, "")
|
nil,
|
||||||
|
database.NewTestWebhookDBManager(t.TempDir()),
|
||||||
|
slog.New(slog.NewTextHandler(
|
||||||
|
os.Stderr,
|
||||||
|
&slog.HandlerOptions{Level: slog.LevelDebug},
|
||||||
|
)),
|
||||||
|
&http.Client{Timeout: 5 * time.Second},
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
|
||||||
event := seedEvent(t, db, `{"db":"target"}`)
|
event := seedEvent(t, db, `{"db":"target"}`)
|
||||||
d := seedDatabaseTargetDelivery(t, db, event, tgt)
|
d := seedDatabaseTargetDelivery(t, db, event, "")
|
||||||
|
|
||||||
env.eng.ExportDeliverDatabase(db, d)
|
e.ExportDeliverDatabase(db, d)
|
||||||
|
|
||||||
var updated database.Delivery
|
var updated database.Delivery
|
||||||
|
|
||||||
@@ -1059,21 +1064,6 @@ func TestParseHTTPConfig_MissingURL(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseHTTPConfig_Undecodable(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
e := testEngine(t, 1)
|
|
||||||
|
|
||||||
_, err := e.ExportParseHTTPConfig(
|
|
||||||
`{"url":"https://example.com/hook","timeout":"soon"}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Error(t, err,
|
|
||||||
"config that does not decode should return error, "+
|
|
||||||
"even when the part that did names a URL",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestScheduleRetry_SendsToRetryChannel(
|
func TestScheduleRetry_SendsToRetryChannel(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
@@ -1257,33 +1247,11 @@ func TestDoHTTPRequest_ForwardsHeaders(t *testing.T) {
|
|||||||
testContentType,
|
testContentType,
|
||||||
receivedHeaders.Get("Content-Type"),
|
receivedHeaders.Get("Content-Type"),
|
||||||
)
|
)
|
||||||
}
|
|
||||||
|
|
||||||
// A response that ends before the length it announced is an error, not
|
assert.Equal(t,
|
||||||
// a short body.
|
"webhooker/1.0",
|
||||||
func TestDoHTTPRequest_CutShortResponseIsAnError(t *testing.T) {
|
receivedHeaders.Get("User-Agent"),
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
ts := httptest.NewServer(
|
|
||||||
http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.Header().Set("Content-Length", "100")
|
|
||||||
_, _ = w.Write([]byte("cut short"))
|
|
||||||
},
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
e := testEngine(t, 1)
|
|
||||||
|
|
||||||
_, body, _, err := e.ExportDoHTTPRequest(
|
|
||||||
context.TODO(),
|
|
||||||
&delivery.HTTPTargetConfig{URL: ts.URL},
|
|
||||||
&database.Event{},
|
|
||||||
)
|
|
||||||
|
|
||||||
require.ErrorIs(t, err, io.ErrUnexpectedEOF)
|
|
||||||
assert.Empty(t, body)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// The event's stored inbound headers carry the same Content-Type the
|
// The event's stored inbound headers carry the same Content-Type the
|
||||||
@@ -1352,7 +1320,6 @@ func TestApplyRequestHeaders_SendsOneContentType(t *testing.T) {
|
|||||||
ContentType: tc.event,
|
ContentType: tc.event,
|
||||||
},
|
},
|
||||||
cfg,
|
cfg,
|
||||||
"webhooker/dev",
|
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Equal(t,
|
assert.Equal(t,
|
||||||
@@ -1362,34 +1329,6 @@ func TestApplyRequestHeaders_SendsOneContentType(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stored inbound headers that do not decode forward nothing, not the
|
|
||||||
// part of them that happened to decode.
|
|
||||||
func TestApplyRequestHeaders_UndecodableInboundForwardsNothing(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(
|
|
||||||
context.Background(),
|
|
||||||
http.MethodPost,
|
|
||||||
"https://target.example.com/hook",
|
|
||||||
http.NoBody,
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
names := delivery.ExportApplyRequestHeaders(
|
|
||||||
req,
|
|
||||||
&database.Event{
|
|
||||||
Headers: `{"X-Custom":["value1"],"X-Broken":"not a list"}`,
|
|
||||||
},
|
|
||||||
&delivery.HTTPTargetConfig{},
|
|
||||||
"webhooker/dev",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Empty(t, names)
|
|
||||||
assert.Empty(t, req.Header.Get("X-Custom"))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProcessDelivery_RoutesToCorrectHandler(
|
func TestProcessDelivery_RoutesToCorrectHandler(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
@@ -1397,27 +1336,32 @@ func TestProcessDelivery_RoutesToCorrectHandler(
|
|||||||
|
|
||||||
db := testWebhookDB(t)
|
db := testWebhookDB(t)
|
||||||
|
|
||||||
// The database target archives for real, so the engine needs
|
// The database target archives for real now, so the engine
|
||||||
// the target in the main database and a data directory.
|
// needs a webhook DB manager to locate the data directory.
|
||||||
env := setupArchiveTest(t)
|
e := delivery.NewTestEngineWithDB(
|
||||||
archive := env.seedDatabaseTarget(t, "")
|
nil,
|
||||||
|
database.NewTestWebhookDBManager(t.TempDir()),
|
||||||
|
slog.New(slog.NewTextHandler(
|
||||||
|
os.Stderr,
|
||||||
|
&slog.HandlerOptions{Level: slog.LevelDebug},
|
||||||
|
)),
|
||||||
|
&http.Client{Timeout: 5 * time.Second},
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
targetType database.TargetType
|
targetType database.TargetType
|
||||||
targetID string
|
|
||||||
wantStatus database.DeliveryStatus
|
wantStatus database.DeliveryStatus
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
"database target",
|
"database target",
|
||||||
database.TargetTypeDatabase,
|
database.TargetTypeDatabase,
|
||||||
archive.ID,
|
|
||||||
database.DeliveryStatusDelivered,
|
database.DeliveryStatusDelivered,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"log target",
|
"log target",
|
||||||
database.TargetTypeLog,
|
database.TargetTypeLog,
|
||||||
uuid.New().String(),
|
|
||||||
database.DeliveryStatusDelivered,
|
database.DeliveryStatusDelivered,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -1427,7 +1371,7 @@ func TestProcessDelivery_RoutesToCorrectHandler(
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
runRoutingSubtest(
|
runRoutingSubtest(
|
||||||
t, db, env.eng, tt.targetType, tt.targetID,
|
t, db, e, tt.targetType,
|
||||||
tt.wantStatus,
|
tt.wantStatus,
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
@@ -1439,7 +1383,6 @@ func runRoutingSubtest(
|
|||||||
db *gorm.DB,
|
db *gorm.DB,
|
||||||
e *delivery.Engine,
|
e *delivery.Engine,
|
||||||
targetType database.TargetType,
|
targetType database.TargetType,
|
||||||
targetID string,
|
|
||||||
wantStatus database.DeliveryStatus,
|
wantStatus database.DeliveryStatus,
|
||||||
) {
|
) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -1447,7 +1390,8 @@ func runRoutingSubtest(
|
|||||||
event := seedEvent(t, db, `{"routing":"test"}`)
|
event := seedEvent(t, db, `{"routing":"test"}`)
|
||||||
|
|
||||||
dlv := seedDelivery(
|
dlv := seedDelivery(
|
||||||
t, db, event.ID, targetID,
|
t, db, event.ID,
|
||||||
|
uuid.New().String(),
|
||||||
database.DeliveryStatusPending,
|
database.DeliveryStatusPending,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ import (
|
|||||||
"net/url"
|
"net/url"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
@@ -41,6 +40,11 @@ const (
|
|||||||
ExportPendingSweepMinAge = pendingSweepMinAge
|
ExportPendingSweepMinAge = pendingSweepMinAge
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ExportIsBlockedIP exposes isBlockedIP for testing.
|
||||||
|
func ExportIsBlockedIP(ip net.IP) bool {
|
||||||
|
return isBlockedIP(ip)
|
||||||
|
}
|
||||||
|
|
||||||
// NewTestGuard builds an SSRF Guard from an explicit egress
|
// NewTestGuard builds an SSRF Guard from an explicit egress
|
||||||
// allowlist, without going through config. Passing no prefixes
|
// allowlist, without going through config. Passing no prefixes
|
||||||
// yields the default guard, which blocks every private/reserved
|
// yields the default guard, which blocks every private/reserved
|
||||||
@@ -66,11 +70,6 @@ func ExportBlockedNetworks() []*net.IPNet {
|
|||||||
return blockedNetworks
|
return blockedNetworks
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportBlockedPublicNetworks exposes blockedPublicNetworks.
|
|
||||||
func ExportBlockedPublicNetworks() []*net.IPNet {
|
|
||||||
return blockedPublicNetworks
|
|
||||||
}
|
|
||||||
|
|
||||||
// ExportIsForwardableHeader exposes isForwardableHeader.
|
// ExportIsForwardableHeader exposes isForwardableHeader.
|
||||||
func ExportIsForwardableHeader(name string) bool {
|
func ExportIsForwardableHeader(name string) bool {
|
||||||
return isForwardableHeader(name)
|
return isForwardableHeader(name)
|
||||||
@@ -83,9 +82,8 @@ func ExportApplyRequestHeaders(
|
|||||||
req *http.Request,
|
req *http.Request,
|
||||||
event *database.Event,
|
event *database.Event,
|
||||||
cfg *HTTPTargetConfig,
|
cfg *HTTPTargetConfig,
|
||||||
userAgent string,
|
|
||||||
) []string {
|
) []string {
|
||||||
return applyRequestHeaders(req, event, cfg, userAgent)
|
return applyRequestHeaders(req, event, cfg)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportTruncate exposes truncate for testing.
|
// ExportTruncate exposes truncate for testing.
|
||||||
@@ -401,7 +399,7 @@ func NewTestEngine(
|
|||||||
deliveryCh: make(chan Task, deliveryChannelSize),
|
deliveryCh: make(chan Task, deliveryChannelSize),
|
||||||
retryCh: make(chan Task, retryChannelSize),
|
retryCh: make(chan Task, retryChannelSize),
|
||||||
workers: workers,
|
workers: workers,
|
||||||
mtr: metrics.New(prometheus.NewRegistry()),
|
mtr: metrics.Default(),
|
||||||
}
|
}
|
||||||
e.initTargets(client)
|
e.initTargets(client)
|
||||||
|
|
||||||
@@ -416,7 +414,7 @@ func NewTestEngineSmallRetry(
|
|||||||
e := &Engine{
|
e := &Engine{
|
||||||
log: log,
|
log: log,
|
||||||
retryCh: make(chan Task, 1),
|
retryCh: make(chan Task, 1),
|
||||||
mtr: metrics.New(prometheus.NewRegistry()),
|
mtr: metrics.Default(),
|
||||||
}
|
}
|
||||||
e.initTargets(nil)
|
e.initTargets(nil)
|
||||||
|
|
||||||
@@ -439,7 +437,7 @@ func NewTestEngineWithDB(
|
|||||||
deliveryCh: make(chan Task, deliveryChannelSize),
|
deliveryCh: make(chan Task, deliveryChannelSize),
|
||||||
retryCh: make(chan Task, retryChannelSize),
|
retryCh: make(chan Task, retryChannelSize),
|
||||||
workers: workers,
|
workers: workers,
|
||||||
mtr: metrics.New(prometheus.NewRegistry()),
|
mtr: metrics.Default(),
|
||||||
}
|
}
|
||||||
e.initTargets(client)
|
e.initTargets(client)
|
||||||
|
|
||||||
@@ -447,7 +445,8 @@ func NewTestEngineWithDB(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ExportSetMetrics substitutes the engine's metric set, so a test can
|
// ExportSetMetrics substitutes the engine's metric set, so a test can
|
||||||
// assert on collectors registered on a registry it holds.
|
// assert on collectors registered on a private registry instead of
|
||||||
|
// the process-wide ones every other test is also moving.
|
||||||
func (e *Engine) ExportSetMetrics(mtr *metrics.Set) {
|
func (e *Engine) ExportSetMetrics(mtr *metrics.Set) {
|
||||||
e.mtr = mtr
|
e.mtr = mtr
|
||||||
}
|
}
|
||||||
@@ -474,7 +473,7 @@ func NewTestCircuitBreaker(
|
|||||||
type ExportArchivedEvent = archivedEvent
|
type ExportArchivedEvent = archivedEvent
|
||||||
|
|
||||||
// ExportArchiveWriter wraps an archiveWriter so black-box tests
|
// ExportArchiveWriter wraps an archiveWriter so black-box tests
|
||||||
// can exercise the archive file mechanics.
|
// can exercise the per-webhook archive file mechanics.
|
||||||
type ExportArchiveWriter struct {
|
type ExportArchiveWriter struct {
|
||||||
w *archiveWriter
|
w *archiveWriter
|
||||||
}
|
}
|
||||||
@@ -512,12 +511,6 @@ func (e *ExportArchiveWriter) Reopen(
|
|||||||
return e.w.reopen(expiry)
|
return e.w.reopen(expiry)
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetNow replaces the clock the writer measures its reopen
|
|
||||||
// debounce on.
|
|
||||||
func (e *ExportArchiveWriter) SetNow(now func() time.Time) {
|
|
||||||
e.w.now = now
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reopens reports how many times the file has been opened.
|
// Reopens reports how many times the file has been opened.
|
||||||
func (e *ExportArchiveWriter) Reopens() int {
|
func (e *ExportArchiveWriter) Reopens() int {
|
||||||
return e.w.reopens
|
return e.w.reopens
|
||||||
@@ -555,12 +548,6 @@ func (e *ExportArchiveWriter) Evict() {
|
|||||||
e.w.evict()
|
e.w.evict()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Rename gives the archive file a new name in the same directory,
|
|
||||||
// as a rename of the webhook or target does.
|
|
||||||
func (e *ExportArchiveWriter) Rename(name string) error {
|
|
||||||
return e.w.rename(name)
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleOpen reports whether the writer currently holds an open
|
// HandleOpen reports whether the writer currently holds an open
|
||||||
// archive handle.
|
// archive handle.
|
||||||
func (e *ExportArchiveWriter) HandleOpen() bool {
|
func (e *ExportArchiveWriter) HandleOpen() bool {
|
||||||
@@ -580,16 +567,16 @@ func (e *ExportArchiveWriter) Same(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ExportArchiveWriterFor returns the archive writer the registry
|
// ExportArchiveWriterFor returns the archive writer the registry
|
||||||
// currently caches for a database target, or nil when none is
|
// currently caches for a webhook, or nil when none is cached. It
|
||||||
// cached. It never creates one, so a test can hold a reference to
|
// never creates one, so a test can hold a reference to the very
|
||||||
// the very writer an eviction is about to detach.
|
// writer an eviction is about to detach.
|
||||||
func (e *Engine) ExportArchiveWriterFor(
|
func (e *Engine) ExportArchiveWriterFor(
|
||||||
targetID string,
|
webhookID string,
|
||||||
) *ExportArchiveWriter {
|
) *ExportArchiveWriter {
|
||||||
e.dbTarget.mu.Lock()
|
e.dbTarget.mu.Lock()
|
||||||
defer e.dbTarget.mu.Unlock()
|
defer e.dbTarget.mu.Unlock()
|
||||||
|
|
||||||
w, ok := e.dbTarget.writers[targetID]
|
w, ok := e.dbTarget.writers[webhookID]
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -598,26 +585,26 @@ func (e *Engine) ExportArchiveWriterFor(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ExportHasArchiveWriter reports whether the database target
|
// ExportHasArchiveWriter reports whether the database target
|
||||||
// type currently caches an archive writer for a target.
|
// currently caches an archive writer for a webhook.
|
||||||
func (e *Engine) ExportHasArchiveWriter(
|
func (e *Engine) ExportHasArchiveWriter(
|
||||||
targetID string,
|
webhookID string,
|
||||||
) bool {
|
) bool {
|
||||||
e.dbTarget.mu.Lock()
|
e.dbTarget.mu.Lock()
|
||||||
defer e.dbTarget.mu.Unlock()
|
defer e.dbTarget.mu.Unlock()
|
||||||
|
|
||||||
_, ok := e.dbTarget.writers[targetID]
|
_, ok := e.dbTarget.writers[webhookID]
|
||||||
|
|
||||||
return ok
|
return ok
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportArchiveHandleOpen reports whether the cached archive
|
// ExportArchiveHandleOpen reports whether the cached archive
|
||||||
// writer for a target holds an open database handle. It
|
// writer for a webhook holds an open database handle. It
|
||||||
// returns false when no writer is cached.
|
// returns false when no writer is cached.
|
||||||
func (e *Engine) ExportArchiveHandleOpen(
|
func (e *Engine) ExportArchiveHandleOpen(
|
||||||
targetID string,
|
webhookID string,
|
||||||
) bool {
|
) bool {
|
||||||
e.dbTarget.mu.Lock()
|
e.dbTarget.mu.Lock()
|
||||||
w, ok := e.dbTarget.writers[targetID]
|
w, ok := e.dbTarget.writers[webhookID]
|
||||||
e.dbTarget.mu.Unlock()
|
e.dbTarget.mu.Unlock()
|
||||||
|
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -631,12 +618,12 @@ func (e *Engine) ExportArchiveHandleOpen(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ExportEnsureArchiveWriter creates (if needed) and returns the
|
// ExportEnsureArchiveWriter creates (if needed) and returns the
|
||||||
// archive file path of the cached writer for a target, so a
|
// archive file path of the cached writer for a webhook, so a
|
||||||
// test can prime the registry the way a delivery would.
|
// test can prime the registry the way a delivery would.
|
||||||
func (e *Engine) ExportEnsureArchiveWriter(
|
func (e *Engine) ExportEnsureArchiveWriter(
|
||||||
targetID string,
|
webhookID string,
|
||||||
) (string, error) {
|
) (string, error) {
|
||||||
w, err := e.dbTarget.writerFor(targetID)
|
w, err := e.dbTarget.writerFor(webhookID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
@@ -644,14 +631,14 @@ func (e *Engine) ExportEnsureArchiveWriter(
|
|||||||
return w.path, nil
|
return w.path, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ExportSweepWriterFor takes a target's registry writer exactly
|
// ExportSweepWriterFor takes a webhook's registry writer exactly
|
||||||
// as the idle sweep does, reporting whether the sweep had to
|
// as the idle sweep does, reporting whether the sweep had to
|
||||||
// create the entry. It lets a test drive the registry through the
|
// create the entry. It lets a test drive the registry through the
|
||||||
// sweep's own entry point instead of choreographing goroutines.
|
// sweep's own entry point instead of choreographing goroutines.
|
||||||
func (e *Engine) ExportSweepWriterFor(
|
func (e *Engine) ExportSweepWriterFor(
|
||||||
targetID string,
|
webhookID string,
|
||||||
) (*ExportArchiveWriter, bool, error) {
|
) (*ExportArchiveWriter, bool, error) {
|
||||||
w, created, err := e.dbTarget.sweepWriterFor(targetID)
|
w, created, err := e.dbTarget.sweepWriterFor(webhookID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, false, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
@@ -662,9 +649,9 @@ func (e *Engine) ExportSweepWriterFor(
|
|||||||
// ExportReleaseSweepWriter releases a sweep-created registry entry
|
// ExportReleaseSweepWriter releases a sweep-created registry entry
|
||||||
// exactly as a finished sweep does.
|
// exactly as a finished sweep does.
|
||||||
func (e *Engine) ExportReleaseSweepWriter(
|
func (e *Engine) ExportReleaseSweepWriter(
|
||||||
targetID string, w *ExportArchiveWriter,
|
webhookID string, w *ExportArchiveWriter,
|
||||||
) {
|
) {
|
||||||
e.dbTarget.releaseSweepWriter(targetID, w.w)
|
e.dbTarget.releaseSweepWriter(webhookID, w.w)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewTestArchiveSweeper builds an ArchiveSweeper backed by the
|
// NewTestArchiveSweeper builds an ArchiveSweeper backed by the
|
||||||
|
|||||||
@@ -35,8 +35,9 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// mIsolate gives the setup's engine a metric set registered on a
|
// mIsolate gives the setup's engine a metric set registered on a
|
||||||
// registry this test holds, so its exact assertions can gather from
|
// private registry. The process-wide collectors are moved by every
|
||||||
// it.
|
// other delivery test running in parallel, so exact assertions are
|
||||||
|
// only possible against a registry this test owns.
|
||||||
func mIsolate(
|
func mIsolate(
|
||||||
t *testing.T, s iSetup,
|
t *testing.T, s iSetup,
|
||||||
) *prometheus.Registry {
|
) *prometheus.Registry {
|
||||||
|
|||||||
@@ -376,97 +376,3 @@ func TestFailedResultWriteLeavesDeliveryRecoverable(
|
|||||||
database.DeliveryStatusPending,
|
database.DeliveryStatusPending,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestFailedResultWriteWithRetriesLeavesDeliveryRecoverable is the same
|
|
||||||
// rule for a target with retries: whatever the receiver answered, the
|
|
||||||
// delivery stays pending and no retry is scheduled. The circuit breaker
|
|
||||||
// still learns the answer, because it describes the target's health,
|
|
||||||
// not the database's.
|
|
||||||
func TestFailedResultWriteWithRetriesLeavesDeliveryRecoverable(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// The "send succeeded" case starts with the breaker tripped open,
|
|
||||||
// so the delivery goes out as its probe and only a recorded
|
|
||||||
// success closes it again.
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
answer int
|
|
||||||
tripped bool
|
|
||||||
wantBreaker delivery.CircuitState
|
|
||||||
}{
|
|
||||||
{"send succeeded", http.StatusOK, true, delivery.CircuitClosed},
|
|
||||||
{"send failed", http.StatusBadGateway, false, delivery.CircuitOpen},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range tests {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, _ *http.Request) {
|
|
||||||
w.WriteHeader(tc.answer)
|
|
||||||
},
|
|
||||||
))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
event := iSeedEvent(
|
|
||||||
t, s.WebhookDB, s.WebhookID, `{"unwritable":true}`,
|
|
||||||
)
|
|
||||||
|
|
||||||
d := iSeedDelivery(
|
|
||||||
t, s.WebhookDB, event.ID, targetID,
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t,
|
|
||||||
s.WebhookDB.Exec("drop table delivery_results").Error,
|
|
||||||
)
|
|
||||||
|
|
||||||
// A single failure opens this breaker, and with no
|
|
||||||
// cooldown an open breaker lets the next delivery
|
|
||||||
// through as a probe.
|
|
||||||
cb := delivery.NewTestCircuitBreaker(1, 0)
|
|
||||||
if tc.tripped {
|
|
||||||
cb.RecordFailure()
|
|
||||||
}
|
|
||||||
|
|
||||||
s.Engine.ExportSetCircuitBreaker(targetID, cb)
|
|
||||||
|
|
||||||
full := &database.Delivery{
|
|
||||||
EventID: event.ID,
|
|
||||||
TargetID: targetID,
|
|
||||||
Status: database.DeliveryStatusPending,
|
|
||||||
Event: event,
|
|
||||||
Target: database.Target{
|
|
||||||
Name: "unwritable",
|
|
||||||
Type: database.TargetTypeHTTP,
|
|
||||||
Config: iHTTPConfig(ts.URL),
|
|
||||||
MaxRetries: 3,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
full.ID = d.ID
|
|
||||||
|
|
||||||
sched := &recordingScheduler{}
|
|
||||||
|
|
||||||
s.Engine.ExportDeliverHTTPWithScheduler(
|
|
||||||
context.Background(), s.WebhookDB, full,
|
|
||||||
&delivery.Task{
|
|
||||||
DeliveryID: d.ID,
|
|
||||||
TargetID: targetID,
|
|
||||||
AttemptNum: 1,
|
|
||||||
},
|
|
||||||
sched,
|
|
||||||
)
|
|
||||||
|
|
||||||
iAssertStatus(t, s.WebhookDB, d.ID, database.DeliveryStatusPending)
|
|
||||||
assert.Empty(t, sched.delays, "no retry may be scheduled")
|
|
||||||
assert.Equal(t, tc.wantBreaker, cb.State())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -375,7 +375,6 @@ func TestApplyRequestHeaders_ReportsOriginScopedNames(t *testing.T) {
|
|||||||
"Content-Type": testContentType,
|
"Content-Type": testContentType,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"webhooker/dev",
|
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Equal(t,
|
assert.Equal(t,
|
||||||
|
|||||||
+32
-104
@@ -25,64 +25,36 @@ var (
|
|||||||
errNoIPs = errors.New(
|
errNoIPs = errors.New(
|
||||||
"hostname resolved to no IP addresses",
|
"hostname resolved to no IP addresses",
|
||||||
)
|
)
|
||||||
// ErrBlockedPrivateOrReservedIP reports an address in the
|
errBlockedIP = errors.New(
|
||||||
// default blocklist's private and reserved ranges,
|
"blocked private, reserved or cloud metadata address",
|
||||||
// blockedNetworks.
|
|
||||||
ErrBlockedPrivateOrReservedIP = errors.New(
|
|
||||||
"blocked private or reserved address",
|
|
||||||
)
|
|
||||||
// errBlockedPublicMetadata reports a public address on the
|
|
||||||
// default blocklist, one in blockedPublicNetworks.
|
|
||||||
errBlockedPublicMetadata = errors.New(
|
|
||||||
"blocked cloud metadata address",
|
|
||||||
)
|
)
|
||||||
errBlockedMetadata = errors.New(
|
errBlockedMetadata = errors.New(
|
||||||
"blocked link-local, cloud instance metadata or " +
|
"blocked link-local or cloud instance metadata " +
|
||||||
"unspecified address: ALLOWED_EGRESS_CIDRS cannot open it",
|
"address: ALLOWED_EGRESS_CIDRS cannot open it",
|
||||||
)
|
)
|
||||||
errInvalidScheme = errors.New(
|
errInvalidScheme = errors.New(
|
||||||
"only http and https are allowed",
|
"only http and https are allowed",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
// blockedNetworks and blockedPublicNetworks together are the
|
// blockedNetworks is the default blocklist: the private and
|
||||||
// default blocklist: the private and reserved IP ranges, plus
|
// reserved IP ranges, plus the public cloud metadata addresses,
|
||||||
// the public cloud metadata addresses, that are blocked to
|
// that are blocked to prevent SSRF attacks. An operator can
|
||||||
// prevent SSRF attacks. An operator can permit specific blocks
|
// permit specific blocks out of this set with
|
||||||
// out of this set with ALLOWED_EGRESS_CIDRS; see Guard.
|
// ALLOWED_EGRESS_CIDRS; see Guard.
|
||||||
//
|
|
||||||
// blockedNetworks holds the private and reserved IP ranges.
|
|
||||||
//
|
//
|
||||||
//nolint:gochecknoglobals // package-level network list is appropriate here
|
//nolint:gochecknoglobals // package-level network list is appropriate here
|
||||||
var blockedNetworks []*net.IPNet
|
var blockedNetworks []*net.IPNet
|
||||||
|
|
||||||
// blockedPublicNetworks holds the default blocklist's public
|
|
||||||
// addresses, kept apart from blockedNetworks so that they are
|
|
||||||
// refused as cloud metadata addresses, never as private or
|
|
||||||
// reserved ones.
|
|
||||||
//
|
|
||||||
// A public address belongs on the default blocklist only if it
|
|
||||||
// hands credentials, user data or bootstrap material to whatever
|
|
||||||
// can reach it, without the caller presenting anything; it goes
|
|
||||||
// in this list. A provider's other public addresses are not
|
|
||||||
// refused, since reaching them can be legitimate and no list of
|
|
||||||
// them could be complete.
|
|
||||||
//
|
|
||||||
//nolint:gochecknoglobals // package-level network list is appropriate here
|
|
||||||
var blockedPublicNetworks []*net.IPNet
|
|
||||||
|
|
||||||
// alwaysBlockedNetworks are the ranges no configuration can
|
// alwaysBlockedNetworks are the ranges no configuration can
|
||||||
// open, so a supplied CIDR that covers one still leaves it
|
// open: the link-local blocks and the cloud instance metadata
|
||||||
// blocked. An entry is here for one of two reasons: it is a
|
// endpoints that live outside them. Reaching one is credential
|
||||||
// metadata endpoint (the link-local blocks and the cloud
|
// or user-data theft rather than delivery to an internal
|
||||||
// instance metadata endpoints that live outside them), or it is
|
// service, so a supplied CIDR that covers such an address still
|
||||||
// an unspecified address. Reaching a metadata endpoint is
|
// leaves it blocked.
|
||||||
// credential or user-data theft rather than delivery to an
|
|
||||||
// internal service.
|
|
||||||
//
|
//
|
||||||
// Inclusion criterion for metadata endpoints — one belongs here
|
// Inclusion criterion — an address belongs here only if BOTH
|
||||||
// only if BOTH hold, and every metadata entry below satisfies
|
// hold, and every entry below satisfies both:
|
||||||
// both:
|
|
||||||
//
|
//
|
||||||
// 1. It is a fixed address assigned by the provider, or a
|
// 1. It is a fixed address assigned by the provider, or a
|
||||||
// range reserved by IANA — never one the operator chose.
|
// range reserved by IANA — never one the operator chose.
|
||||||
@@ -93,8 +65,8 @@ var blockedPublicNetworks []*net.IPNet
|
|||||||
// not cheaply rotated.
|
// not cheaply rotated.
|
||||||
//
|
//
|
||||||
// Both halves are load-bearing, so use them to refuse a
|
// Both halves are load-bearing, so use them to refuse a
|
||||||
// metadata candidate and say why. An endpoint disclosing only
|
// candidate and say why. An endpoint disclosing only the
|
||||||
// the operator's own inventory (instance id, region, disks, NICs)
|
// operator's own inventory (instance id, region, disks, NICs)
|
||||||
// fails (2): letting a delivery target reach the operator's own
|
// fails (2): letting a delivery target reach the operator's own
|
||||||
// infrastructure is the feature ALLOWED_EGRESS_CIDRS exists to
|
// infrastructure is the feature ALLOWED_EGRESS_CIDRS exists to
|
||||||
// provide. But (2) is not "IAM credentials only" either —
|
// provide. But (2) is not "IAM credentials only" either —
|
||||||
@@ -109,21 +81,12 @@ var blockedPublicNetworks []*net.IPNet
|
|||||||
// when it clears both halves. Nothing in this list can be
|
// when it clears both halves. Nothing in this list can be
|
||||||
// reopened, so putting a public address here leaves the operator
|
// reopened, so putting a public address here leaves the operator
|
||||||
// no escape hatch at all — the condition ALLOWED_EGRESS_CIDRS
|
// no escape hatch at all — the condition ALLOWED_EGRESS_CIDRS
|
||||||
// exists to remove. Default-block it in blockedPublicNetworks
|
// exists to remove. Default-block it in blockedNetworks instead,
|
||||||
// instead, which an allowlist can override.
|
// which an allowlist can override.
|
||||||
//
|
//
|
||||||
// This is a criterion, not an enumeration of every metadata
|
// This is a criterion, not an enumeration of every metadata
|
||||||
// address in existence.
|
// address in existence.
|
||||||
//
|
//
|
||||||
// The unspecified addresses 0.0.0.0 and :: are here for a
|
|
||||||
// separate reason: they disclose nothing, but no host can have
|
|
||||||
// either, and on Linux a connection to one reaches this host's
|
|
||||||
// own loopback. Listing them means an allowlist reaches loopback
|
|
||||||
// only through an entry that covers a loopback address
|
|
||||||
// (127.0.0.0/8, ::1/128, 0.0.0.0/0), never through one that
|
|
||||||
// covers only 0.0.0.0 or :: (0.0.0.0/8, for example). Nothing
|
|
||||||
// else lives at either address, so refusing them costs nothing.
|
|
||||||
//
|
|
||||||
// Every entry is either already in blockedNetworks — this list is
|
// Every entry is either already in blockedNetworks — this list is
|
||||||
// what makes it unconditional — or an alternate encoding of
|
// what makes it unconditional — or an alternate encoding of
|
||||||
// 169.254.169.254 that Contains does not match against
|
// 169.254.169.254 that Contains does not match against
|
||||||
@@ -143,49 +106,23 @@ var alwaysBlockedNetworks []*net.IPNet
|
|||||||
//nolint:gochecknoinits // init is the idiomatic way to parse CIDRs once at startup
|
//nolint:gochecknoinits // init is the idiomatic way to parse CIDRs once at startup
|
||||||
func init() {
|
func init() {
|
||||||
blockedNetworks = mustParseCIDRs([]string{
|
blockedNetworks = mustParseCIDRs([]string{
|
||||||
// IPv4 loopback.
|
|
||||||
"127.0.0.0/8",
|
"127.0.0.0/8",
|
||||||
// RFC 1918 private network.
|
|
||||||
"10.0.0.0/8",
|
"10.0.0.0/8",
|
||||||
// RFC 1918 private network.
|
|
||||||
"172.16.0.0/12",
|
"172.16.0.0/12",
|
||||||
// RFC 1918 private network.
|
|
||||||
"192.168.0.0/16",
|
"192.168.0.0/16",
|
||||||
// IPv4 link-local.
|
|
||||||
"169.254.0.0/16",
|
"169.254.0.0/16",
|
||||||
// "This network", holding the IPv4 unspecified address 0.0.0.0.
|
|
||||||
"0.0.0.0/8",
|
"0.0.0.0/8",
|
||||||
// Carrier-grade NAT shared address space.
|
|
||||||
"100.64.0.0/10",
|
"100.64.0.0/10",
|
||||||
// IETF protocol assignments.
|
|
||||||
"192.0.0.0/24",
|
"192.0.0.0/24",
|
||||||
// IPv4 documentation (TEST-NET-1).
|
|
||||||
"192.0.2.0/24",
|
"192.0.2.0/24",
|
||||||
// Benchmarking.
|
|
||||||
"198.18.0.0/15",
|
"198.18.0.0/15",
|
||||||
// IPv4 documentation (TEST-NET-2).
|
|
||||||
"198.51.100.0/24",
|
"198.51.100.0/24",
|
||||||
// IPv4 documentation (TEST-NET-3).
|
|
||||||
"203.0.113.0/24",
|
"203.0.113.0/24",
|
||||||
// IPv4 multicast.
|
|
||||||
"224.0.0.0/4",
|
"224.0.0.0/4",
|
||||||
// Reserved, including the broadcast address.
|
|
||||||
"240.0.0.0/4",
|
"240.0.0.0/4",
|
||||||
// IPv6 loopback.
|
|
||||||
"::1/128",
|
"::1/128",
|
||||||
// IPv6 unspecified address.
|
|
||||||
"::/128",
|
|
||||||
// IPv6 unique local addresses.
|
|
||||||
"fc00::/7",
|
"fc00::/7",
|
||||||
// IPv6 link-local.
|
|
||||||
"fe80::/10",
|
"fe80::/10",
|
||||||
// IPv6 multicast.
|
|
||||||
"ff00::/8",
|
|
||||||
// IPv6 documentation.
|
|
||||||
"2001:db8::/32",
|
|
||||||
})
|
|
||||||
|
|
||||||
blockedPublicNetworks = mustParseCIDRs([]string{
|
|
||||||
// Azure WireServer, a public address that serves VM credentials.
|
// Azure WireServer, a public address that serves VM credentials.
|
||||||
"168.63.129.16/32",
|
"168.63.129.16/32",
|
||||||
})
|
})
|
||||||
@@ -242,14 +179,6 @@ func init() {
|
|||||||
// allowlist from opening it.
|
// allowlist from opening it.
|
||||||
"192.0.0.192/32",
|
"192.0.0.192/32",
|
||||||
|
|
||||||
// The unspecified addresses, each of which reaches this
|
|
||||||
// host's loopback on Linux.
|
|
||||||
//
|
|
||||||
// IPv4 unspecified address, inside the blocked 0.0.0.0/8.
|
|
||||||
"0.0.0.0/32",
|
|
||||||
// IPv6 unspecified address.
|
|
||||||
"::/128",
|
|
||||||
|
|
||||||
// 169.254.169.254 as an IPv4-compatible IPv6 address.
|
// 169.254.169.254 as an IPv4-compatible IPv6 address.
|
||||||
"::a9fe:a9fe/128",
|
"::a9fe:a9fe/128",
|
||||||
// 169.254.169.254 behind the NAT64 well-known prefix.
|
// 169.254.169.254 behind the NAT64 well-known prefix.
|
||||||
@@ -289,6 +218,13 @@ func matchesAny(networks []*net.IPNet, ip net.IP) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// isBlockedIP checks whether an IP address falls within
|
||||||
|
// the default blocklist, before any operator allowlist is
|
||||||
|
// considered.
|
||||||
|
func isBlockedIP(ip net.IP) bool {
|
||||||
|
return matchesAny(blockedNetworks, ip)
|
||||||
|
}
|
||||||
|
|
||||||
// Guard makes every SSRF decision in the process.
|
// Guard makes every SSRF decision in the process.
|
||||||
//
|
//
|
||||||
// It holds the operator's ALLOWED_EGRESS_CIDRS allowlist and
|
// It holds the operator's ALLOWED_EGRESS_CIDRS allowlist and
|
||||||
@@ -386,12 +322,10 @@ func (g *Guard) allows(ip net.IP) bool {
|
|||||||
// The order is the policy:
|
// The order is the policy:
|
||||||
//
|
//
|
||||||
// 1. alwaysBlockedNetworks is refused before the allowlist is
|
// 1. alwaysBlockedNetworks is refused before the allowlist is
|
||||||
// consulted, so no configured CIDR reaches link-local, a
|
// consulted, so no configured CIDR reaches link-local or a
|
||||||
// cloud metadata endpoint at a non-public address, or an
|
// cloud metadata endpoint at a non-public address.
|
||||||
// unspecified address.
|
|
||||||
// 2. The allowlist is consulted next, so a listed private
|
// 2. The allowlist is consulted next, so a listed private
|
||||||
// network, or a listed public address on the default
|
// network becomes reachable.
|
||||||
// blocklist, becomes reachable.
|
|
||||||
// 3. Everything else keeps the default blocklist's answer.
|
// 3. Everything else keeps the default blocklist's answer.
|
||||||
func (g *Guard) checkIP(ip net.IP) error {
|
func (g *Guard) checkIP(ip net.IP) error {
|
||||||
if matchesAny(alwaysBlockedNetworks, ip) {
|
if matchesAny(alwaysBlockedNetworks, ip) {
|
||||||
@@ -404,15 +338,9 @@ func (g *Guard) checkIP(ip net.IP) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if matchesAny(blockedNetworks, ip) {
|
if isBlockedIP(ip) {
|
||||||
return fmt.Errorf(
|
return fmt.Errorf(
|
||||||
"target IP %s: %w", ip, ErrBlockedPrivateOrReservedIP,
|
"target IP %s: %w", ip, errBlockedIP,
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
if matchesAny(blockedPublicNetworks, ip) {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"target IP %s: %w", ip, errBlockedPublicMetadata,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -23,10 +23,6 @@ const (
|
|||||||
metadataIP = "169.254.169.254"
|
metadataIP = "169.254.169.254"
|
||||||
metadataURL = "http://" + metadataIP + "/latest/meta-data/"
|
metadataURL = "http://" + metadataIP + "/latest/meta-data/"
|
||||||
|
|
||||||
// linkLocalIPv4 is the IPv4 link-local block, which holds
|
|
||||||
// metadataIP.
|
|
||||||
linkLocalIPv4 = "169.254.0.0/16"
|
|
||||||
|
|
||||||
// loopbackHookURL is a target on this host: blocked by
|
// loopbackHookURL is a target on this host: blocked by
|
||||||
// default, reachable only once an operator allowlists
|
// default, reachable only once an operator allowlists
|
||||||
// loopback.
|
// loopback.
|
||||||
@@ -168,13 +164,12 @@ func TestGuardAllowlist_UnlistedPrivateStillRefused(t *testing.T) {
|
|||||||
|
|
||||||
// TestGuardAllowlist_MetadataAlwaysRefused is the load-bearing
|
// TestGuardAllowlist_MetadataAlwaysRefused is the load-bearing
|
||||||
// case: cloud instance metadata endpoints are credential theft
|
// case: cloud instance metadata endpoints are credential theft
|
||||||
// rather than delivery to an internal service, and the
|
// rather than delivery to an internal service, so no allowlist
|
||||||
// unspecified addresses 0.0.0.0 and :: reach this host's loopback
|
// reaches one. Every guard below names a CIDR that covers its
|
||||||
// on Linux, so no allowlist reaches any of them. Every guard
|
// target — including 0.0.0.0/0, ::/0, and the ordinary ULA and
|
||||||
// below names a CIDR that covers its target — including
|
// CGNAT blocks an operator would really list — and the address
|
||||||
// 0.0.0.0/0, ::/0, and the ordinary ULA and CGNAT blocks an
|
// must stay refused anyway, on both the validation and the
|
||||||
// operator would really list — and the address must stay
|
// delivery path.
|
||||||
// refused anyway, on both the validation and the delivery path.
|
|
||||||
func TestGuardAllowlist_MetadataAlwaysRefused(t *testing.T) {
|
func TestGuardAllowlist_MetadataAlwaysRefused(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -220,17 +215,15 @@ type metadataAlwaysRefusedCase struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// metadataAlwaysRefusedCases enumerates every unconditionally
|
// metadataAlwaysRefusedCases enumerates every unconditionally
|
||||||
// blocked address (link-local, the cloud metadata endpoints and
|
// blocked address together with an allowlist entry that would
|
||||||
// the unspecified addresses) together with an allowlist entry
|
// otherwise reach it. Split by family of address only to stay
|
||||||
// that would otherwise reach it. Split by family of address only
|
// under the function-length limit.
|
||||||
// to stay under the function-length limit.
|
|
||||||
func metadataAlwaysRefusedCases() []metadataAlwaysRefusedCase {
|
func metadataAlwaysRefusedCases() []metadataAlwaysRefusedCase {
|
||||||
cases := linkLocalRefusedCases()
|
cases := linkLocalRefusedCases()
|
||||||
cases = append(cases, ulaMetadataRefusedCases()...)
|
cases = append(cases, ulaMetadataRefusedCases()...)
|
||||||
cases = append(cases, ipv4MetadataRefusedCases()...)
|
cases = append(cases, ipv4MetadataRefusedCases()...)
|
||||||
cases = append(cases, encodedMetadataRefusedCases()...)
|
|
||||||
|
|
||||||
return append(cases, unspecifiedRefusedCases()...)
|
return append(cases, encodedMetadataRefusedCases()...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// linkLocalRefusedCases covers the link-local blocks, including
|
// linkLocalRefusedCases covers the link-local blocks, including
|
||||||
@@ -244,7 +237,7 @@ func linkLocalRefusedCases() []metadataAlwaysRefusedCase {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "whole link-local block",
|
name: "whole link-local block",
|
||||||
allow: linkLocalIPv4,
|
allow: "169.254.0.0/16",
|
||||||
target: metadataURL,
|
target: metadataURL,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -370,23 +363,6 @@ func encodedMetadataRefusedCases() []metadataAlwaysRefusedCase {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// unspecifiedRefusedCases covers the unspecified addresses, each
|
|
||||||
// of which reaches this host's loopback on Linux.
|
|
||||||
func unspecifiedRefusedCases() []metadataAlwaysRefusedCase {
|
|
||||||
return []metadataAlwaysRefusedCase{
|
|
||||||
{
|
|
||||||
name: "IPv4 unspecified address under 0.0.0.0/0",
|
|
||||||
allow: allowAllIPv4,
|
|
||||||
target: "http://0.0.0.0:8080/hook",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "IPv6 unspecified address under ::/0",
|
|
||||||
allow: allowAllIPv6,
|
|
||||||
target: "http://[::]:8080/hook",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestGuardAllowlist_PublicUnaffected asserts the allowlist does
|
// TestGuardAllowlist_PublicUnaffected asserts the allowlist does
|
||||||
// not narrow anything: public addresses were reachable before it
|
// not narrow anything: public addresses were reachable before it
|
||||||
// existed and stay reachable, whether or not a list is set.
|
// existed and stay reachable, whether or not a list is set.
|
||||||
@@ -436,9 +412,6 @@ func TestGuardAllowlist_AzureWireServerReopenable(t *testing.T) {
|
|||||||
"WireServer must be refused by the default blocklist, "+
|
"WireServer must be refused by the default blocklist, "+
|
||||||
"which an allowlist can override",
|
"which an allowlist can override",
|
||||||
)
|
)
|
||||||
require.NotErrorIs(t, err, delivery.ErrBlockedPrivateOrReservedIP,
|
|
||||||
"WireServer is public, not private or reserved",
|
|
||||||
)
|
|
||||||
|
|
||||||
assertDialRefused(t, defaultGuard, target)
|
assertDialRefused(t, defaultGuard, target)
|
||||||
|
|
||||||
@@ -523,7 +496,7 @@ func TestAlwaysBlockedNetworks_PinnedSet(t *testing.T) {
|
|||||||
want := []string{
|
want := []string{
|
||||||
// IPv4 link-local: the 169.254.169.254 metadata
|
// IPv4 link-local: the 169.254.169.254 metadata
|
||||||
// service on AWS, Azure and others.
|
// service on AWS, Azure and others.
|
||||||
linkLocalIPv4,
|
"169.254.0.0/16",
|
||||||
// IPv6 link-local.
|
// IPv6 link-local.
|
||||||
"fe80::/10",
|
"fe80::/10",
|
||||||
// AWS IPv6 IMDS, inside the ULA space an operator may
|
// AWS IPv6 IMDS, inside the ULA space an operator may
|
||||||
@@ -544,10 +517,6 @@ func TestAlwaysBlockedNetworks_PinnedSet(t *testing.T) {
|
|||||||
// Oracle Cloud Classic metadata, inside the blocked
|
// Oracle Cloud Classic metadata, inside the blocked
|
||||||
// 192.0.0.0/24.
|
// 192.0.0.0/24.
|
||||||
"192.0.0.192/32",
|
"192.0.0.192/32",
|
||||||
// The IPv4 and IPv6 unspecified addresses, each of
|
|
||||||
// which reaches this host's loopback on Linux.
|
|
||||||
"0.0.0.0/32",
|
|
||||||
"::/128",
|
|
||||||
// 169.254.169.254 as an IPv4-compatible IPv6 address.
|
// 169.254.169.254 as an IPv4-compatible IPv6 address.
|
||||||
"::a9fe:a9fe/128",
|
"::a9fe:a9fe/128",
|
||||||
// 169.254.169.254 behind the NAT64 well-known prefix.
|
// 169.254.169.254 behind the NAT64 well-known prefix.
|
||||||
@@ -557,94 +526,6 @@ func TestAlwaysBlockedNetworks_PinnedSet(t *testing.T) {
|
|||||||
assert.Equal(t, want, got)
|
assert.Equal(t, want, got)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestDefaultBlocklist_PinnedSet pins each list of the default
|
|
||||||
// blocklist on its own, the private and reserved ranges in
|
|
||||||
// blockedNetworks and the public addresses in
|
|
||||||
// blockedPublicNetworks, so moving an entry from one list to the
|
|
||||||
// other fails it. For the first address of each entry it then
|
|
||||||
// checks that the default guard refuses it, and that listing the
|
|
||||||
// entry in ALLOWED_EGRESS_CIDRS opens it unless the unconditional
|
|
||||||
// set holds that address.
|
|
||||||
func TestDefaultBlocklist_PinnedSet(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// public marks an entry of blockedPublicNetworks; every other
|
|
||||||
// entry belongs in blockedNetworks.
|
|
||||||
tests := []struct {
|
|
||||||
cidr string
|
|
||||||
public bool
|
|
||||||
reopenable bool
|
|
||||||
}{
|
|
||||||
{cidr: "127.0.0.0/8", reopenable: true},
|
|
||||||
{cidr: "10.0.0.0/8", reopenable: true},
|
|
||||||
{cidr: "172.16.0.0/12", reopenable: true},
|
|
||||||
{cidr: "192.168.0.0/16", reopenable: true},
|
|
||||||
{cidr: linkLocalIPv4, reopenable: false},
|
|
||||||
// Its first address, 0.0.0.0, is in the unconditional set.
|
|
||||||
{cidr: "0.0.0.0/8", reopenable: false},
|
|
||||||
{cidr: "100.64.0.0/10", reopenable: true},
|
|
||||||
{cidr: "192.0.0.0/24", reopenable: true},
|
|
||||||
{cidr: "192.0.2.0/24", reopenable: true},
|
|
||||||
{cidr: "198.18.0.0/15", reopenable: true},
|
|
||||||
{cidr: "198.51.100.0/24", reopenable: true},
|
|
||||||
{cidr: "203.0.113.0/24", reopenable: true},
|
|
||||||
{cidr: "224.0.0.0/4", reopenable: true},
|
|
||||||
{cidr: "240.0.0.0/4", reopenable: true},
|
|
||||||
{cidr: "::1/128", reopenable: true},
|
|
||||||
{cidr: "::/128", reopenable: false},
|
|
||||||
{cidr: "fc00::/7", reopenable: true},
|
|
||||||
{cidr: "fe80::/10", reopenable: false},
|
|
||||||
{cidr: "ff00::/8", reopenable: true},
|
|
||||||
{cidr: "2001:db8::/32", reopenable: true},
|
|
||||||
{cidr: "168.63.129.16/32", public: true, reopenable: true},
|
|
||||||
}
|
|
||||||
|
|
||||||
wantPrivate := make([]string, 0, len(tests))
|
|
||||||
wantPublic := make([]string, 0, len(tests))
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
if tt.public {
|
|
||||||
wantPublic = append(wantPublic, tt.cidr)
|
|
||||||
} else {
|
|
||||||
wantPrivate = append(wantPrivate, tt.cidr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
gotPrivate := make([]string, 0, len(tests))
|
|
||||||
for _, n := range delivery.ExportBlockedNetworks() {
|
|
||||||
gotPrivate = append(gotPrivate, n.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
gotPublic := make([]string, 0, len(tests))
|
|
||||||
for _, n := range delivery.ExportBlockedPublicNetworks() {
|
|
||||||
gotPublic = append(gotPublic, n.String())
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.ElementsMatch(t, wantPrivate, gotPrivate, "blockedNetworks")
|
|
||||||
assert.ElementsMatch(t, wantPublic, gotPublic, "blockedPublicNetworks")
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.cidr, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
prefix := netip.MustParsePrefix(tt.cidr)
|
|
||||||
ip := net.IP(prefix.Addr().AsSlice())
|
|
||||||
|
|
||||||
require.Error(t,
|
|
||||||
delivery.NewTestGuard().ExportCheckIP(ip),
|
|
||||||
"the default guard must refuse %s", ip,
|
|
||||||
)
|
|
||||||
|
|
||||||
err := delivery.NewTestGuard(prefix).ExportCheckIP(ip)
|
|
||||||
if tt.reopenable {
|
|
||||||
assert.NoError(t, err, "listing %s must open it", tt.cidr)
|
|
||||||
} else {
|
|
||||||
assert.Error(t, err, "listing %s must not open it", tt.cidr)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// requireLoopback fails the test unless rawURL's host is a
|
// requireLoopback fails the test unless rawURL's host is a
|
||||||
// loopback address, so the allowlist test cannot silently stop
|
// loopback address, so the allowlist test cannot silently stop
|
||||||
// exercising a blocked range.
|
// exercising a blocked range.
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestGuardCheckIP_PrivateRanges(t *testing.T) {
|
func TestIsBlockedIP_PrivateRanges(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
@@ -56,14 +56,12 @@ func TestGuardCheckIP_PrivateRanges(t *testing.T) {
|
|||||||
"failed to parse IP %s", tt.ip,
|
"failed to parse IP %s", tt.ip,
|
||||||
)
|
)
|
||||||
|
|
||||||
refused := delivery.NewTestGuard().ExportCheckIP(ip) != nil
|
|
||||||
|
|
||||||
assert.Equal(t,
|
assert.Equal(t,
|
||||||
tt.blocked,
|
tt.blocked,
|
||||||
refused,
|
delivery.ExportIsBlockedIP(ip),
|
||||||
"default guard refuses %s = %v, want %v",
|
"isBlockedIP(%s) = %v, want %v",
|
||||||
tt.ip,
|
tt.ip,
|
||||||
refused,
|
delivery.ExportIsBlockedIP(ip),
|
||||||
tt.blocked,
|
tt.blocked,
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
@@ -101,42 +99,6 @@ func TestValidateTargetURL_Blocked(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestDefaultGuard_RefusesUnspecifiedMulticastAndDocumentation
|
|
||||||
// covers the unspecified addresses and the IPv6 multicast and
|
|
||||||
// documentation ranges: with no allowlist set, each is refused
|
|
||||||
// both when a target is created and when a delivery dials it.
|
|
||||||
func TestDefaultGuard_RefusesUnspecifiedMulticastAndDocumentation(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
guard := delivery.NewTestGuard()
|
|
||||||
|
|
||||||
targets := []string{
|
|
||||||
// The unspecified addresses. On Linux a connection to
|
|
||||||
// either reaches this host's loopback.
|
|
||||||
"http://0.0.0.0:8080/hook",
|
|
||||||
"http://[::]:8080/hook",
|
|
||||||
// IPv6 multicast, all nodes.
|
|
||||||
"http://[ff02::1]/hook",
|
|
||||||
// IPv6 documentation.
|
|
||||||
"http://[2001:db8::1]/hook",
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, target := range targets {
|
|
||||||
t.Run(target, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
require.Error(t,
|
|
||||||
guard.ValidateTargetURL(context.Background(), target),
|
|
||||||
"%s must be refused at target creation", target,
|
|
||||||
)
|
|
||||||
|
|
||||||
assertDialRefused(t, guard, target)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestValidateTargetURL_Allowed(t *testing.T) {
|
func TestValidateTargetURL_Allowed(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ package delivery
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"path/filepath"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -11,75 +11,22 @@ import (
|
|||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
)
|
)
|
||||||
|
|
||||||
// archiveNameMaxLen is how many characters of a webhook or target
|
// databaseTarget is a no-retry target that archives the
|
||||||
// name an archive file name keeps.
|
// full inbound event into a per-webhook archive SQLite file,
|
||||||
const archiveNameMaxLen = 40
|
// separate from the per-webhook event database. The event is
|
||||||
|
// already persisted in the per-webhook event DB by the time
|
||||||
// databaseTarget is a no-retry target that archives the full
|
// delivery runs; the database target additionally writes a
|
||||||
// inbound event into the target's own archive SQLite file, separate
|
// durable long-term copy into archive-{webhookID}.db and then
|
||||||
// from the per-webhook event database. The event is already
|
// records a single attempt whose outcome reflects whether the
|
||||||
// persisted in the per-webhook event DB by the time delivery runs;
|
// archive write succeeded. See archiveWriter for the
|
||||||
// the database target additionally writes a durable long-term copy
|
// close/reopen, auto-recreate, and expiry semantics.
|
||||||
// into the file ArchiveFileName names and then records a single
|
|
||||||
// attempt whose outcome reflects whether the archive write
|
|
||||||
// succeeded. See archiveWriter for the close/reopen, auto-recreate,
|
|
||||||
// and expiry semantics.
|
|
||||||
type databaseTarget struct {
|
type databaseTarget struct {
|
||||||
eng *Engine
|
eng *Engine
|
||||||
|
|
||||||
// writers holds one archive writer per database target, keyed
|
|
||||||
// by target ID.
|
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
writers map[string]*archiveWriter
|
writers map[string]*archiveWriter
|
||||||
}
|
}
|
||||||
|
|
||||||
// ArchiveFileName returns the file name of a database target's
|
|
||||||
// archive: archive-WEBHOOKNAME-TARGETNAME-TARGETID.db, with both
|
|
||||||
// names passed through archiveNamePart. The target ID keeps the
|
|
||||||
// name unique when two targets' names come out the same.
|
|
||||||
func ArchiveFileName(webhookName, targetName, targetID string) string {
|
|
||||||
return "archive-" + archiveNamePart(webhookName) + "-" +
|
|
||||||
archiveNamePart(targetName) + "-" + targetID + ".db"
|
|
||||||
}
|
|
||||||
|
|
||||||
// archiveNamePart makes a webhook or target name safe to put in a
|
|
||||||
// file name. It is lowercased; ASCII letters and digits are kept,
|
|
||||||
// every other run of characters becomes a single "-", and no "-" is
|
|
||||||
// left at either end. It is cut to archiveNameMaxLen characters, and
|
|
||||||
// a name with nothing left is "unnamed".
|
|
||||||
func archiveNamePart(name string) string {
|
|
||||||
var b strings.Builder
|
|
||||||
|
|
||||||
dash := false
|
|
||||||
|
|
||||||
for _, r := range strings.ToLower(name) {
|
|
||||||
if (r < 'a' || r > 'z') && (r < '0' || r > '9') {
|
|
||||||
dash = b.Len() > 0
|
|
||||||
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if dash {
|
|
||||||
b.WriteByte('-')
|
|
||||||
|
|
||||||
dash = false
|
|
||||||
}
|
|
||||||
|
|
||||||
b.WriteRune(r)
|
|
||||||
}
|
|
||||||
|
|
||||||
part := b.String()
|
|
||||||
if len(part) > archiveNameMaxLen {
|
|
||||||
part = strings.TrimRight(part[:archiveNameMaxLen], "-")
|
|
||||||
}
|
|
||||||
|
|
||||||
if part == "" {
|
|
||||||
return "unnamed"
|
|
||||||
}
|
|
||||||
|
|
||||||
return part
|
|
||||||
}
|
|
||||||
|
|
||||||
// Deliver implements Target. It archives the event, then
|
// Deliver implements Target. It archives the event, then
|
||||||
// records one successful attempt and marks the delivery
|
// records one successful attempt and marks the delivery
|
||||||
// delivered. An archiving error fails the delivery: the
|
// delivered. An archiving error fails the delivery: the
|
||||||
@@ -145,7 +92,7 @@ func (t *databaseTarget) Deliver(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// archive writes the full event as a row into the target's
|
// archive writes the full event as a row into the webhook's
|
||||||
// archive database, honouring the optional per-target expiry
|
// archive database, honouring the optional per-target expiry
|
||||||
// parsed from the target config JSON.
|
// parsed from the target config JSON.
|
||||||
func (t *databaseTarget) archive(d *database.Delivery) error {
|
func (t *databaseTarget) archive(d *database.Delivery) error {
|
||||||
@@ -159,7 +106,7 @@ func (t *databaseTarget) archive(d *database.Delivery) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
w, err := t.writerFor(d.TargetID)
|
w, err := t.writerFor(webhookID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -177,31 +124,30 @@ func (t *databaseTarget) archive(d *database.Delivery) error {
|
|||||||
return w.write(row, expiry)
|
return w.write(row, expiry)
|
||||||
}
|
}
|
||||||
|
|
||||||
// writerFor returns the archive writer for a database target,
|
// writerFor returns the archiveWriter for a webhook, creating
|
||||||
// creating and caching it on first use. Each target has one writer
|
// and caching it on first use. Each webhook has one writer so
|
||||||
// so its close/reopen debounce state is shared across concurrent
|
// its close/reopen debounce state is shared across concurrent
|
||||||
// deliveries, and so a rename and the idle sweep take the same lock
|
// deliveries. The archive file lives beside the per-webhook
|
||||||
// as its writes.
|
// event database in the data directory.
|
||||||
func (t *databaseTarget) writerFor(
|
func (t *databaseTarget) writerFor(
|
||||||
targetID string,
|
webhookID string,
|
||||||
) (*archiveWriter, error) {
|
) (*archiveWriter, error) {
|
||||||
|
path, err := t.archivePath(webhookID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
t.mu.Lock()
|
t.mu.Lock()
|
||||||
defer t.mu.Unlock()
|
defer t.mu.Unlock()
|
||||||
|
|
||||||
w, ok := t.writers[targetID]
|
if t.writers == nil {
|
||||||
|
t.writers = make(map[string]*archiveWriter)
|
||||||
|
}
|
||||||
|
|
||||||
|
w, ok := t.writers[webhookID]
|
||||||
if !ok {
|
if !ok {
|
||||||
var err error
|
w = newArchiveWriter(path, t.eng.log)
|
||||||
|
t.writers[webhookID] = w
|
||||||
w, err = t.newWriter(targetID)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if t.writers == nil {
|
|
||||||
t.writers = make(map[string]*archiveWriter)
|
|
||||||
}
|
|
||||||
|
|
||||||
t.writers[targetID] = w
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// A delivery claims the entry: even if the idle sweep created
|
// A delivery claims the entry: even if the idle sweep created
|
||||||
@@ -213,39 +159,40 @@ func (t *databaseTarget) writerFor(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// sweepWriterFor returns the archive writer the idle sweep should
|
// sweepWriterFor returns the archive writer the idle sweep should
|
||||||
// prune a target's archive through, together with whether the sweep
|
// prune a webhook through, together with whether the sweep itself
|
||||||
// itself created the registry entry.
|
// created the registry entry.
|
||||||
//
|
//
|
||||||
// The sweep must route its prune through the registered writer so
|
// The sweep must route its prune through the registered writer so
|
||||||
// the writer's mutex orders it against concurrent writes, but it
|
// the writer's mutex orders it against concurrent writes, but it
|
||||||
// must never leave a registry entry behind: a sweep that ran
|
// must never leave a registry entry behind: a sweep that ran
|
||||||
// concurrently with the target's deletion would otherwise
|
// concurrently with the webhook's deletion would otherwise
|
||||||
// re-create an entry that nothing will ever evict again, which is
|
// re-create an entry that nothing will ever evict again, which is
|
||||||
// exactly the leak eviction exists to prevent. An entry the sweep
|
// exactly the leak eviction exists to prevent. An entry the sweep
|
||||||
// creates is therefore marked sweep-owned and handed back to
|
// creates is therefore marked sweep-owned and handed back to
|
||||||
// releaseSweepWriter when the sweep is done.
|
// releaseSweepWriter when the sweep is done.
|
||||||
func (t *databaseTarget) sweepWriterFor(
|
func (t *databaseTarget) sweepWriterFor(
|
||||||
targetID string,
|
webhookID string,
|
||||||
) (*archiveWriter, bool, error) {
|
) (*archiveWriter, bool, error) {
|
||||||
t.mu.Lock()
|
path, err := t.archivePath(webhookID)
|
||||||
defer t.mu.Unlock()
|
|
||||||
|
|
||||||
w, ok := t.writers[targetID]
|
|
||||||
if ok {
|
|
||||||
return w, false, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
w, err := t.newWriter(targetID)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, false, err
|
return nil, false, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
|
||||||
if t.writers == nil {
|
if t.writers == nil {
|
||||||
t.writers = make(map[string]*archiveWriter)
|
t.writers = make(map[string]*archiveWriter)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
w, ok := t.writers[webhookID]
|
||||||
|
if ok {
|
||||||
|
return w, false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
w = newArchiveWriter(path, t.eng.log)
|
||||||
w.sweepOwned = true
|
w.sweepOwned = true
|
||||||
t.writers[targetID] = w
|
t.writers[webhookID] = w
|
||||||
|
|
||||||
return w, true, nil
|
return w, true, nil
|
||||||
}
|
}
|
||||||
@@ -262,92 +209,57 @@ func (t *databaseTarget) sweepWriterFor(
|
|||||||
// delivery that adopted the writer keeps a registered, evictable
|
// delivery that adopted the writer keeps a registered, evictable
|
||||||
// one.
|
// one.
|
||||||
func (t *databaseTarget) releaseSweepWriter(
|
func (t *databaseTarget) releaseSweepWriter(
|
||||||
targetID string, w *archiveWriter,
|
webhookID string, w *archiveWriter,
|
||||||
) {
|
) {
|
||||||
t.mu.Lock()
|
t.mu.Lock()
|
||||||
defer t.mu.Unlock()
|
defer t.mu.Unlock()
|
||||||
|
|
||||||
cur, ok := t.writers[targetID]
|
cur, ok := t.writers[webhookID]
|
||||||
if !ok || cur != w || !cur.sweepOwned {
|
if !ok || cur != w || !cur.sweepOwned {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
delete(t.writers, targetID)
|
delete(t.writers, webhookID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// newWriter builds the writer for a database target's archive. The
|
// archivePath returns the archive file path for a webhook: it
|
||||||
// file is the one ArchivePath gives for the webhook and the target as
|
// lives beside the per-webhook event database in the data
|
||||||
// the main database names them now; from then on only rename changes
|
// directory. It does not touch the filesystem.
|
||||||
// the name the writer uses. It does not touch the archive file.
|
func (t *databaseTarget) archivePath(
|
||||||
func (t *databaseTarget) newWriter(
|
webhookID string,
|
||||||
targetID string,
|
) (string, error) {
|
||||||
) (*archiveWriter, error) {
|
|
||||||
if t.eng.dbManager == nil {
|
if t.eng.dbManager == nil {
|
||||||
return nil, errArchiveNoDataDir
|
return "", errArchiveNoDataDir
|
||||||
}
|
}
|
||||||
|
|
||||||
var target database.Target
|
dir := filepath.Dir(t.eng.dbManager.DBPath(webhookID))
|
||||||
|
|
||||||
err := t.eng.database.DB().
|
return filepath.Join(
|
||||||
Preload("Webhook").
|
dir, fmt.Sprintf("archive-%s.db", webhookID),
|
||||||
First(&target, "id = ?", targetID).Error
|
), nil
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"loading database target %s: %w", targetID, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
w := newArchiveWriter(
|
|
||||||
ArchivePath(t.eng.dbManager, &target.Webhook, &target),
|
|
||||||
t.eng.log,
|
|
||||||
)
|
|
||||||
w.webhookID = target.WebhookID
|
|
||||||
|
|
||||||
return w, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// rename moves a database target's archive file to the name for
|
// evict drops a webhook's archive writer from the registry and
|
||||||
// webhookName and targetName. It goes through the target's writer,
|
// closes its handle, so a deleted webhook does not leave a
|
||||||
// so the move holds the lock that writes and the idle sweep take,
|
// writer (and an open archive handle within its debounce
|
||||||
// and later writes use the new name.
|
// window) alive for the process lifetime.
|
||||||
//
|
|
||||||
// The writer is created if there is none, and it stays cached. The
|
|
||||||
// handlers rename before they save the new name, so until the save
|
|
||||||
// the main database still has the old one; a delivery in that window
|
|
||||||
// must find this writer rather than build one from the old name.
|
|
||||||
func (t *databaseTarget) rename(
|
|
||||||
targetID, webhookName, targetName string,
|
|
||||||
) error {
|
|
||||||
w, err := t.writerFor(targetID)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return w.rename(ArchiveFileName(webhookName, targetName, targetID))
|
|
||||||
}
|
|
||||||
|
|
||||||
// evict drops a database target's archive writer from the registry
|
|
||||||
// and closes its handle, so a deleted target does not leave a
|
|
||||||
// writer (and an open archive handle within its debounce window)
|
|
||||||
// alive for the process lifetime.
|
|
||||||
//
|
//
|
||||||
// The map entry is removed under the registry lock, which is
|
// The map entry is removed under the registry lock, which is
|
||||||
// then released before the handle is closed under the writer's
|
// then released before the handle is closed under the writer's
|
||||||
// own lock: that ordering keeps the registry available to other
|
// own lock: that ordering keeps the registry available to other
|
||||||
// targets while an in-flight write on this one drains, and
|
// webhooks while an in-flight write on this one drains, and
|
||||||
// closing under the writer's lock means eviction can never race
|
// closing under the writer's lock means eviction can never race
|
||||||
// a write.
|
// a write.
|
||||||
//
|
//
|
||||||
// Eviction is idempotent and silent for a target with no writer,
|
// Eviction is idempotent and silent for a webhook with no
|
||||||
// which is the common case: only a database target that has
|
// writer, which is the common case: a webhook with no database
|
||||||
// received an event or been renamed has one. It never deletes the
|
// target never creates one. It never deletes the archive file.
|
||||||
// archive file.
|
func (t *databaseTarget) evict(webhookID string) {
|
||||||
func (t *databaseTarget) evict(targetID string) {
|
|
||||||
t.mu.Lock()
|
t.mu.Lock()
|
||||||
|
|
||||||
w, ok := t.writers[targetID]
|
w, ok := t.writers[webhookID]
|
||||||
if ok {
|
if ok {
|
||||||
delete(t.writers, targetID)
|
delete(t.writers, webhookID)
|
||||||
}
|
}
|
||||||
|
|
||||||
t.mu.Unlock()
|
t.mu.Unlock()
|
||||||
@@ -360,41 +272,13 @@ func (t *databaseTarget) evict(targetID string) {
|
|||||||
|
|
||||||
t.eng.log.Info(
|
t.eng.log.Info(
|
||||||
"evicted archive writer",
|
"evicted archive writer",
|
||||||
"target_id", targetID,
|
"webhook_id", webhookID,
|
||||||
"path", w.path,
|
"path", w.path,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// evictWebhook evicts, exactly as evict does, the writer of every
|
|
||||||
// database target of a webhook.
|
|
||||||
func (t *databaseTarget) evictWebhook(webhookID string) {
|
|
||||||
t.mu.Lock()
|
|
||||||
|
|
||||||
var gone []*archiveWriter
|
|
||||||
|
|
||||||
for targetID, w := range t.writers {
|
|
||||||
if w.webhookID == webhookID {
|
|
||||||
delete(t.writers, targetID)
|
|
||||||
|
|
||||||
gone = append(gone, w)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
t.mu.Unlock()
|
|
||||||
|
|
||||||
for _, w := range gone {
|
|
||||||
w.evict()
|
|
||||||
|
|
||||||
t.eng.log.Info(
|
|
||||||
"evicted archive writer",
|
|
||||||
"webhook_id", webhookID,
|
|
||||||
"path", w.path,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// evictAll evicts every cached archive writer, exactly as evict
|
// evictAll evicts every cached archive writer, exactly as evict
|
||||||
// does for one target. The engine calls it at shutdown, once its
|
// does for one webhook. The engine calls it at shutdown, once its
|
||||||
// workers have returned. Closing the last handle on an archive
|
// workers have returned. Closing the last handle on an archive
|
||||||
// moves the contents of its -wal into the .db and removes the
|
// moves the contents of its -wal into the .db and removes the
|
||||||
// -wal, so a clean stop leaves each archive as a single file.
|
// -wal, so a clean stop leaves each archive as a single file.
|
||||||
@@ -411,25 +295,38 @@ func (t *databaseTarget) evictAll() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// sweepArchive prunes one database target's archive of rows older
|
// sweepWebhook prunes one webhook's archive of rows older than
|
||||||
// than expiry, without requiring a write. A missing archive file is
|
// expiry, without requiring a write. It returns nil (nothing to
|
||||||
// left missing (see sweepExpired), so a sweep never creates an
|
// do) when the archive file does not exist, so a sweep never
|
||||||
// archive for a target that has never received an event.
|
// creates an archive for a webhook that has a database target
|
||||||
|
// but has never received an event.
|
||||||
//
|
//
|
||||||
// It also never leaves a registry entry behind: an entry it had
|
// It also never leaves a registry entry behind: an entry it had
|
||||||
// to create to reach the writer's mutex is released again once
|
// to create to reach the writer's mutex is released again once
|
||||||
// the prune is done, so a sweep racing a target deletion cannot
|
// the prune is done, so a sweep racing a webhook deletion cannot
|
||||||
// resurrect the writer the eviction just dropped.
|
// resurrect the writer the eviction just dropped.
|
||||||
func (t *databaseTarget) sweepArchive(
|
func (t *databaseTarget) sweepWebhook(
|
||||||
targetID string, expiry time.Duration,
|
webhookID string, expiry time.Duration,
|
||||||
) error {
|
) error {
|
||||||
w, created, err := t.sweepWriterFor(targetID)
|
path, err := t.archivePath(webhookID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check before taking a writer at all: a webhook whose
|
||||||
|
// archive has never been created gets no writer, no handle,
|
||||||
|
// and no file.
|
||||||
|
if !fileExists(path) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
w, created, err := t.sweepWriterFor(webhookID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if created {
|
if created {
|
||||||
defer t.releaseSweepWriter(targetID, w)
|
defer t.releaseSweepWriter(webhookID, w)
|
||||||
}
|
}
|
||||||
|
|
||||||
return w.sweepExpired(expiry)
|
return w.sweepExpired(expiry)
|
||||||
|
|||||||
@@ -4,10 +4,8 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/fs"
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -43,7 +41,7 @@ const (
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
// errArchiveMissingWebhookID is returned when an event to
|
// errArchiveMissingWebhookID is returned when an event to
|
||||||
// archive has no webhook id to record in its archive row.
|
// archive has no webhook id to key its archive file on.
|
||||||
errArchiveMissingWebhookID = errors.New(
|
errArchiveMissingWebhookID = errors.New(
|
||||||
"cannot archive event without a webhook id",
|
"cannot archive event without a webhook id",
|
||||||
)
|
)
|
||||||
@@ -63,19 +61,13 @@ var (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// errArchiveWriterEvicted is returned when a writer that has
|
// errArchiveWriterEvicted is returned when a writer that has
|
||||||
// been evicted (its target or its webhook was deleted) is used
|
// been evicted (its webhook was deleted, or its last database
|
||||||
// again. An evicted writer is no longer in the registry, so
|
// target was removed) is used again. An evicted writer is no
|
||||||
// reopening its file would leak a handle nothing owns.
|
// longer in the registry, so reopening its file would leak a
|
||||||
|
// handle nothing owns.
|
||||||
errArchiveWriterEvicted = errors.New(
|
errArchiveWriterEvicted = errors.New(
|
||||||
"archive writer has been evicted",
|
"archive writer has been evicted",
|
||||||
)
|
)
|
||||||
|
|
||||||
// ErrArchiveNameTaken is returned when an archive cannot be
|
|
||||||
// renamed because a file already has the new name. That file may
|
|
||||||
// be an archive with rows of its own, so it is never replaced.
|
|
||||||
ErrArchiveNameTaken = errors.New(
|
|
||||||
"a file already has the archive's new name",
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// databaseTargetConfig is the optional per-target JSON config
|
// databaseTargetConfig is the optional per-target JSON config
|
||||||
@@ -88,7 +80,7 @@ type databaseTargetConfig struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// archivedEvent is one fully captured webhook event stored in a
|
// archivedEvent is one fully captured webhook event stored in a
|
||||||
// database target's archive for long-term retention. It is a
|
// per-webhook archive database for long-term retention. It is a
|
||||||
// self-contained copy — independent of the per-webhook event
|
// self-contained copy — independent of the per-webhook event
|
||||||
// database, which may prune events under its own retention.
|
// database, which may prune events under its own retention.
|
||||||
type archivedEvent struct {
|
type archivedEvent struct {
|
||||||
@@ -178,8 +170,8 @@ func ValidateArchiveExpiry(expiry string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// archiveWriter owns one database target's archive SQLite file.
|
// archiveWriter owns one per-webhook archive SQLite file. It
|
||||||
// It serialises writes, and after each write closes and reopens
|
// serialises writes, and after each write closes and reopens
|
||||||
// the file (debounced to at most once per debounce window) so
|
// the file (debounced to at most once per debounce window) so
|
||||||
// an operator can move the file away for offline archiving. The
|
// an operator can move the file away for offline archiving. The
|
||||||
// next write recreates a moved or removed file, because the
|
// next write recreates a moved or removed file, because the
|
||||||
@@ -194,26 +186,17 @@ type archiveWriter struct {
|
|||||||
lastReopen time.Time
|
lastReopen time.Time
|
||||||
reopens int
|
reopens int
|
||||||
|
|
||||||
// now is the clock the reopen debounce is measured on. It is
|
|
||||||
// time.Now outside tests.
|
|
||||||
now func() time.Time
|
|
||||||
|
|
||||||
// evicted marks a writer that has been removed from the
|
// evicted marks a writer that has been removed from the
|
||||||
// registry. Its handle is closed and it must never open the
|
// per-webhook registry. Its handle is closed and it must
|
||||||
// file again: nothing holds it any more, so a reopen would
|
// never open the file again: nothing holds it any more, so a
|
||||||
// leak the handle for the process lifetime.
|
// reopen would leak the handle for the process lifetime.
|
||||||
evicted bool
|
evicted bool
|
||||||
|
|
||||||
// webhookID is the webhook the archive's target belongs to,
|
|
||||||
// so deleting the webhook can find its writers. It is set
|
|
||||||
// when the writer is created and never changes.
|
|
||||||
webhookID string
|
|
||||||
|
|
||||||
// sweepOwned marks a registry entry that the idle sweep
|
// sweepOwned marks a registry entry that the idle sweep
|
||||||
// created because no writer was cached for the target. The
|
// created because no writer was cached for the webhook. The
|
||||||
// sweep removes such an entry again when it is done, so a
|
// sweep removes such an entry again when it is done, so a
|
||||||
// sweep can never leave — or resurrect — a registry entry
|
// sweep can never leave — or resurrect — a registry entry
|
||||||
// for a target that has been deleted. A delivery that adopts
|
// for a webhook that has been deleted. A delivery that adopts
|
||||||
// the writer clears the flag, handing the entry to the
|
// the writer clears the flag, handing the entry to the
|
||||||
// registry proper.
|
// registry proper.
|
||||||
//
|
//
|
||||||
@@ -232,7 +215,6 @@ func newArchiveWriter(
|
|||||||
path: path,
|
path: path,
|
||||||
log: log,
|
log: log,
|
||||||
debounce: archiveReopenDebounce,
|
debounce: archiveReopenDebounce,
|
||||||
now: time.Now,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -268,7 +250,7 @@ func (w *archiveWriter) write(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
if w.now().Sub(w.lastReopen) >= w.debounce {
|
if time.Since(w.lastReopen) >= w.debounce {
|
||||||
return w.reopen(expiry)
|
return w.reopen(expiry)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -328,7 +310,7 @@ func (w *archiveWriter) openMode(
|
|||||||
}
|
}
|
||||||
|
|
||||||
w.db = gdb
|
w.db = gdb
|
||||||
w.lastReopen = w.now()
|
w.lastReopen = time.Now()
|
||||||
w.reopens++
|
w.reopens++
|
||||||
|
|
||||||
if expiry > 0 {
|
if expiry > 0 {
|
||||||
@@ -403,78 +385,11 @@ func (w *archiveWriter) sweepExpired(expiry time.Duration) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// rename gives the archive file a new name in the same directory,
|
|
||||||
// and the writer uses the file under that name from now on. The
|
|
||||||
// handle is closed first, which folds the -wal into the .db; any
|
|
||||||
// -wal or -shm still beside the file (left by a crash) is moved with
|
|
||||||
// it, because SQLite finds them by name. A missing file is not an
|
|
||||||
// error: the operator may have moved it away, and the next write
|
|
||||||
// creates it under the new name.
|
|
||||||
//
|
|
||||||
// If a file already has the new name, nothing is moved and the
|
|
||||||
// error is ErrArchiveNameTaken. If one file fails to move, those
|
|
||||||
// already moved are moved back before the error is returned, so the
|
|
||||||
// archive is never split across two names.
|
|
||||||
func (w *archiveWriter) rename(name string) error {
|
|
||||||
w.mu.Lock()
|
|
||||||
defer w.mu.Unlock()
|
|
||||||
|
|
||||||
if w.evicted {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"%w: %s", errArchiveWriterEvicted, w.path,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
path := filepath.Join(filepath.Dir(w.path), name)
|
|
||||||
if path == w.path {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
suffixes := []string{"", "-wal", "-shm"}
|
|
||||||
|
|
||||||
for _, suffix := range suffixes {
|
|
||||||
if fileExists(path + suffix) {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"%w: %s", ErrArchiveNameTaken, name+suffix,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
w.close()
|
|
||||||
|
|
||||||
for i, suffix := range suffixes {
|
|
||||||
err := os.Rename(w.path+suffix, path+suffix)
|
|
||||||
if err == nil || errors.Is(err, fs.ErrNotExist) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, moved := range suffixes[:i] {
|
|
||||||
backErr := os.Rename(path+moved, w.path+moved)
|
|
||||||
if backErr != nil && !errors.Is(backErr, fs.ErrNotExist) {
|
|
||||||
w.log.Error(
|
|
||||||
"failed to move archive file back",
|
|
||||||
"from", path+moved,
|
|
||||||
"to", w.path+moved,
|
|
||||||
"error", backErr,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return fmt.Errorf(
|
|
||||||
"renaming archive %s to %s: %w", w.path+suffix, path+suffix, err,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
w.path = path
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// evict closes the writer's handle and marks it unusable. It is
|
// evict closes the writer's handle and marks it unusable. It is
|
||||||
// called when the writer leaves the registry, because its target
|
// called when the writer leaves the registry, either because the
|
||||||
// or its webhook was deleted, or at shutdown. The archive FILE is
|
// webhook was deleted or because its last database target was
|
||||||
// deliberately left on disk: it is long-term storage an operator
|
// removed. The archive FILE is deliberately left on disk: it is
|
||||||
// may still want.
|
// long-term storage an operator may still want.
|
||||||
func (w *archiveWriter) evict() {
|
func (w *archiveWriter) evict() {
|
||||||
w.mu.Lock()
|
w.mu.Lock()
|
||||||
defer w.mu.Unlock()
|
defer w.mu.Unlock()
|
||||||
|
|||||||
@@ -17,109 +17,85 @@ import (
|
|||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// deliverTo archives one event to a database target, leaving the
|
// evictTestEngine builds an engine backed by a temporary data
|
||||||
// target's writer cached with its handle open.
|
// directory and returns it along with that directory.
|
||||||
func deliverTo(
|
func evictTestEngine(t *testing.T) (*delivery.Engine, string) {
|
||||||
t *testing.T, env *archiveEnv, tgt *database.Target,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
dataDir := t.TempDir()
|
||||||
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
|
||||||
|
|
||||||
env.eng.ExportDeliverDatabase(
|
eng := delivery.NewTestEngineWithDB(
|
||||||
webhookDB, seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
|
nil,
|
||||||
|
database.NewTestWebhookDBManager(dataDir),
|
||||||
|
archiveTestLogger(),
|
||||||
|
&http.Client{Timeout: 5 * time.Second},
|
||||||
|
1,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
return eng, dataDir
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestEvictWebhook_ClosesAndRemovesWriter proves that evicting
|
// TestEvictWebhook_ClosesAndRemovesWriter proves that evicting
|
||||||
// a webhook drops the archive writers of its database targets
|
// a webhook drops its archive writer from the registry and
|
||||||
// from the registry and closes their open handles, rather than
|
// closes the open archive handle, rather than leaving both
|
||||||
// leaving them alive for the process lifetime, and leaves another
|
// alive for the process lifetime.
|
||||||
// webhook's writer alone.
|
|
||||||
func TestEvictWebhook_ClosesAndRemovesWriter(t *testing.T) {
|
func TestEvictWebhook_ClosesAndRemovesWriter(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
eng, dataDir := evictTestEngine(t)
|
||||||
first := env.seedDatabaseTarget(t, "")
|
|
||||||
second := env.addDatabaseTarget(t, first.WebhookID, "")
|
|
||||||
other := env.seedDatabaseTarget(t, "")
|
|
||||||
|
|
||||||
for _, tgt := range []*database.Target{first, second, other} {
|
webhookDB := testWebhookDB(t)
|
||||||
deliverTo(t, env, tgt)
|
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
||||||
|
d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
|
||||||
|
|
||||||
require.True(
|
eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
t, env.eng.ExportArchiveHandleOpen(tgt.ID),
|
|
||||||
"the writer should hold an open handle after a write",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
env.eng.EvictWebhook(first.WebhookID)
|
webhookID := event.WebhookID
|
||||||
|
|
||||||
for _, tgt := range []*database.Target{first, second} {
|
require.True(
|
||||||
assert.False(
|
t, eng.ExportHasArchiveWriter(webhookID),
|
||||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
"a delivery should have cached an archive writer",
|
||||||
"eviction should remove the registry entry",
|
)
|
||||||
)
|
require.True(
|
||||||
assert.False(
|
t, eng.ExportArchiveHandleOpen(webhookID),
|
||||||
t, env.eng.ExportArchiveHandleOpen(tgt.ID),
|
"the writer should hold an open handle after a write",
|
||||||
"eviction should close the archive handle",
|
|
||||||
)
|
|
||||||
assert.FileExists(
|
|
||||||
t, env.archivePath(tgt),
|
|
||||||
"eviction must not delete the archive file",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.True(
|
|
||||||
t, env.eng.ExportArchiveHandleOpen(other.ID),
|
|
||||||
"another webhook's writer must be left alone",
|
|
||||||
)
|
)
|
||||||
}
|
|
||||||
|
|
||||||
// TestEvictTarget_LeavesOtherTargets proves that evicting one
|
eng.EvictWebhook(webhookID)
|
||||||
// database target leaves the writer of another target of the same
|
|
||||||
// webhook in place.
|
|
||||||
func TestEvictTarget_LeavesOtherTargets(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
assert.False(
|
||||||
doomed := env.seedDatabaseTarget(t, "")
|
t, eng.ExportHasArchiveWriter(webhookID),
|
||||||
kept := env.addDatabaseTarget(t, doomed.WebhookID, "")
|
"eviction should remove the registry entry",
|
||||||
|
)
|
||||||
|
assert.False(
|
||||||
|
t, eng.ExportArchiveHandleOpen(webhookID),
|
||||||
|
"eviction should close the archive handle",
|
||||||
|
)
|
||||||
|
|
||||||
deliverTo(t, env, doomed)
|
archivePath := filepath.Join(
|
||||||
deliverTo(t, env, kept)
|
dataDir, fmt.Sprintf("archive-%s.db", webhookID),
|
||||||
|
)
|
||||||
env.eng.EvictTarget(doomed.ID)
|
|
||||||
|
|
||||||
assert.False(t, env.eng.ExportHasArchiveWriter(doomed.ID))
|
|
||||||
assert.FileExists(
|
assert.FileExists(
|
||||||
t, env.archivePath(doomed),
|
t, archivePath,
|
||||||
"eviction must not delete the archive file",
|
"eviction must not delete the archive file",
|
||||||
)
|
)
|
||||||
assert.True(
|
|
||||||
t, env.eng.ExportArchiveHandleOpen(kept.ID),
|
|
||||||
"the other target's writer must be left alone",
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestEvictWebhook_UnknownWebhookIsNoOp proves eviction is safe
|
// TestEvictWebhook_UnknownWebhookIsNoOp proves eviction is safe
|
||||||
// for the common case of a webhook or target that never had an
|
// for the common case of a webhook that never had a database
|
||||||
// archive writer, and that repeating it does not panic.
|
// target, and that repeating it does not panic.
|
||||||
func TestEvictWebhook_UnknownWebhookIsNoOp(t *testing.T) {
|
func TestEvictWebhook_UnknownWebhookIsNoOp(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
eng, _ := evictTestEngine(t)
|
||||||
|
|
||||||
assert.NotPanics(t, func() {
|
assert.NotPanics(t, func() {
|
||||||
env.eng.EvictWebhook("no-such-webhook")
|
eng.EvictWebhook("no-such-webhook")
|
||||||
env.eng.EvictWebhook("no-such-webhook")
|
eng.EvictWebhook("no-such-webhook")
|
||||||
env.eng.EvictTarget("no-such-target")
|
|
||||||
env.eng.EvictTarget("no-such-target")
|
|
||||||
})
|
})
|
||||||
|
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.eng.ExportHasArchiveWriter("no-such-target"),
|
t, eng.ExportHasArchiveWriter("no-such-webhook"),
|
||||||
"eviction must not create a writer",
|
"eviction must not create a writer",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -313,14 +289,17 @@ func TestEvictWebhook_RacingWriteDoesNotReopenHandle(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
eng, _ := evictTestEngine(t)
|
||||||
tgt := env.seedDatabaseTarget(t, "")
|
|
||||||
|
webhookDB := testWebhookDB(t)
|
||||||
|
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
||||||
|
d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
|
||||||
|
|
||||||
// Prime the registry so the test can hold the very writer the
|
// Prime the registry so the test can hold the very writer the
|
||||||
// eviction is about to detach.
|
// eviction is about to detach.
|
||||||
deliverTo(t, env, tgt)
|
eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
|
|
||||||
w := env.eng.ExportArchiveWriterFor(tgt.ID)
|
w := eng.ExportArchiveWriterFor(event.WebhookID)
|
||||||
require.NotNil(t, w)
|
require.NotNil(t, w)
|
||||||
require.True(t, w.HandleOpen())
|
require.True(t, w.HandleOpen())
|
||||||
|
|
||||||
@@ -330,7 +309,7 @@ func TestEvictWebhook_RacingWriteDoesNotReopenHandle(
|
|||||||
// eviction has to contend for the writer's mutex.
|
// eviction has to contend for the writer's mutex.
|
||||||
race.awaitFirstWrite()
|
race.awaitFirstWrite()
|
||||||
|
|
||||||
env.eng.EvictWebhook(tgt.WebhookID)
|
eng.EvictWebhook(event.WebhookID)
|
||||||
|
|
||||||
sawEvicted, otherErr := race.wait()
|
sawEvicted, otherErr := race.wait()
|
||||||
|
|
||||||
@@ -345,33 +324,41 @@ func TestEvictWebhook_RacingWriteDoesNotReopenHandle(
|
|||||||
"been evicted",
|
"been evicted",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
t, eng.ExportHasArchiveWriter(event.WebhookID),
|
||||||
"the registry entry must stay gone",
|
"the registry entry must stay gone",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestEvictWebhook_LaterDeliveryRecreatesWriter proves eviction
|
// TestEvictWebhook_LaterDeliveryRecreatesWriter proves eviction
|
||||||
// does not break archiving for a target that is still alive: a
|
// does not break archiving for a webhook that is still alive: a
|
||||||
// subsequent delivery gets a brand new writer from the registry.
|
// subsequent delivery gets a brand new writer from the registry.
|
||||||
// It says nothing about the evicted writer itself — that is what
|
// It says nothing about the evicted writer itself — that is what
|
||||||
// TestEvictedWriter_WriteDoesNotReopenFile covers.
|
// TestEvictedWriter_WriteDoesNotReopenFile covers.
|
||||||
func TestEvictWebhook_LaterDeliveryRecreatesWriter(t *testing.T) {
|
func TestEvictWebhook_LaterDeliveryRecreatesWriter(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
eng, _ := evictTestEngine(t)
|
||||||
tgt := env.seedDatabaseTarget(t, "")
|
|
||||||
|
|
||||||
deliverTo(t, env, tgt)
|
webhookDB := testWebhookDB(t)
|
||||||
require.True(t, env.eng.ExportHasArchiveWriter(tgt.ID))
|
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
||||||
|
d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
|
||||||
|
|
||||||
env.eng.EvictWebhook(tgt.WebhookID)
|
eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
|
require.True(
|
||||||
|
t, eng.ExportHasArchiveWriter(event.WebhookID),
|
||||||
|
)
|
||||||
|
|
||||||
// A fresh delivery for the same target gets a brand new
|
eng.EvictWebhook(event.WebhookID)
|
||||||
|
|
||||||
|
// A fresh delivery for the same webhook gets a brand new
|
||||||
// writer from the registry, so archiving keeps working.
|
// writer from the registry, so archiving keeps working.
|
||||||
deliverTo(t, env, tgt)
|
second := seedDatabaseTargetDelivery(
|
||||||
|
t, webhookDB, event, "",
|
||||||
|
)
|
||||||
|
eng.ExportDeliverDatabase(webhookDB, second)
|
||||||
|
|
||||||
assert.True(
|
assert.True(
|
||||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
t, eng.ExportHasArchiveWriter(event.WebhookID),
|
||||||
"a later delivery should recreate the writer",
|
"a later delivery should recreate the writer",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -383,16 +370,19 @@ func TestEvictWebhook_LaterDeliveryRecreatesWriter(t *testing.T) {
|
|||||||
func TestEngineStop_WriteAfterStopIsRefused(t *testing.T) {
|
func TestEngineStop_WriteAfterStopIsRefused(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
eng, _ := evictTestEngine(t)
|
||||||
tgt := env.seedDatabaseTarget(t, "")
|
|
||||||
|
|
||||||
deliverTo(t, env, tgt)
|
webhookDB := testWebhookDB(t)
|
||||||
|
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
||||||
|
d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
|
||||||
|
|
||||||
w := env.eng.ExportArchiveWriterFor(tgt.ID)
|
eng.ExportDeliverDatabase(webhookDB, d)
|
||||||
|
|
||||||
|
w := eng.ExportArchiveWriterFor(event.WebhookID)
|
||||||
require.NotNil(t, w)
|
require.NotNil(t, w)
|
||||||
require.True(t, w.HandleOpen())
|
require.True(t, w.HandleOpen())
|
||||||
|
|
||||||
require.NoError(t, env.eng.ExportStop(context.Background()))
|
require.NoError(t, eng.ExportStop(context.Background()))
|
||||||
|
|
||||||
err := w.Write(evictTestRow("ev-after-stop"), 0)
|
err := w.Write(evictTestRow("ev-after-stop"), 0)
|
||||||
|
|
||||||
@@ -405,7 +395,7 @@ func TestEngineStop_WriteAfterStopIsRefused(t *testing.T) {
|
|||||||
"a refused write must not reopen the archive",
|
"a refused write must not reopen the archive",
|
||||||
)
|
)
|
||||||
assert.False(
|
assert.False(
|
||||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
t, eng.ExportHasArchiveWriter(event.WebhookID),
|
||||||
"the stop should empty the registry",
|
"the stop should empty the registry",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -1,275 +0,0 @@
|
|||||||
package delivery
|
|
||||||
|
|
||||||
import (
|
|
||||||
"compress/gzip"
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
"encoding/base64"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"log/slog"
|
|
||||||
"path/filepath"
|
|
||||||
"time"
|
|
||||||
"unicode/utf8"
|
|
||||||
|
|
||||||
"gorm.io/driver/sqlite"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// archiveTableQuery counts the archive's table: 0 when the archive
|
|
||||||
// writer has created the file but not yet the table in it.
|
|
||||||
const archiveTableQuery = "SELECT count(*) FROM sqlite_master " +
|
|
||||||
"WHERE type = 'table' AND name = 'archived_events'"
|
|
||||||
|
|
||||||
// ArchivePath returns where a database target's archive file is: in
|
|
||||||
// the data directory, beside the webhook's event database, under the
|
|
||||||
// name ArchiveFileName gives it.
|
|
||||||
func ArchivePath(
|
|
||||||
dbMgr *database.WebhookDBManager,
|
|
||||||
webhook *database.Webhook,
|
|
||||||
target *database.Target,
|
|
||||||
) string {
|
|
||||||
return filepath.Join(
|
|
||||||
filepath.Dir(dbMgr.DBPath(webhook.ID)),
|
|
||||||
ArchiveFileName(webhook.Name, target.Name, target.ID),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ArchiveExportFileName returns the name a database target's archive
|
|
||||||
// downloads under:
|
|
||||||
// archive-WEBHOOKNAME-TARGETNAME-YYYYMMDDTHHMMSSZ.json.gz, the names
|
|
||||||
// made safe as in ArchiveFileName and the time in UTC.
|
|
||||||
func ArchiveExportFileName(
|
|
||||||
webhookName, targetName string, at time.Time,
|
|
||||||
) string {
|
|
||||||
return "archive-" + archiveNamePart(webhookName) + "-" +
|
|
||||||
archiveNamePart(targetName) + "-" +
|
|
||||||
at.UTC().Format("20060102T150405Z") + ".json.gz"
|
|
||||||
}
|
|
||||||
|
|
||||||
// ArchiveExport is a database target's archive opened for download.
|
|
||||||
// It reads the file on its own connection, inside one read-only
|
|
||||||
// transaction, so it writes out the archive as it stood when
|
|
||||||
// OpenArchiveExport returned.
|
|
||||||
//
|
|
||||||
// Archives are in WAL mode, where a reader works from a snapshot and
|
|
||||||
// never blocks a writer: archive writes go on while an export is open,
|
|
||||||
// and the export does not see them. SQLite cannot checkpoint the -wal
|
|
||||||
// past an open snapshot, so the -wal grows until the export is closed.
|
|
||||||
type ArchiveExport struct {
|
|
||||||
db *sql.DB
|
|
||||||
tx *gorm.DB
|
|
||||||
|
|
||||||
// empty is true when there is nothing to read: no file, or a file
|
|
||||||
// without the archive's table yet.
|
|
||||||
empty bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// exportedName is how an export names its webhook and its target.
|
|
||||||
type exportedName struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
Name string `json:"name"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// OpenArchiveExport opens the archive file at path for export and
|
|
||||||
// takes the snapshot the export reads. It never creates the file: with
|
|
||||||
// no file at path, the export has no rows.
|
|
||||||
//
|
|
||||||
// Once it has returned, the file is open, so a rename or a move of it
|
|
||||||
// does not affect the export, which reads the same file under its new
|
|
||||||
// name.
|
|
||||||
//
|
|
||||||
// The transaction lasts as long as ctx does, so ctx must last for the
|
|
||||||
// whole export.
|
|
||||||
func OpenArchiveExport(
|
|
||||||
ctx context.Context, path string, log *slog.Logger,
|
|
||||||
) (*ArchiveExport, error) {
|
|
||||||
if !fileExists(path) {
|
|
||||||
return &ArchiveExport{empty: true}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
db, err := database.OpenSQLite(path, archiveModeExisting)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("opening archive %s: %w", path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
gdb, err := gorm.Open(
|
|
||||||
sqlite.Dialector{Conn: db}, &gorm.Config{
|
|
||||||
// Never leave this at GORM's default. See
|
|
||||||
// internal/gormlog.
|
|
||||||
Logger: gormlog.New(log),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
_ = db.Close()
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("opening archive %s: %w", path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ReadOnly makes the driver begin a deferred transaction in place
|
|
||||||
// of the BEGIN IMMEDIATE the connection string asks for, so the
|
|
||||||
// export never takes the archive's write lock.
|
|
||||||
tx := gdb.WithContext(ctx).Begin(&sql.TxOptions{ReadOnly: true})
|
|
||||||
if tx.Error != nil {
|
|
||||||
_ = db.Close()
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("reading archive %s: %w", path, tx.Error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// The transaction's first read is what takes the snapshot.
|
|
||||||
var tables int
|
|
||||||
|
|
||||||
err = tx.Raw(archiveTableQuery).Row().Scan(&tables)
|
|
||||||
if err != nil {
|
|
||||||
_ = tx.Rollback()
|
|
||||||
_ = db.Close()
|
|
||||||
|
|
||||||
return nil, fmt.Errorf("reading archive %s: %w", path, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &ArchiveExport{db: db, tx: tx, empty: tables == 0}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// WriteGzipJSON writes the export to w as one gzipped JSON object:
|
|
||||||
// webhook and target, each an id and a name; exported_at; and
|
|
||||||
// archived_events, one object per archived row, keyed by column name.
|
|
||||||
// A body that is not valid UTF-8 cannot be a JSON string, so it is
|
|
||||||
// written in base64, with "body_encoding": "base64" beside it.
|
|
||||||
//
|
|
||||||
// Each row is written out before the next is read, so neither the
|
|
||||||
// archive nor its JSON is ever held in memory whole. After an error
|
|
||||||
// the gzip stream is left unfinished, so what was written does not
|
|
||||||
// decompress as a whole file.
|
|
||||||
func (x *ArchiveExport) WriteGzipJSON(
|
|
||||||
ctx context.Context,
|
|
||||||
w io.Writer,
|
|
||||||
webhook *database.Webhook,
|
|
||||||
target *database.Target,
|
|
||||||
exportedAt time.Time,
|
|
||||||
) error {
|
|
||||||
head, err := json.Marshal(map[string]any{
|
|
||||||
"webhook": exportedName{ID: webhook.ID, Name: webhook.Name},
|
|
||||||
"target": exportedName{ID: target.ID, Name: target.Name},
|
|
||||||
"exported_at": exportedAt.UTC(),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("encoding archive export: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
zw := gzip.NewWriter(w)
|
|
||||||
|
|
||||||
err = x.writeJSON(ctx, zw, head)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("writing archive export: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return zw.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Close ends the export's transaction and closes its connection.
|
|
||||||
func (x *ArchiveExport) Close() error {
|
|
||||||
if x.db == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
_ = x.tx.Rollback()
|
|
||||||
|
|
||||||
return x.db.Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeJSON writes head with archived_events added as its last key,
|
|
||||||
// the rows going into it one at a time.
|
|
||||||
func (x *ArchiveExport) writeJSON(
|
|
||||||
ctx context.Context, w io.Writer, head []byte,
|
|
||||||
) error {
|
|
||||||
// head goes out without its closing brace, so that
|
|
||||||
// archived_events can follow it.
|
|
||||||
_, err := w.Write(head[:len(head)-1])
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = io.WriteString(w, `,"archived_events":[`)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = x.writeRows(ctx, w)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = io.WriteString(w, "\n]}\n")
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeRows writes each archived row to w, oldest first, one per line,
|
|
||||||
// separated by commas.
|
|
||||||
func (x *ArchiveExport) writeRows(ctx context.Context, w io.Writer) error {
|
|
||||||
if x.empty {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
rows, err := x.tx.WithContext(ctx).
|
|
||||||
Model(&archivedEvent{}).Order("id").Rows()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
defer func() { _ = rows.Close() }()
|
|
||||||
|
|
||||||
for sep := "\n"; rows.Next(); sep = ",\n" {
|
|
||||||
var ev archivedEvent
|
|
||||||
|
|
||||||
err = x.tx.ScanRows(rows, &ev)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = io.WriteString(w, sep)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = writeRow(w, &ev)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return rows.Err()
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeRow writes an archived row to w as a JSON object keyed by
|
|
||||||
// column name, its body in base64 when it is not valid UTF-8.
|
|
||||||
func writeRow(w io.Writer, ev *archivedEvent) error {
|
|
||||||
row := map[string]any{
|
|
||||||
"id": ev.ID,
|
|
||||||
"event_id": ev.EventID,
|
|
||||||
"webhook_id": ev.WebhookID,
|
|
||||||
"entrypoint_id": ev.EntrypointID,
|
|
||||||
"method": ev.Method,
|
|
||||||
"headers": ev.Headers,
|
|
||||||
"body": ev.Body,
|
|
||||||
"content_type": ev.ContentType,
|
|
||||||
"archived_at": ev.ArchivedAt.UTC(),
|
|
||||||
}
|
|
||||||
|
|
||||||
if !utf8.ValidString(ev.Body) {
|
|
||||||
row["body"] = base64.StdEncoding.EncodeToString([]byte(ev.Body))
|
|
||||||
row["body_encoding"] = "base64"
|
|
||||||
}
|
|
||||||
|
|
||||||
line, err := json.Marshal(row)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = w.Write(line)
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
@@ -1,412 +0,0 @@
|
|||||||
package delivery_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bufio"
|
|
||||||
"bytes"
|
|
||||||
"compress/gzip"
|
|
||||||
"crypto/rand"
|
|
||||||
"encoding/base64"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"runtime"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The webhook and the target the export tests' archives belong to.
|
|
||||||
const (
|
|
||||||
exportWebhookID = "wh-export"
|
|
||||||
exportWebhookName = "Orders (EU)"
|
|
||||||
exportTargetID = "tgt-export"
|
|
||||||
exportTargetName = "Long-term archive"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// binaryBody is a body that is not valid UTF-8.
|
|
||||||
binaryBody = "\xff\xfe\x00\x01binary\x80"
|
|
||||||
|
|
||||||
// openedEventID is the event the snapshot tests archive before
|
|
||||||
// they open the export.
|
|
||||||
openedEventID = "opened"
|
|
||||||
)
|
|
||||||
|
|
||||||
// writeExportTo writes export to w as the archive of the export tests'
|
|
||||||
// webhook and target, exported at 2026-10-02T12:03:04Z.
|
|
||||||
func writeExportTo(
|
|
||||||
t *testing.T, export *delivery.ArchiveExport, w io.Writer,
|
|
||||||
) error {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return export.WriteGzipJSON(
|
|
||||||
t.Context(), w,
|
|
||||||
&database.Webhook{
|
|
||||||
BaseModel: database.BaseModel{ID: exportWebhookID},
|
|
||||||
Name: exportWebhookName,
|
|
||||||
},
|
|
||||||
&database.Target{
|
|
||||||
BaseModel: database.BaseModel{ID: exportTargetID},
|
|
||||||
Name: exportTargetName,
|
|
||||||
},
|
|
||||||
time.Date(2026, 10, 2, 12, 3, 4, 0, time.UTC),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// exportArchive runs a whole export of the archive at path and returns
|
|
||||||
// its JSON, decompressed and parsed.
|
|
||||||
func exportArchive(t *testing.T, path string) map[string]any {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
export, err := delivery.OpenArchiveExport(
|
|
||||||
t.Context(), path, archiveTestLogger(),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, export.Close()) }()
|
|
||||||
|
|
||||||
return writeExport(t, export)
|
|
||||||
}
|
|
||||||
|
|
||||||
// writeExport writes an opened export and returns its JSON,
|
|
||||||
// decompressed and parsed. Reading to the end makes the gzip reader
|
|
||||||
// check that the stream was finished.
|
|
||||||
func writeExport(
|
|
||||||
t *testing.T, export *delivery.ArchiveExport,
|
|
||||||
) map[string]any {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
require.NoError(t, writeExportTo(t, export, &buf))
|
|
||||||
|
|
||||||
zr, err := gzip.NewReader(&buf)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
raw, err := io.ReadAll(zr)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
var got map[string]any
|
|
||||||
|
|
||||||
require.NoError(t, json.Unmarshal(raw, &got))
|
|
||||||
|
|
||||||
return got
|
|
||||||
}
|
|
||||||
|
|
||||||
// exportedEvents returns an export's archived_events.
|
|
||||||
func exportedEvents(t *testing.T, got map[string]any) []map[string]any {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
list, ok := got["archived_events"].([]any)
|
|
||||||
require.True(t, ok, "archived_events must be an array: %v", got)
|
|
||||||
|
|
||||||
events := make([]map[string]any, len(list))
|
|
||||||
|
|
||||||
for i, v := range list {
|
|
||||||
events[i], ok = v.(map[string]any)
|
|
||||||
require.True(t, ok, "an archived event must be an object: %v", v)
|
|
||||||
}
|
|
||||||
|
|
||||||
return events
|
|
||||||
}
|
|
||||||
|
|
||||||
// exportedEventIDs returns the event_id of each of an export's
|
|
||||||
// archived_events.
|
|
||||||
func exportedEventIDs(t *testing.T, got map[string]any) []string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
events := exportedEvents(t, got)
|
|
||||||
ids := make([]string, 0, len(events))
|
|
||||||
|
|
||||||
for _, ev := range events {
|
|
||||||
ids = append(ids, fmt.Sprint(ev["event_id"]))
|
|
||||||
}
|
|
||||||
|
|
||||||
return ids
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveExport_MatchesStoredRows proves an export holds the
|
|
||||||
// webhook, the target, the time, and every column of every stored
|
|
||||||
// row: a body that is valid UTF-8 as a string, and one that is not in
|
|
||||||
// base64, marked as such.
|
|
||||||
func TestArchiveExport_MatchesStoredRows(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive.db")
|
|
||||||
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
|
||||||
bodies := []string{`{"order":1}`, "plain text", "", binaryBody}
|
|
||||||
|
|
||||||
for i, body := range bodies {
|
|
||||||
require.NoError(t, w.Write(delivery.ExportArchivedEvent{
|
|
||||||
EventID: fmt.Sprintf("ev-%d", i),
|
|
||||||
WebhookID: exportWebhookID,
|
|
||||||
EntrypointID: "ep-1",
|
|
||||||
Method: "POST",
|
|
||||||
Headers: `{"X-Test":["yes"]}`,
|
|
||||||
Body: body,
|
|
||||||
ContentType: testContentType,
|
|
||||||
}, 0))
|
|
||||||
}
|
|
||||||
|
|
||||||
var stored []delivery.ExportArchivedEvent
|
|
||||||
|
|
||||||
require.NoError(t, openArchiveDBForRead(t, path).
|
|
||||||
Order("id").Find(&stored).Error)
|
|
||||||
|
|
||||||
got := exportArchive(t, path)
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
map[string]any{"id": exportWebhookID, "name": exportWebhookName},
|
|
||||||
got["webhook"],
|
|
||||||
)
|
|
||||||
assert.Equal(t,
|
|
||||||
map[string]any{"id": exportTargetID, "name": exportTargetName},
|
|
||||||
got["target"],
|
|
||||||
)
|
|
||||||
assert.Equal(t, "2026-10-02T12:03:04Z", got["exported_at"])
|
|
||||||
|
|
||||||
events := exportedEvents(t, got)
|
|
||||||
require.Len(t, events, len(bodies))
|
|
||||||
|
|
||||||
for i, row := range stored {
|
|
||||||
assertExportedRow(t, row, events[i])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// assertExportedRow checks that ev, from an export, holds every column
|
|
||||||
// of the stored row.
|
|
||||||
func assertExportedRow(
|
|
||||||
t *testing.T, row delivery.ExportArchivedEvent, ev map[string]any,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
archivedAt, err := time.Parse(
|
|
||||||
time.RFC3339Nano, fmt.Sprint(ev["archived_at"]),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.True(t, archivedAt.Equal(row.ArchivedAt))
|
|
||||||
|
|
||||||
assert.EqualValues(t, row.ID, ev["id"])
|
|
||||||
assert.Equal(t, row.EventID, ev["event_id"])
|
|
||||||
assert.Equal(t, row.WebhookID, ev["webhook_id"])
|
|
||||||
assert.Equal(t, row.EntrypointID, ev["entrypoint_id"])
|
|
||||||
assert.Equal(t, row.Method, ev["method"])
|
|
||||||
assert.Equal(t, row.Headers, ev["headers"])
|
|
||||||
assert.Equal(t, row.ContentType, ev["content_type"])
|
|
||||||
|
|
||||||
if row.Body != binaryBody {
|
|
||||||
assert.Equal(t, row.Body, ev["body"])
|
|
||||||
assert.Len(t, ev, 9, "the nine columns and nothing else: %v", ev)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
body, err := base64.StdEncoding.DecodeString(fmt.Sprint(ev["body"]))
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, binaryBody, string(body))
|
|
||||||
assert.Equal(t, "base64", ev["body_encoding"])
|
|
||||||
assert.Len(t, ev, 10, "the nine columns and body_encoding: %v", ev)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveExport_Empty proves an archive with nothing in it exports
|
|
||||||
// as an empty archived_events: no file, which the export must not
|
|
||||||
// create; a file the archive writer has not yet put its table in; and
|
|
||||||
// a table with no rows.
|
|
||||||
func TestArchiveExport_Empty(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
missing := filepath.Join(dir, "missing.db")
|
|
||||||
noTable := filepath.Join(dir, "no-table.db")
|
|
||||||
noRows := filepath.Join(dir, "no-rows.db")
|
|
||||||
|
|
||||||
require.NoError(t, os.WriteFile(noTable, nil, 0o600))
|
|
||||||
require.NoError(t,
|
|
||||||
delivery.NewExportArchiveWriter(noRows, archiveTestLogger(), 0).
|
|
||||||
Open(0),
|
|
||||||
)
|
|
||||||
|
|
||||||
for _, path := range []string{missing, noTable, noRows} {
|
|
||||||
assert.Empty(t, exportedEvents(t, exportArchive(t, path)), path)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, suffix := range archiveFileSuffixes() {
|
|
||||||
assert.NoFileExists(t, missing+suffix)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveExport_ReadsOneSnapshot proves an export writes the
|
|
||||||
// archive as it was when it was opened, and holds up no archive
|
|
||||||
// write: a row written while the export is open is stored, and is not
|
|
||||||
// in the export. A write held up for the whole busy timeout would
|
|
||||||
// fail.
|
|
||||||
func TestArchiveExport_ReadsOneSnapshot(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive.db")
|
|
||||||
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
|
||||||
|
|
||||||
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: openedEventID}, 0))
|
|
||||||
|
|
||||||
export, err := delivery.OpenArchiveExport(
|
|
||||||
t.Context(), path, archiveTestLogger(),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, export.Close()) }()
|
|
||||||
|
|
||||||
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "during"}, 0))
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
[]string{openedEventID}, exportedEventIDs(t, writeExport(t, export)),
|
|
||||||
)
|
|
||||||
|
|
||||||
var stored int64
|
|
||||||
|
|
||||||
require.NoError(t, openArchiveDBForRead(t, path).
|
|
||||||
Model(&delivery.ExportArchivedEvent{}).Count(&stored).Error)
|
|
||||||
assert.Equal(t, int64(2), stored)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveExport_SurvivesRename proves that renaming the archive
|
|
||||||
// while an export of it is open, as renaming its webhook or target
|
|
||||||
// does, leaves the export reading the same file.
|
|
||||||
func TestArchiveExport_SurvivesRename(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive-old.db")
|
|
||||||
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
|
||||||
|
|
||||||
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: openedEventID}, 0))
|
|
||||||
|
|
||||||
export, err := delivery.OpenArchiveExport(
|
|
||||||
t.Context(), path, archiveTestLogger(),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, export.Close()) }()
|
|
||||||
|
|
||||||
require.NoError(t, w.Rename("archive-new.db"))
|
|
||||||
require.NoError(t, w.Write(delivery.ExportArchivedEvent{EventID: "after"}, 0))
|
|
||||||
require.NoFileExists(t, path)
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
[]string{openedEventID}, exportedEventIDs(t, writeExport(t, export)),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// heapPeak is an io.Writer that discards what it is given and records
|
|
||||||
// the largest heap it saw at a write. It collects garbage before each
|
|
||||||
// reading, so the heap it reads is what is still held.
|
|
||||||
type heapPeak struct {
|
|
||||||
max uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
func (p *heapPeak) Write(b []byte) (int, error) {
|
|
||||||
var m runtime.MemStats
|
|
||||||
|
|
||||||
runtime.GC()
|
|
||||||
runtime.ReadMemStats(&m)
|
|
||||||
p.max = max(p.max, m.HeapAlloc)
|
|
||||||
|
|
||||||
return len(b), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// exportHeapGrowth exports an archive of rows random bodies, each
|
|
||||||
// bodySize bytes of base64, and returns how far the heap rose above
|
|
||||||
// where it stood when the export began, at its highest.
|
|
||||||
func exportHeapGrowth(t *testing.T, rows, bodySize int) uint64 {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
path := filepath.Join(t.TempDir(), "archive.db")
|
|
||||||
w := delivery.NewExportArchiveWriter(path, archiveTestLogger(), 0)
|
|
||||||
|
|
||||||
// Base64 makes four characters of every three bytes.
|
|
||||||
random := make([]byte, bodySize/4*3)
|
|
||||||
|
|
||||||
for range rows {
|
|
||||||
_, _ = rand.Read(random)
|
|
||||||
|
|
||||||
require.NoError(t, w.Write(delivery.ExportArchivedEvent{
|
|
||||||
Body: base64.StdEncoding.EncodeToString(random),
|
|
||||||
}, 0))
|
|
||||||
}
|
|
||||||
|
|
||||||
export, err := delivery.OpenArchiveExport(
|
|
||||||
t.Context(), path, archiveTestLogger(),
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
defer func() { require.NoError(t, export.Close()) }()
|
|
||||||
|
|
||||||
runtime.GC()
|
|
||||||
|
|
||||||
var start runtime.MemStats
|
|
||||||
|
|
||||||
runtime.ReadMemStats(&start)
|
|
||||||
|
|
||||||
// Through a buffer, the heap is read once per 8 KiB of output
|
|
||||||
// rather than at each of gzip's small writes, which takes far
|
|
||||||
// longer.
|
|
||||||
peak := &heapPeak{max: start.HeapAlloc}
|
|
||||||
buffered := bufio.NewWriterSize(peak, 8<<10)
|
|
||||||
|
|
||||||
require.NoError(t, writeExportTo(t, export, buffered))
|
|
||||||
require.NoError(t, buffered.Flush())
|
|
||||||
|
|
||||||
return peak.max - start.HeapAlloc
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveExport_Streams proves an export holds neither the archive
|
|
||||||
// nor its output in memory whole: exporting 384 KiB more of archive
|
|
||||||
// raises the heap's peak by less than half of that. The export's own
|
|
||||||
// memory, mostly gzip's compressor, is the same for both archives, so
|
|
||||||
// it cancels out. The bodies are random bytes in base64, which gzip
|
|
||||||
// shrinks by only a quarter, so an export that read every row before
|
|
||||||
// writing, or built the JSON or the gzipped file before writing it,
|
|
||||||
// would raise the peak by at least three quarters of the difference.
|
|
||||||
//
|
|
||||||
// The smaller archive has two rows so that its export, too, writes
|
|
||||||
// out more than the 8 KiB buffer in exportHeapGrowth before it ends:
|
|
||||||
// the heap must be read while the export's own memory is held.
|
|
||||||
//
|
|
||||||
//nolint:paralleltest // It measures the heap, which tests share.
|
|
||||||
func TestArchiveExport_Streams(t *testing.T) {
|
|
||||||
const (
|
|
||||||
bodySize = 16 << 10
|
|
||||||
smallRows = 2
|
|
||||||
largeRows = smallRows + 24
|
|
||||||
limit = (largeRows - smallRows) * bodySize / 2
|
|
||||||
)
|
|
||||||
|
|
||||||
small := exportHeapGrowth(t, smallRows, bodySize)
|
|
||||||
large := exportHeapGrowth(t, largeRows, bodySize)
|
|
||||||
|
|
||||||
assert.Less(t, large, small+limit,
|
|
||||||
"the heap rose by %d for %d rows and by %d for %d rows",
|
|
||||||
small, smallRows, large, largeRows,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveExportFileName proves the download is named for the
|
|
||||||
// webhook and the target, with the names made safe as for the archive
|
|
||||||
// file, and the export time in UTC.
|
|
||||||
func TestArchiveExportFileName(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cest := time.FixedZone("CEST", int((2 * time.Hour).Seconds()))
|
|
||||||
|
|
||||||
assert.Equal(t,
|
|
||||||
"archive-orders-eu-long-term-archive-20261002T120304Z.json.gz",
|
|
||||||
delivery.ArchiveExportFileName(
|
|
||||||
exportWebhookName, exportTargetName,
|
|
||||||
time.Date(2026, 10, 2, 14, 3, 4, 0, cest),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -4,12 +4,13 @@ import (
|
|||||||
"database/sql"
|
"database/sql"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"gorm.io/driver/sqlite"
|
"gorm.io/driver/sqlite"
|
||||||
@@ -17,7 +18,6 @@ import (
|
|||||||
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
_ "modernc.org/sqlite" // Pure Go SQLite driver.
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
"sneak.berlin/go/webhooker/internal/gormlog"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func archiveTestLogger() *slog.Logger {
|
func archiveTestLogger() *slog.Logger {
|
||||||
@@ -43,8 +43,7 @@ func openArchiveDBForRead(
|
|||||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||||
|
|
||||||
gdb, err := gorm.Open(
|
gdb, err := gorm.Open(
|
||||||
sqlite.Dialector{Conn: sqlDB},
|
sqlite.Dialector{Conn: sqlDB}, &gorm.Config{},
|
||||||
&gorm.Config{Logger: gormlog.New(slog.New(slog.DiscardHandler))},
|
|
||||||
)
|
)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
|
||||||
@@ -75,18 +74,25 @@ func removeArchiveFiles(t *testing.T, path string) {
|
|||||||
|
|
||||||
// TestDeliverDatabase_ArchivesEvent verifies that delivering to
|
// TestDeliverDatabase_ArchivesEvent verifies that delivering to
|
||||||
// a database target marks the delivery delivered and archives
|
// a database target marks the delivery delivered and archives
|
||||||
// the full event into the target's own archive file.
|
// the full event into a separate per-webhook archive file.
|
||||||
func TestDeliverDatabase_ArchivesEvent(t *testing.T) {
|
func TestDeliverDatabase_ArchivesEvent(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
dataDir := t.TempDir()
|
||||||
tgt := env.seedDatabaseTarget(t, "")
|
dbMgr := database.NewTestWebhookDBManager(dataDir)
|
||||||
|
|
||||||
|
e := delivery.NewTestEngineWithDB(
|
||||||
|
nil, dbMgr,
|
||||||
|
archiveTestLogger(),
|
||||||
|
&http.Client{Timeout: 5 * time.Second},
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
webhookDB := testWebhookDB(t)
|
||||||
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
||||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
|
d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
|
||||||
|
|
||||||
env.eng.ExportDeliverDatabase(webhookDB, d)
|
e.ExportDeliverDatabase(webhookDB, d)
|
||||||
|
|
||||||
var updated database.Delivery
|
var updated database.Delivery
|
||||||
|
|
||||||
@@ -99,7 +105,8 @@ func TestDeliverDatabase_ArchivesEvent(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
archivePath := filepath.Join(
|
archivePath := filepath.Join(
|
||||||
env.dataDir, "archive-sweep-test-archive-"+tgt.ID+".db",
|
dataDir,
|
||||||
|
fmt.Sprintf("archive-%s.db", event.WebhookID),
|
||||||
)
|
)
|
||||||
assert.FileExists(t, archivePath)
|
assert.FileExists(t, archivePath)
|
||||||
|
|
||||||
@@ -186,20 +193,13 @@ func TestArchiveWriter_RecreatesAfterRemoval(
|
|||||||
func TestArchiveWriter_ReopenDebounce(t *testing.T) {
|
func TestArchiveWriter_ReopenDebounce(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
const debounce = 2 * time.Second
|
// A generous debounce keeps the two rapid writes inside
|
||||||
|
// the window even on a heavily loaded test machine.
|
||||||
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
path := filepath.Join(t.TempDir(), "archive-wh.db")
|
||||||
w := delivery.NewExportArchiveWriter(
|
w := delivery.NewExportArchiveWriter(
|
||||||
path, archiveTestLogger(), debounce,
|
path, archiveTestLogger(), 2*time.Second,
|
||||||
)
|
)
|
||||||
|
|
||||||
// The writer measures its reopen debounce on this clock, which
|
|
||||||
// only the test moves, so how long the host takes between
|
|
||||||
// writes cannot change the result.
|
|
||||||
now := time.Now()
|
|
||||||
|
|
||||||
w.SetNow(func() time.Time { return now })
|
|
||||||
|
|
||||||
require.NoError(t, w.Write(
|
require.NoError(t, w.Write(
|
||||||
delivery.ExportArchivedEvent{EventID: "a"}, 0,
|
delivery.ExportArchivedEvent{EventID: "a"}, 0,
|
||||||
))
|
))
|
||||||
@@ -211,7 +211,7 @@ func TestArchiveWriter_ReopenDebounce(t *testing.T) {
|
|||||||
// initial open — no extra close/reopen.
|
// initial open — no extra close/reopen.
|
||||||
assert.Equal(t, 1, w.Reopens())
|
assert.Equal(t, 1, w.Reopens())
|
||||||
|
|
||||||
now = now.Add(debounce)
|
time.Sleep(2100 * time.Millisecond)
|
||||||
|
|
||||||
require.NoError(t, w.Write(
|
require.NoError(t, w.Write(
|
||||||
delivery.ExportArchivedEvent{EventID: "c"}, 0,
|
delivery.ExportArchivedEvent{EventID: "c"}, 0,
|
||||||
@@ -288,31 +288,31 @@ func TestParseArchiveExpiry(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// seedDatabaseTargetDelivery seeds a pending delivery of an event
|
// seedDatabaseTargetDelivery seeds a pending delivery for a
|
||||||
// to a database target and returns the in-memory delivery the
|
// database target with the given config JSON and returns the
|
||||||
// target handler is invoked with.
|
// in-memory delivery the target handler is invoked with.
|
||||||
func seedDatabaseTargetDelivery(
|
func seedDatabaseTargetDelivery(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
webhookDB *gorm.DB,
|
webhookDB *gorm.DB,
|
||||||
event database.Event,
|
event database.Event,
|
||||||
tgt *database.Target,
|
config string,
|
||||||
) *database.Delivery {
|
) *database.Delivery {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
dlv := seedDelivery(
|
dlv := seedDelivery(
|
||||||
t, webhookDB, event.ID, tgt.ID,
|
t, webhookDB, event.ID, uuid.New().String(),
|
||||||
database.DeliveryStatusPending,
|
database.DeliveryStatusPending,
|
||||||
)
|
)
|
||||||
|
|
||||||
d := &database.Delivery{
|
d := &database.Delivery{
|
||||||
EventID: event.ID,
|
EventID: event.ID,
|
||||||
TargetID: tgt.ID,
|
TargetID: dlv.TargetID,
|
||||||
Status: database.DeliveryStatusPending,
|
Status: database.DeliveryStatusPending,
|
||||||
Event: event,
|
Event: event,
|
||||||
Target: database.Target{
|
Target: database.Target{
|
||||||
Name: tgt.Name,
|
Name: "test-db",
|
||||||
Type: database.TargetTypeDatabase,
|
Type: database.TargetTypeDatabase,
|
||||||
Config: tgt.Config,
|
Config: config,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
d.ID = dlv.ID
|
d.ID = dlv.ID
|
||||||
@@ -330,14 +330,22 @@ func TestDeliverDatabase_ArchiveFailureFailsDelivery(
|
|||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
dataDir := t.TempDir()
|
||||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"nonsense"}`)
|
|
||||||
|
e := delivery.NewTestEngineWithDB(
|
||||||
|
nil, database.NewTestWebhookDBManager(dataDir),
|
||||||
|
archiveTestLogger(),
|
||||||
|
&http.Client{Timeout: 5 * time.Second},
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
webhookDB := testWebhookDB(t)
|
||||||
event := seedEvent(t, webhookDB, `{"archived":false}`)
|
event := seedEvent(t, webhookDB, `{"archived":false}`)
|
||||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
|
d := seedDatabaseTargetDelivery(
|
||||||
|
t, webhookDB, event, `{"expiry":"nonsense"}`,
|
||||||
|
)
|
||||||
|
|
||||||
env.eng.ExportDeliverDatabase(webhookDB, d)
|
e.ExportDeliverDatabase(webhookDB, d)
|
||||||
|
|
||||||
var updated database.Delivery
|
var updated database.Delivery
|
||||||
|
|
||||||
@@ -365,7 +373,10 @@ func TestDeliverDatabase_ArchiveFailureFailsDelivery(
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert.NoFileExists(t,
|
assert.NoFileExists(t,
|
||||||
env.archivePath(tgt),
|
filepath.Join(
|
||||||
|
dataDir,
|
||||||
|
fmt.Sprintf("archive-%s.db", event.WebhookID),
|
||||||
|
),
|
||||||
"no archive file should exist for a failed config",
|
"no archive file should exist for a failed config",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -389,290 +400,3 @@ func TestValidateArchiveExpiry(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestArchiveFileName pins the archive file name and the rules
|
|
||||||
// that make a webhook or target name safe to put in it.
|
|
||||||
func TestArchiveFileName(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const id = "3f2a1c9e-8d4b-4c1a-9e2f-0a1b2c3d4e5f"
|
|
||||||
|
|
||||||
cases := []struct {
|
|
||||||
name string
|
|
||||||
webhook string
|
|
||||||
target string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
"plain names", "orders", "archive",
|
|
||||||
"archive-orders-archive-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"lowercased", "Orders", "Main Archive",
|
|
||||||
"archive-orders-main-archive-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"a run of other characters is one dash",
|
|
||||||
`a /\..b`, "c__--d",
|
|
||||||
"archive-a-b-c-d-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"no dash at either end", " --orders!! ", "(archive)",
|
|
||||||
"archive-orders-archive-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"path separators", "../../etc/passwd", "a/b",
|
|
||||||
"archive-etc-passwd-a-b-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"letters outside ASCII are dropped",
|
|
||||||
"Bestellungen Größe", "café",
|
|
||||||
"archive-bestellungen-gr-e-caf-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"nothing left is unnamed", "", "!!!",
|
|
||||||
"archive-unnamed-unnamed-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cut to 40 characters", strings.Repeat("a", 50), "x",
|
|
||||||
"archive-" + strings.Repeat("a", 40) + "-x-" + id + ".db",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"no dash left by the cut",
|
|
||||||
strings.Repeat("a", 39) + " b", "x",
|
|
||||||
"archive-" + strings.Repeat("a", 39) + "-x-" + id + ".db",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range cases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t, tc.want,
|
|
||||||
delivery.ArchiveFileName(tc.webhook, tc.target, id),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestDeliverDatabase_EachTargetHasItsOwnArchive proves two
|
|
||||||
// database targets of one webhook archive into separate files.
|
|
||||||
func TestDeliverDatabase_EachTargetHasItsOwnArchive(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
first := env.seedDatabaseTarget(t, "")
|
|
||||||
second := env.addDatabaseTarget(t, first.WebhookID, "")
|
|
||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
|
||||||
event := seedEvent(t, webhookDB, `{"n":1}`)
|
|
||||||
|
|
||||||
for _, tgt := range []*database.Target{first, second} {
|
|
||||||
env.eng.ExportDeliverDatabase(
|
|
||||||
webhookDB,
|
|
||||||
seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
require.NotEqual(
|
|
||||||
t, env.archivePath(first), env.archivePath(second),
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t, []string{event.ID},
|
|
||||||
archivedEventIDs(t, env.archivePath(first)),
|
|
||||||
)
|
|
||||||
assert.Equal(
|
|
||||||
t, []string{event.ID},
|
|
||||||
archivedEventIDs(t, env.archivePath(second)),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRename_MovesTheFile proves a rename moves the archive, rows
|
|
||||||
// and all, and that later writes go to the new name.
|
|
||||||
func TestRename_MovesTheFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, "")
|
|
||||||
oldPath := env.archivePath(tgt)
|
|
||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
|
||||||
first := seedEvent(t, webhookDB, `{"n":1}`)
|
|
||||||
env.eng.ExportDeliverDatabase(
|
|
||||||
webhookDB, seedDatabaseTargetDelivery(t, webhookDB, first, tgt),
|
|
||||||
)
|
|
||||||
require.FileExists(t, oldPath)
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t, env.eng.Rename(tgt.ID, "Orders", "Long Term"),
|
|
||||||
)
|
|
||||||
|
|
||||||
newPath := filepath.Join(
|
|
||||||
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.NoFileExists(t, oldPath)
|
|
||||||
assert.Equal(t, []string{first.ID}, archivedEventIDs(t, newPath))
|
|
||||||
|
|
||||||
second := seedEvent(t, webhookDB, `{"n":2}`)
|
|
||||||
env.eng.ExportDeliverDatabase(
|
|
||||||
webhookDB,
|
|
||||||
seedDatabaseTargetDelivery(t, webhookDB, second, tgt),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.ElementsMatch(
|
|
||||||
t, []string{first.ID, second.ID},
|
|
||||||
archivedEventIDs(t, newPath),
|
|
||||||
)
|
|
||||||
assert.NoFileExists(
|
|
||||||
t, oldPath, "a write after the rename must use the new name",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRename_NeverReplacesAFile plants a file at the new name, once
|
|
||||||
// the .db alone, once a lone -wal and once a lone -shm, and proves
|
|
||||||
// each time that the rename is refused, the planted file survives,
|
|
||||||
// and the archive keeps its name and its rows.
|
|
||||||
func TestRename_NeverReplacesAFile(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
for _, suffix := range archiveFileSuffixes() {
|
|
||||||
t.Run("planted .db"+suffix, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, "")
|
|
||||||
oldPath := env.archivePath(tgt)
|
|
||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
|
||||||
first := seedEvent(t, webhookDB, `{"n":1}`)
|
|
||||||
env.eng.ExportDeliverDatabase(
|
|
||||||
webhookDB,
|
|
||||||
seedDatabaseTargetDelivery(t, webhookDB, first, tgt),
|
|
||||||
)
|
|
||||||
|
|
||||||
newPath := filepath.Join(
|
|
||||||
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
|
||||||
)
|
|
||||||
plantedPath := newPath + suffix
|
|
||||||
require.NoError(
|
|
||||||
t, os.WriteFile(plantedPath, []byte("planted"), 0o600),
|
|
||||||
)
|
|
||||||
|
|
||||||
require.ErrorIs(
|
|
||||||
t, env.eng.Rename(tgt.ID, "Orders", "Long Term"),
|
|
||||||
delivery.ErrArchiveNameTaken,
|
|
||||||
)
|
|
||||||
|
|
||||||
//nolint:gosec // reads the file the test planted under t.TempDir()
|
|
||||||
planted, err := os.ReadFile(plantedPath)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "planted", string(planted))
|
|
||||||
|
|
||||||
second := seedEvent(t, webhookDB, `{"n":2}`)
|
|
||||||
env.eng.ExportDeliverDatabase(
|
|
||||||
webhookDB,
|
|
||||||
seedDatabaseTargetDelivery(t, webhookDB, second, tgt),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.ElementsMatch(
|
|
||||||
t, []string{first.ID, second.ID},
|
|
||||||
archivedEventIDs(t, oldPath),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestRename_BeforeTheNameIsSaved covers the order the handlers
|
|
||||||
// use: they rename before they save the new name, so a delivery in
|
|
||||||
// between must write under the new name although the main database
|
|
||||||
// still has the old one. It also shows that renaming an archive that
|
|
||||||
// does not exist yet is not an error.
|
|
||||||
func TestRename_BeforeTheNameIsSaved(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupArchiveTest(t)
|
|
||||||
tgt := env.seedDatabaseTarget(t, "")
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t, env.eng.Rename(tgt.ID, "Orders", "Archive"),
|
|
||||||
)
|
|
||||||
|
|
||||||
webhookDB := testWebhookDB(t)
|
|
||||||
event := seedEvent(t, webhookDB, `{"n":1}`)
|
|
||||||
env.eng.ExportDeliverDatabase(
|
|
||||||
webhookDB, seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.FileExists(
|
|
||||||
t,
|
|
||||||
filepath.Join(
|
|
||||||
env.dataDir, "archive-orders-archive-"+tgt.ID+".db",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
assert.NoFileExists(t, env.archivePath(tgt))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveWriter_RenameMovesSidecars proves a rename carries
|
|
||||||
// the -wal and -shm a crash can leave beside an archive no handle
|
|
||||||
// has opened since. SQLite finds them by name, so a -wal left
|
|
||||||
// behind would lose the transactions it holds.
|
|
||||||
func TestArchiveWriter_RenameMovesSidecars(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
oldPath := filepath.Join(dir, "archive-old.db")
|
|
||||||
newPath := filepath.Join(dir, "archive-new.db")
|
|
||||||
|
|
||||||
for _, suffix := range archiveFileSuffixes() {
|
|
||||||
require.NoError(
|
|
||||||
t, os.WriteFile(oldPath+suffix, []byte(suffix), 0o600),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
w := delivery.NewExportArchiveWriter(
|
|
||||||
oldPath, archiveTestLogger(), 0,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, w.Rename("archive-new.db"))
|
|
||||||
|
|
||||||
for _, suffix := range archiveFileSuffixes() {
|
|
||||||
assert.NoFileExists(t, oldPath+suffix)
|
|
||||||
assert.FileExists(t, newPath+suffix)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, newPath, w.Path())
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestArchiveWriter_RenameMovesBackOnFailure makes the -wal fail to
|
|
||||||
// move after the .db has moved, and proves the .db is moved back, so
|
|
||||||
// the archive is never split across two names. The new name is 255
|
|
||||||
// bytes, the longest a file name may be, so the .db can take it but
|
|
||||||
// the -wal, four bytes longer, cannot.
|
|
||||||
func TestArchiveWriter_RenameMovesBackOnFailure(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
dir := t.TempDir()
|
|
||||||
oldPath := filepath.Join(dir, "archive-old.db")
|
|
||||||
newName := strings.Repeat("a", 252) + ".db"
|
|
||||||
|
|
||||||
for _, suffix := range archiveFileSuffixes() {
|
|
||||||
require.NoError(
|
|
||||||
t, os.WriteFile(oldPath+suffix, []byte(suffix), 0o600),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
w := delivery.NewExportArchiveWriter(
|
|
||||||
oldPath, archiveTestLogger(), 0,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.Error(t, w.Rename(newName))
|
|
||||||
|
|
||||||
for _, suffix := range archiveFileSuffixes() {
|
|
||||||
assert.FileExists(t, oldPath+suffix)
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.NoFileExists(t, filepath.Join(dir, newName))
|
|
||||||
assert.Equal(t, oldPath, w.Path())
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -442,9 +442,7 @@ func (t *httpTarget) doHTTPRequest(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
originScoped := applyRequestHeaders(
|
originScoped := applyRequestHeaders(req, event, cfg)
|
||||||
req, event, cfg, t.eng.userAgent(),
|
|
||||||
)
|
|
||||||
|
|
||||||
client := t.clientForRequest(cfg, originScoped)
|
client := t.clientForRequest(cfg, originScoped)
|
||||||
|
|
||||||
@@ -564,13 +562,10 @@ func isForwardableHeader(name string) bool {
|
|||||||
// Content-Type goes out once: a Content-Type configured on the target
|
// Content-Type goes out once: a Content-Type configured on the target
|
||||||
// wins, otherwise the event's ContentType, otherwise none. The inbound
|
// wins, otherwise the event's ContentType, otherwise none. The inbound
|
||||||
// Content-Type in the event's headers is never forwarded.
|
// Content-Type in the event's headers is never forwarded.
|
||||||
//
|
|
||||||
// userAgent is set last, over any configured or inbound User-Agent.
|
|
||||||
func applyRequestHeaders(
|
func applyRequestHeaders(
|
||||||
req *http.Request,
|
req *http.Request,
|
||||||
event *database.Event,
|
event *database.Event,
|
||||||
cfg *HTTPTargetConfig,
|
cfg *HTTPTargetConfig,
|
||||||
userAgent string,
|
|
||||||
) []string {
|
) []string {
|
||||||
if event.ContentType != "" {
|
if event.ContentType != "" {
|
||||||
req.Header.Set(
|
req.Header.Set(
|
||||||
@@ -585,7 +580,7 @@ func applyRequestHeaders(
|
|||||||
originScoped[http.CanonicalHeaderKey(k)] = struct{}{}
|
originScoped[http.CanonicalHeaderKey(k)] = struct{}{}
|
||||||
}
|
}
|
||||||
|
|
||||||
req.Header.Set("User-Agent", userAgent)
|
req.Header.Set("User-Agent", "webhooker/1.0")
|
||||||
|
|
||||||
// A Content-Type configured on the target describes the body
|
// A Content-Type configured on the target describes the body
|
||||||
// being sent rather than the sender. A 307/308 preserves the
|
// being sent rather than the sender. A 307/308 preserves the
|
||||||
|
|||||||
@@ -136,7 +136,7 @@ func (t *slackTarget) attempt(
|
|||||||
}
|
}
|
||||||
|
|
||||||
req.Header.Set("Content-Type", "application/json")
|
req.Header.Set("Content-Type", "application/json")
|
||||||
req.Header.Set("User-Agent", t.eng.userAgent())
|
req.Header.Set("User-Agent", "webhooker/1.0")
|
||||||
|
|
||||||
resp, doErr := executeHTTPRequest(t.client, req)
|
resp, doErr := executeHTTPRequest(t.client, req)
|
||||||
durationMs := time.Since(start).Milliseconds()
|
durationMs := time.Since(start).Milliseconds()
|
||||||
|
|||||||
@@ -418,38 +418,6 @@ func TestProcessRetryTask_TargetDeleted_MakesNoAttempt(
|
|||||||
assert.Zero(t, s.Engine.ExportInflightHeld())
|
assert.Zero(t, s.Engine.ExportInflightHeld())
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestProcessRetryTask_TargetDeleted_CountsFailureOnTarget verifies
|
|
||||||
// that the failure of a retry abandoned because its target is gone is
|
|
||||||
// added to that target's own totals, not to a row with no target.
|
|
||||||
func TestProcessRetryTask_TargetDeleted_CountsFailureOnTarget(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
s := newISetup(t)
|
|
||||||
|
|
||||||
var hits atomic.Int64
|
|
||||||
|
|
||||||
task, targetID := tRetryChainSetup(
|
|
||||||
t, s, "gone-counted", &hits,
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(t, s.MainDB.Delete(
|
|
||||||
&database.Target{}, "id = ?", targetID,
|
|
||||||
).Error)
|
|
||||||
|
|
||||||
s.Engine.ExportProcessRetryTask(
|
|
||||||
context.Background(), &task,
|
|
||||||
)
|
|
||||||
|
|
||||||
var rows []database.TargetTotals
|
|
||||||
|
|
||||||
require.NoError(t, s.WebhookDB.Find(&rows).Error)
|
|
||||||
assert.Equal(t, []database.TargetTotals{
|
|
||||||
{TargetID: targetID, Failed: 1},
|
|
||||||
}, rows)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestProcessRetryTask_TargetPresent_StillDelivers is the guard's
|
// TestProcessRetryTask_TargetPresent_StillDelivers is the guard's
|
||||||
// mutation check: a liveness check that refused every retry would pass
|
// mutation check: a liveness check that refused every retry would pass
|
||||||
// the test above and break every retry there is.
|
// the test above and break every retry there is.
|
||||||
|
|||||||
@@ -179,27 +179,6 @@ func TestDoHTTPRequest_TransportErrorMasksURL(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestDoHTTPRequest_UnparsableURLIsMasked is the same for an HTTP
|
|
||||||
// target URL that no request can be built from.
|
|
||||||
func TestDoHTTPRequest_UnparsableURLIsMasked(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
e := testEngine(t, 1)
|
|
||||||
|
|
||||||
statusCode, _, _, reqErr := e.ExportDoHTTPRequest(
|
|
||||||
context.TODO(),
|
|
||||||
&delivery.HTTPTargetConfig{
|
|
||||||
URL: "https://hooks.example.com" + maskSecretPath + "\n",
|
|
||||||
},
|
|
||||||
&database.Event{},
|
|
||||||
)
|
|
||||||
require.Error(t, reqErr)
|
|
||||||
assert.Zero(t, statusCode)
|
|
||||||
|
|
||||||
assertNoCredential(t, reqErr.Error())
|
|
||||||
assert.Contains(t, reqErr.Error(), "invalid control character")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestValidateTargetURL_UnparsableURLIsMasked proves the SSRF
|
// TestValidateTargetURL_UnparsableURLIsMasked proves the SSRF
|
||||||
// validator's error does not carry the submitted URL, which
|
// validator's error does not carry the submitted URL, which
|
||||||
// the handler both logs and shows.
|
// the handler both logs and shows.
|
||||||
|
|||||||
@@ -1,91 +0,0 @@
|
|||||||
package delivery_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/netip"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/google/uuid"
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"go.uber.org/fx/fxtest"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
"sneak.berlin/go/webhooker/internal/globals"
|
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
|
||||||
"sneak.berlin/go/webhooker/internal/metrics"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Both the http and the slack target send webhooker/ and the version
|
|
||||||
// in Globals, the value the web UI footer shows. A User-Agent
|
|
||||||
// configured on the target or carried in by the sender does not
|
|
||||||
// replace it.
|
|
||||||
func TestUserAgent_IsTheBuildVersion(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const want = "webhooker/1.2.3-test"
|
|
||||||
|
|
||||||
userAgents := make(chan string, 1)
|
|
||||||
|
|
||||||
ts := httptest.NewServer(http.HandlerFunc(
|
|
||||||
func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
userAgents <- r.Header.Get("User-Agent")
|
|
||||||
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
},
|
|
||||||
))
|
|
||||||
defer ts.Close()
|
|
||||||
|
|
||||||
g := &globals.Globals{Version: "1.2.3-test"}
|
|
||||||
lc := fxtest.NewLifecycle(t)
|
|
||||||
|
|
||||||
log, err := logger.New(lc, logger.LoggerParams{Globals: g})
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
e := delivery.New(lc, delivery.EngineParams{
|
|
||||||
Globals: g,
|
|
||||||
Logger: log,
|
|
||||||
// httptest listens on loopback, which the default guard
|
|
||||||
// refuses.
|
|
||||||
SSRFGuard: delivery.NewTestGuard(
|
|
||||||
netip.MustParsePrefix("127.0.0.0/8"),
|
|
||||||
),
|
|
||||||
Metrics: metrics.New(prometheus.NewRegistry()),
|
|
||||||
})
|
|
||||||
|
|
||||||
statusCode, _, _, err := e.ExportDoHTTPRequest(
|
|
||||||
context.Background(),
|
|
||||||
&delivery.HTTPTargetConfig{
|
|
||||||
URL: ts.URL,
|
|
||||||
Headers: map[string]string{"User-Agent": "configured/1"},
|
|
||||||
},
|
|
||||||
&database.Event{Headers: `{"User-Agent":["curl/8"]}`},
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, http.StatusOK, statusCode)
|
|
||||||
require.Len(t, userAgents, 1, "the http target sent no request")
|
|
||||||
assert.Equal(t, want, <-userAgents, "http target")
|
|
||||||
|
|
||||||
db := testWebhookDB(t)
|
|
||||||
targetID := uuid.New().String()
|
|
||||||
|
|
||||||
slackCfg, err := json.Marshal(
|
|
||||||
delivery.SlackTargetConfig{WebhookURL: ts.URL},
|
|
||||||
)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
event := seedEvent(t, db, `{"action":"test"}`)
|
|
||||||
dlv := seedDelivery(
|
|
||||||
t, db, event.ID, targetID, database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
|
|
||||||
e.ExportDeliverSlack(context.Background(), db, buildSlackDelivery(
|
|
||||||
dlv, event, targetID, "test-slack", string(slackCfg),
|
|
||||||
))
|
|
||||||
require.Len(t, userAgents, 1, "the slack target sent no request")
|
|
||||||
assert.Equal(t, want, <-userAgents, "slack target")
|
|
||||||
}
|
|
||||||
@@ -117,8 +117,8 @@ func readFirstBootSecrets(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// bootAtDebug starts and stops the real application graph against
|
// bootAtDebug starts and stops the real application graph against
|
||||||
// dataDir with DEBUG=true and nothing else set, and returns everything
|
// dataDir with DEBUG=true, and returns everything it wrote to standard
|
||||||
// it wrote to standard output.
|
// output.
|
||||||
//
|
//
|
||||||
// config.New reads DEBUG from the environment exactly as the binary
|
// config.New reads DEBUG from the environment exactly as the binary
|
||||||
// does, internal/logger builds the handler it builds in production,
|
// does, internal/logger builds the handler it builds in production,
|
||||||
@@ -128,7 +128,6 @@ func readFirstBootSecrets(
|
|||||||
func bootAtDebug(t *testing.T, dataDir string) string {
|
func bootAtDebug(t *testing.T, dataDir string) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
config.ClearEnvForTest(t)
|
|
||||||
t.Setenv("DEBUG", "true")
|
t.Setenv("DEBUG", "true")
|
||||||
t.Setenv("DATA_DIR", dataDir)
|
t.Setenv("DATA_DIR", dataDir)
|
||||||
|
|
||||||
@@ -138,10 +137,6 @@ func bootAtDebug(t *testing.T, dataDir string) string {
|
|||||||
|
|
||||||
app := fxtest.New(
|
app := fxtest.New(
|
||||||
t,
|
t,
|
||||||
// fx's own log is discarded, not sent to t.Logf: a hook still
|
|
||||||
// running after a start or stop timeout would write there after
|
|
||||||
// the test has returned.
|
|
||||||
fx.NopLogger,
|
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
|
|||||||
@@ -7,7 +7,7 @@
|
|||||||
// SQL — parameters and all — for every statement that returns an
|
// SQL — parameters and all — for every statement that returns an
|
||||||
// error, including gorm.ErrRecordNotFound. Two of this service's
|
// error, including gorm.ErrRecordNotFound. Two of this service's
|
||||||
// lookups miss by design on unauthenticated routes: the entrypoint
|
// lookups miss by design on unauthenticated routes: the entrypoint
|
||||||
// lookup on /h/{uuid}, whose path segment the client picks
|
// lookup on /webhook/{uuid}, whose path segment the client picks
|
||||||
// outright, and the user lookup behind the login form, whose username
|
// outright, and the user lookup behind the login form, whose username
|
||||||
// the client picks outright. Under the default logger each of those
|
// the client picks outright. Under the default logger each of those
|
||||||
// misses printed an unbounded, attacker-chosen string, at no level the
|
// misses printed an unbounded, attacker-chosen string, at no level the
|
||||||
@@ -111,9 +111,9 @@ func (l *Logger) LogMode(gormlogger.LogLevel) gormlogger.Interface {
|
|||||||
//
|
//
|
||||||
// One GORM path does not consult this: (*gorm.DB).Scan records the
|
// One GORM path does not consult this: (*gorm.DB).Scan records the
|
||||||
// statement through gorm's own traceRecorder, which does not implement
|
// statement through gorm's own traceRecorder, which does not implement
|
||||||
// this interface. No production code path calls it; only tests do, and
|
// this interface. No production code path calls it; its one caller is
|
||||||
// what a test binds is fixture data. scan_guard_test.go fails if a
|
// internal/database/database_test.go:91, whose SELECT 1 binds nothing.
|
||||||
// non-test file calls it.
|
// scan_guard_test.go fails if a non-test file calls it.
|
||||||
// (*gorm.DB).Pluck, Row and Raw all run through the normal callback
|
// (*gorm.DB).Pluck, Row and Raw all run through the normal callback
|
||||||
// processor and are filtered.
|
// processor and are filtered.
|
||||||
func (l *Logger) ParamsFilter(
|
func (l *Logger) ParamsFilter(
|
||||||
|
|||||||
@@ -14,16 +14,18 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
// isRowProducer reports whether name is GORM's Row or database/sql's
|
// minNonTestFiles guards the walk below against passing because it
|
||||||
// QueryRow or QueryRowContext, which return a *sql.Row whose Scan is
|
// found nothing to look at. The tree held 60 non-test .go files when
|
||||||
// database/sql's and not (*gorm.DB).Scan. GORM's Rows is not listed:
|
// this was written.
|
||||||
// it also returns an error, so Scan is never called on its result
|
const minNonTestFiles = 40
|
||||||
// directly. It matches the method name only and resolves no types, so
|
|
||||||
// a repo-local method with one of these names that returns *gorm.DB
|
// isRowProducer reports whether name is a method that returns a
|
||||||
// gets past it: Scan on that method's result is not reported.
|
// database/sql row handle. GORM's Row and Rows return *sql.Row and
|
||||||
|
// *sql.Rows, so Scan on the result of one of them is database/sql's
|
||||||
|
// Scan and never (*gorm.DB).Scan.
|
||||||
func isRowProducer(name string) bool {
|
func isRowProducer(name string) bool {
|
||||||
switch name {
|
switch name {
|
||||||
case "Row", "QueryRow", "QueryRowContext":
|
case "Row", "Rows", "QueryRow", "QueryRowContext":
|
||||||
return true
|
return true
|
||||||
default:
|
default:
|
||||||
return false
|
return false
|
||||||
@@ -48,14 +50,9 @@ func receiverIsRowHandle(x ast.Expr) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// unguardedScans returns the position of every Scan call in file whose
|
// unguardedScans returns the position of every Scan call in file whose
|
||||||
// receiver is not a call to a row producer. It fails closed: any other
|
// receiver is not a row handle. It fails closed: a receiver it cannot
|
||||||
// receiver — a local variable, a struct field, a call to any other
|
// resolve syntactically — a local variable, a struct field — is
|
||||||
// method — is reported rather than assumed safe.
|
// reported rather than assumed safe.
|
||||||
//
|
|
||||||
// It sees only calls written x.Scan(...). A method value, f := db.Scan
|
|
||||||
// followed by f(&v), is out of scope: Scan is never the called
|
|
||||||
// expression there, and nobody writes a query that way by accident,
|
|
||||||
// which is the mistake this check exists to catch.
|
|
||||||
func unguardedScans(
|
func unguardedScans(
|
||||||
fset *token.FileSet, file *ast.File,
|
fset *token.FileSet, file *ast.File,
|
||||||
) []token.Position {
|
) []token.Position {
|
||||||
@@ -114,15 +111,15 @@ func skipDir(name string) bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// walkNonTestGo parses every non-test .go file under root. It returns
|
// walkNonTestGo parses every non-test .go file under root and returns
|
||||||
// the directories, relative to root, it parsed a file in, along with
|
// how many it parsed along with every unguarded Scan it found.
|
||||||
// every unguarded Scan it found.
|
func walkNonTestGo(t *testing.T, root string) (int, []string) {
|
||||||
func walkNonTestGo(t *testing.T, root string) (map[string]bool, []string) {
|
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
walked := map[string]bool{}
|
var (
|
||||||
|
parsed int
|
||||||
var hits []string
|
hits []string
|
||||||
|
)
|
||||||
|
|
||||||
fset := token.NewFileSet()
|
fset := token.NewFileSet()
|
||||||
|
|
||||||
@@ -150,12 +147,7 @@ func walkNonTestGo(t *testing.T, root string) (map[string]bool, []string) {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
dir, err := filepath.Rel(root, filepath.Dir(path))
|
parsed++
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
walked[dir] = true
|
|
||||||
|
|
||||||
for _, pos := range unguardedScans(fset, file) {
|
for _, pos := range unguardedScans(fset, file) {
|
||||||
hits = append(hits, relPosition(root, pos))
|
hits = append(hits, relPosition(root, pos))
|
||||||
@@ -165,7 +157,7 @@ func walkNonTestGo(t *testing.T, root string) (map[string]bool, []string) {
|
|||||||
},
|
},
|
||||||
))
|
))
|
||||||
|
|
||||||
return walked, hits
|
return parsed, hits
|
||||||
}
|
}
|
||||||
|
|
||||||
// isNonTestGo reports whether a file name is Go source this check
|
// isNonTestGo reports whether a file name is Go source this check
|
||||||
@@ -197,39 +189,19 @@ func relPosition(root string, pos token.Position) string {
|
|||||||
// logged with its values interpolated. The package comment states the
|
// logged with its values interpolated. The package comment states the
|
||||||
// limit; this fails when someone adds a call site anyway.
|
// limit; this fails when someone adds a call site anyway.
|
||||||
//
|
//
|
||||||
// Test files are not governed: what a test binds is fixture data.
|
// The current tree has one caller, internal/database/database_test.go,
|
||||||
|
// which this check does not govern: it is test-only and its SELECT 1
|
||||||
|
// binds nothing.
|
||||||
func TestGormScanIsNeverCalledOutsideTests(t *testing.T) {
|
func TestGormScanIsNeverCalledOutsideTests(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
root := moduleRoot(t)
|
parsed, offenders := walkNonTestGo(t, moduleRoot(t))
|
||||||
walked, offenders := walkNonTestGo(t, root)
|
|
||||||
|
|
||||||
// The module's packages are static, templates, and every directory
|
|
||||||
// directly under cmd and internal. Each holds non-test code, so one
|
|
||||||
// the walk parsed nothing in was skipped, and a Scan there would
|
|
||||||
// pass unseen.
|
|
||||||
packages := []string{"static", "templates"}
|
|
||||||
|
|
||||||
for _, parent := range []string{"cmd", "internal"} {
|
|
||||||
entries, err := os.ReadDir(filepath.Join(root, parent))
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
for _, entry := range entries {
|
|
||||||
if !entry.IsDir() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
packages = append(packages, filepath.Join(parent, entry.Name()))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, dir := range packages {
|
|
||||||
require.True(
|
|
||||||
t, walked[dir],
|
|
||||||
"the walk parsed no non-test .go file in %s", dir,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
|
require.GreaterOrEqual(
|
||||||
|
t, parsed, minNonTestFiles,
|
||||||
|
"parsed %d non-test .go files, so this check found "+
|
||||||
|
"nothing to look at", parsed,
|
||||||
|
)
|
||||||
require.Empty(
|
require.Empty(
|
||||||
t, offenders,
|
t, offenders,
|
||||||
"Scan called on a receiver this check cannot show is a "+
|
"Scan called on a receiver this check cannot show is a "+
|
||||||
@@ -250,51 +222,18 @@ type scanGuardCase struct {
|
|||||||
want int
|
want int
|
||||||
}
|
}
|
||||||
|
|
||||||
// scanGuardCases covers each receiver form unguardedScans names, plus
|
|
||||||
// each row producer isRowProducer lets through. Each body is valid Go
|
|
||||||
// inside plantedFile.
|
|
||||||
func scanGuardCases() []scanGuardCase {
|
func scanGuardCases() []scanGuardCase {
|
||||||
return []scanGuardCase{
|
return []scanGuardCase{
|
||||||
{"local variable", "q := gdb.Raw(\"SELECT 1\")\n\tq.Scan(&v)", 1},
|
{"gorm chain", `db.DB().Raw("SELECT 1").Scan(&v)`, 1},
|
||||||
{"struct field", `s.db.Scan(&v)`, 1},
|
{"gorm receiver", `gdb.Scan(&v)`, 1},
|
||||||
{"gorm chain", `gdb.Raw("SELECT 1").Scan(&v)`, 1},
|
{"gorm via variable", "q := gdb.Raw(\"x\")\nq.Scan(&v)", 1},
|
||||||
{
|
{"gorm model chain", `gdb.Model(&x).Scan(&v)`, 1},
|
||||||
"sql rows in a variable",
|
{"sql row", `gdb.Raw("SELECT 1").Row().Scan(&v)`, 0},
|
||||||
"rows, _ := gdb.Raw(\"SELECT 1\").Rows()\n\trows.Scan(&v)",
|
{"sql rows", `gdb.Raw("SELECT 1").Rows().Scan(&v)`, 0},
|
||||||
1,
|
|
||||||
},
|
|
||||||
{"gorm Row", `gdb.Raw("SELECT 1").Row().Scan(&v)`, 0},
|
|
||||||
{"sql QueryRow", `sqlDB.QueryRow("SELECT 1").Scan(&v)`, 0},
|
|
||||||
{
|
|
||||||
"sql QueryRowContext",
|
|
||||||
`sqlDB.QueryRowContext(ctx, "SELECT 1").Scan(&v)`,
|
|
||||||
0,
|
|
||||||
},
|
|
||||||
{"unrelated call", `gdb.Find(&v)`, 0},
|
{"unrelated call", `gdb.Find(&v)`, 0},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// plantedFile wraps one case body in a function that declares every
|
|
||||||
// name the bodies use, so each body is the Go it stands for. The result
|
|
||||||
// is parsed, never compiled.
|
|
||||||
const plantedFile = `package p
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"database/sql"
|
|
||||||
|
|
||||||
"gorm.io/gorm"
|
|
||||||
)
|
|
||||||
|
|
||||||
type store struct{ db *gorm.DB }
|
|
||||||
|
|
||||||
func f(ctx context.Context, gdb *gorm.DB, sqlDB *sql.DB, s store) {
|
|
||||||
var v int
|
|
||||||
|
|
||||||
%s
|
|
||||||
}
|
|
||||||
`
|
|
||||||
|
|
||||||
// TestScanGuard_ReportsPlantedCalls proves the check fires. Without it
|
// TestScanGuard_ReportsPlantedCalls proves the check fires. Without it
|
||||||
// a detector that matched nothing would satisfy the walk above no
|
// a detector that matched nothing would satisfy the walk above no
|
||||||
// matter what the tree contained.
|
// matter what the tree contained.
|
||||||
@@ -306,7 +245,9 @@ func TestScanGuard_ReportsPlantedCalls(t *testing.T) {
|
|||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
fset := token.NewFileSet()
|
fset := token.NewFileSet()
|
||||||
src := fmt.Sprintf(plantedFile, tc.body)
|
src := fmt.Sprintf(
|
||||||
|
"package p\n\nfunc f() {\n\t%s\n}\n", tc.body,
|
||||||
|
)
|
||||||
|
|
||||||
file, err := parser.ParseFile(
|
file, err := parser.ParseFile(
|
||||||
fset, tc.name+".go", src, 0,
|
fset, tc.name+".go", src, 0,
|
||||||
|
|||||||
+33
-63
@@ -2,56 +2,19 @@ package handlers
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
|
||||||
"unicode"
|
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/logfield"
|
"sneak.berlin/go/webhooker/internal/logfield"
|
||||||
"sneak.berlin/go/webhooker/internal/middleware"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// loginDestination returns where a successful login sends the
|
|
||||||
// browser: next when it is a path on this site, otherwise "/", which
|
|
||||||
// leads to the webhook list.
|
|
||||||
//
|
|
||||||
// A browser reads "//host" as another site, reads "\" as "/", and
|
|
||||||
// drops tabs and newlines before reading at all. So the value must
|
|
||||||
// start with exactly one "/" and hold no "\" or control character
|
|
||||||
// anywhere: http.Redirect cleans "/a/../\host" down to "/\host". It
|
|
||||||
// is checked after percent-decoding, so an encoded form of any of
|
|
||||||
// these is refused too.
|
|
||||||
func loginDestination(next string) string {
|
|
||||||
if len(next) > middleware.MaxNextBytes {
|
|
||||||
return "/"
|
|
||||||
}
|
|
||||||
|
|
||||||
decoded, err := url.PathUnescape(next)
|
|
||||||
if err != nil ||
|
|
||||||
!strings.HasPrefix(decoded, "/") ||
|
|
||||||
strings.HasPrefix(decoded, "//") ||
|
|
||||||
strings.Contains(decoded, `\`) ||
|
|
||||||
strings.ContainsFunc(decoded, unicode.IsControl) {
|
|
||||||
return "/"
|
|
||||||
}
|
|
||||||
|
|
||||||
return next
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleLoginPage returns a handler for the login page (GET)
|
// HandleLoginPage returns a handler for the login page (GET)
|
||||||
func (h *Handlers) HandleLoginPage() http.HandlerFunc {
|
func (h *Handlers) HandleLoginPage() http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
next := loginDestination(
|
|
||||||
r.URL.Query().Get(middleware.NextParam),
|
|
||||||
)
|
|
||||||
|
|
||||||
// Check if already logged in
|
// Check if already logged in
|
||||||
sess, err := h.session.Get(r)
|
sess, err := h.session.Get(r)
|
||||||
if err == nil && h.session.IsAuthenticated(sess) {
|
if err == nil && h.session.IsAuthenticated(sess) {
|
||||||
http.Redirect( //nolint:gosec // checked by loginDestination
|
http.Redirect(w, r, "/", http.StatusSeeOther)
|
||||||
w, r, next, http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -59,7 +22,6 @@ func (h *Handlers) HandleLoginPage() http.HandlerFunc {
|
|||||||
// Render login page
|
// Render login page
|
||||||
data := map[string]any{
|
data := map[string]any{
|
||||||
tmplKeyError: "",
|
tmplKeyError: "",
|
||||||
tmplKeyNext: next,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
h.renderTemplate(w, r, "login.html", data)
|
h.renderTemplate(w, r, "login.html", data)
|
||||||
@@ -74,7 +36,7 @@ func (h *Handlers) HandleLoginSubmit() http.HandlerFunc {
|
|||||||
err := r.ParseForm()
|
err := r.ParseForm()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.log.Error("failed to parse form", "error", err)
|
h.log.Error("failed to parse form", "error", err)
|
||||||
h.renderError(w, r, http.StatusBadRequest)
|
http.Error(w, "Bad request", http.StatusBadRequest)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -115,13 +77,8 @@ func (h *Handlers) HandleLoginSubmit() http.HandlerFunc {
|
|||||||
"user_id", user.ID,
|
"user_id", user.ID,
|
||||||
)
|
)
|
||||||
|
|
||||||
// The form value is the client's to set, so it is checked
|
// Redirect to home page
|
||||||
// again here rather than trusted from the rendered page.
|
http.Redirect(w, r, "/", http.StatusSeeOther)
|
||||||
http.Redirect( //nolint:gosec // checked by loginDestination
|
|
||||||
w, r,
|
|
||||||
loginDestination(r.PostFormValue(middleware.NextParam)),
|
|
||||||
http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -134,12 +91,10 @@ func (h *Handlers) renderLoginError(
|
|||||||
) {
|
) {
|
||||||
data := map[string]any{
|
data := map[string]any{
|
||||||
tmplKeyError: msg,
|
tmplKeyError: msg,
|
||||||
tmplKeyNext: loginDestination(
|
|
||||||
r.PostFormValue(middleware.NextParam),
|
|
||||||
),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
h.renderTemplateStatus(w, r, "login.html", data, status)
|
w.WriteHeader(status)
|
||||||
|
h.renderTemplate(w, r, "login.html", data)
|
||||||
}
|
}
|
||||||
|
|
||||||
// authenticateUser looks up and verifies a user's credentials.
|
// authenticateUser looks up and verifies a user's credentials.
|
||||||
@@ -148,10 +103,9 @@ func (h *Handlers) renderLoginError(
|
|||||||
// The credential check runs BEFORE any rate-limit budget is
|
// The credential check runs BEFORE any rate-limit budget is
|
||||||
// consulted, and only a failed check spends budget. That is what
|
// consulted, and only a failed check spends budget. That is what
|
||||||
// keeps the single administrative path reachable: behind the reverse
|
// keeps the single administrative path reachable: behind the reverse
|
||||||
// proxy this deployment requires, when TRUSTED_PROXIES does not cover
|
// proxy this deployment requires, with TRUSTED_PROXIES unset, every
|
||||||
// it, every client shares one bucket, so a limiter spent on arrival
|
// client shares one bucket, so a limiter spent on arrival lets any
|
||||||
// lets any stranger deny the operator's own correct password
|
// stranger deny the operator's own correct password indefinitely.
|
||||||
// indefinitely.
|
|
||||||
//
|
//
|
||||||
// Verifying first means every login POST costs an Argon2id hash, so
|
// Verifying first means every login POST costs an Argon2id hash, so
|
||||||
// the work is taken under a bounded number of verification slots.
|
// the work is taken under a bounded number of verification slots.
|
||||||
@@ -211,7 +165,11 @@ func (h *Handlers) authenticateUser(
|
|||||||
|
|
||||||
valid, err := database.VerifyPassword(password, user.Password)
|
valid, err := database.VerifyPassword(password, user.Password)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.serverError(w, r, "failed to verify password", err)
|
h.log.Error("failed to verify password", "error", err)
|
||||||
|
http.Error(
|
||||||
|
w, "Internal server error",
|
||||||
|
http.StatusInternalServerError,
|
||||||
|
)
|
||||||
|
|
||||||
return user, err
|
return user, err
|
||||||
}
|
}
|
||||||
@@ -283,14 +241,24 @@ func (h *Handlers) createAuthenticatedSession(
|
|||||||
) error {
|
) error {
|
||||||
oldSess, err := h.session.Get(r)
|
oldSess, err := h.session.Get(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.serverError(w, r, "failed to get session", err)
|
h.log.Error("failed to get session", "error", err)
|
||||||
|
http.Error(
|
||||||
|
w, "Internal server error",
|
||||||
|
http.StatusInternalServerError,
|
||||||
|
)
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
sess, err := h.session.Regenerate(r, w, oldSess)
|
sess, err := h.session.Regenerate(r, w, oldSess)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.serverError(w, r, "failed to regenerate session", err)
|
h.log.Error(
|
||||||
|
"failed to regenerate session", "error", err,
|
||||||
|
)
|
||||||
|
http.Error(
|
||||||
|
w, "Internal server error",
|
||||||
|
http.StatusInternalServerError,
|
||||||
|
)
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -299,7 +267,11 @@ func (h *Handlers) createAuthenticatedSession(
|
|||||||
|
|
||||||
err = h.session.Save(r, w, sess)
|
err = h.session.Save(r, w, sess)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.serverError(w, r, "failed to save session", err)
|
h.log.Error("failed to save session", "error", err)
|
||||||
|
http.Error(
|
||||||
|
w, "Internal server error",
|
||||||
|
http.StatusInternalServerError,
|
||||||
|
)
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -332,9 +304,7 @@ func (h *Handlers) HandleLogout() http.HandlerFunc {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
http.Redirect(
|
// Redirect to login page
|
||||||
w, r, withNotice("/pages/login", signedOut),
|
http.Redirect(w, r, "/pages/login", http.StatusSeeOther)
|
||||||
http.StatusSeeOther,
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package handlers_test
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"html/template"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"net/url"
|
"net/url"
|
||||||
@@ -26,7 +25,7 @@ const (
|
|||||||
|
|
||||||
// sharedProxyPeer is the whole point of this file. Production is
|
// sharedProxyPeer is the whole point of this file. Production is
|
||||||
// required to run behind a TLS-terminating reverse proxy, and
|
// required to run behind a TLS-terminating reverse proxy, and
|
||||||
// when TRUSTED_PROXIES does not cover it every client — attacker
|
// TRUSTED_PROXIES defaults to empty, so every client — attacker
|
||||||
// and operator alike — reaches the process from the proxy's
|
// and operator alike — reaches the process from the proxy's
|
||||||
// address and shares one rate-limit bucket. Both parties in
|
// address and shares one rate-limit bucket. Both parties in
|
||||||
// these tests therefore use the same RemoteAddr.
|
// these tests therefore use the same RemoteAddr.
|
||||||
@@ -116,11 +115,11 @@ func floodFailures(
|
|||||||
// done-criterion of https://git.eeqj.de/sneak/webhooker/issues/150.
|
// done-criterion of https://git.eeqj.de/sneak/webhooker/issues/150.
|
||||||
//
|
//
|
||||||
// The attacker and the operator share one rate-limit bucket, because
|
// The attacker and the operator share one rate-limit bucket, because
|
||||||
// behind the mandated reverse proxy, when TRUSTED_PROXIES does not
|
// behind the mandated reverse proxy with TRUSTED_PROXIES unset every
|
||||||
// cover it, every client keys on the proxy's address. The attacker
|
// client keys on the proxy's address. The attacker floods the
|
||||||
// floods the operator's own username — a single-admin product has a
|
// operator's own username — a single-admin product has a predictable
|
||||||
// predictable one — far past the failure limit. The operator must
|
// one — far past the failure limit. The operator must still be able
|
||||||
// still be able to log in with the correct password.
|
// to log in with the correct password.
|
||||||
//
|
//
|
||||||
// This fails if credentials stop being verified ahead of the limiter.
|
// This fails if credentials stop being verified ahead of the limiter.
|
||||||
func TestLogin_StrangersFloodCannotLockOutTheOperator(t *testing.T) {
|
func TestLogin_StrangersFloodCannotLockOutTheOperator(t *testing.T) {
|
||||||
@@ -405,60 +404,6 @@ func TestLogin_MissingCredentialsRejectedBeforeAnyHash(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestLogin_FormErrorAnswersItsStatusWithThePage proves that the login
|
|
||||||
// form shown again with an error still answers 400 with the whole page.
|
|
||||||
func TestLogin_FormErrorAnswersItsStatusWithThePage(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var h *handlers.Handlers
|
|
||||||
|
|
||||||
app := newTestApp(t, &h)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
w := submitLogin(h, sharedProxyPeer, "", "")
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusBadRequest, w.Code)
|
|
||||||
assert.Contains(
|
|
||||||
t, w.Body.String(), "Username and password are required",
|
|
||||||
)
|
|
||||||
assert.Contains(
|
|
||||||
t, w.Body.String(), "</html>",
|
|
||||||
"the page must render to completion",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLogin_FormErrorRenderFailureAnswers500 proves that a login form
|
|
||||||
// error page whose template fails answers 500 with the error page and
|
|
||||||
// none of the form page, rather than the 400 it meant to send.
|
|
||||||
func TestLogin_FormErrorRenderFailureAnswers500(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var h *handlers.Handlers
|
|
||||||
|
|
||||||
app := newTestApp(t, &h)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
// The page prints its error message and then fails.
|
|
||||||
h.AddTemplateForTest("login.html", template.Must(
|
|
||||||
template.New("login").Funcs(template.FuncMap{
|
|
||||||
"fail": func() (string, error) { return "", errMidRender },
|
|
||||||
}).Parse(`{{.Error}}{{fail}}`),
|
|
||||||
))
|
|
||||||
|
|
||||||
w := submitLogin(h, sharedProxyPeer, "", "")
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusInternalServerError, w.Code)
|
|
||||||
assert.NotContains(
|
|
||||||
t, w.Body.String(), "Username and password are required",
|
|
||||||
"the response must carry no part of the aborted page",
|
|
||||||
)
|
|
||||||
assert.Contains(t, w.Body.String(), "500 Internal Server Error")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLogin_SuccessCreatesSession is the control for the tests above:
|
// TestLogin_SuccessCreatesSession is the control for the tests above:
|
||||||
// the success path they assert on really does authenticate.
|
// the success path they assert on really does authenticate.
|
||||||
func TestLogin_SuccessCreatesSession(t *testing.T) {
|
func TestLogin_SuccessCreatesSession(t *testing.T) {
|
||||||
@@ -509,202 +454,6 @@ func TestLogin_SuccessCreatesSession(t *testing.T) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestLogin_ReturnsOnlyToAPathOnThisSite is the security half of
|
|
||||||
// https://git.eeqj.de/sneak/webhooker/issues/384: the page a login
|
|
||||||
// returns to is client-chosen, so anything that is not a path on this
|
|
||||||
// site, plain or percent-encoded, must land on "/", the webhook list.
|
|
||||||
func TestLogin_ReturnsOnlyToAPathOnThisSite(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
db *database.Database
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &db)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
seedOperator(t, db)
|
|
||||||
|
|
||||||
cases := []struct{ next, want string }{
|
|
||||||
{"/hook/abc/events?page=2", "/hook/abc/events?page=2"},
|
|
||||||
{"", "/"},
|
|
||||||
{"https://evil.example/", "/"},
|
|
||||||
{"https%3A%2F%2Fevil.example%2F", "/"},
|
|
||||||
{"//evil.example/", "/"},
|
|
||||||
{"%2F%2Fevil.example/", "/"},
|
|
||||||
{"/%2Fevil.example/", "/"},
|
|
||||||
{`/\evil.example/`, "/"},
|
|
||||||
{"%2F%5Cevil.example/", "/"},
|
|
||||||
{"/%5Cevil.example/", "/"},
|
|
||||||
{`/a/../\evil.example/`, "/"},
|
|
||||||
{"/\t/evil.example/", "/"},
|
|
||||||
{"/%09/evil.example/", "/"},
|
|
||||||
{"/\n/evil.example/", "/"},
|
|
||||||
{"/%0A/evil.example/", "/"},
|
|
||||||
{"/\r/evil.example/", "/"},
|
|
||||||
{"/%0D/evil.example/", "/"},
|
|
||||||
{"/%00/evil.example/", "/"},
|
|
||||||
{"/%7F/evil.example/", "/"},
|
|
||||||
{"%252F%252Fevil.example/", "/"},
|
|
||||||
{"https%253A%252F%252Fevil.example%252F", "/"},
|
|
||||||
{"/" + strings.Repeat("a", 4096), "/"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, c := range cases {
|
|
||||||
form := url.Values{}
|
|
||||||
form.Set("username", operatorUser)
|
|
||||||
form.Set("password", operatorPassword)
|
|
||||||
form.Set("next", c.next)
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(
|
|
||||||
context.Background(),
|
|
||||||
http.MethodPost,
|
|
||||||
"/pages/login",
|
|
||||||
strings.NewReader(form.Encode()),
|
|
||||||
)
|
|
||||||
req.Header.Set(
|
|
||||||
"Content-Type", "application/x-www-form-urlencoded",
|
|
||||||
)
|
|
||||||
req.RemoteAddr = sharedProxyPeer
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
h.HandleLoginSubmit().ServeHTTP(w, req)
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusSeeOther, w.Code, "next %q", c.next)
|
|
||||||
assert.Equal(
|
|
||||||
t, c.want, w.Header().Get("Location"), "next %q", c.next,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLogin_WrongPasswordKeepsTheRequestedPage: after a wrong
|
|
||||||
// password the login page is shown again with the same next, so the
|
|
||||||
// next attempt still returns to the page that was asked for.
|
|
||||||
func TestLogin_WrongPasswordKeepsTheRequestedPage(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
db *database.Database
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &db)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
seedOperator(t, db)
|
|
||||||
|
|
||||||
form := url.Values{}
|
|
||||||
form.Set("username", operatorUser)
|
|
||||||
form.Set("password", "wrong")
|
|
||||||
form.Set("next", "/hook/abc")
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(
|
|
||||||
context.Background(),
|
|
||||||
http.MethodPost,
|
|
||||||
"/pages/login",
|
|
||||||
strings.NewReader(form.Encode()),
|
|
||||||
)
|
|
||||||
req.Header.Set(
|
|
||||||
"Content-Type", "application/x-www-form-urlencoded",
|
|
||||||
)
|
|
||||||
req.RemoteAddr = sharedProxyPeer
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
h.HandleLoginSubmit().ServeHTTP(w, req)
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusUnauthorized, w.Code)
|
|
||||||
assert.Contains(
|
|
||||||
t, w.Body.String(), `name="next" value="/hook/abc"`,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// loginPageGet renders the login page as a GET with the given next
|
|
||||||
// value and cookies.
|
|
||||||
func loginPageGet(
|
|
||||||
h *handlers.Handlers, next string, cookies []*http.Cookie,
|
|
||||||
) *httptest.ResponseRecorder {
|
|
||||||
req := httptest.NewRequestWithContext(
|
|
||||||
context.Background(), http.MethodGet,
|
|
||||||
"/pages/login?"+url.Values{"next": {next}}.Encode(), nil,
|
|
||||||
)
|
|
||||||
|
|
||||||
for _, c := range cookies {
|
|
||||||
req.AddCookie(c)
|
|
||||||
}
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
h.HandleLoginPage().ServeHTTP(w, req)
|
|
||||||
|
|
||||||
return w
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLoginPage_CarriesOnlyAPathOnThisSite covers the login page
|
|
||||||
// itself: its form carries the requested page only when it is a path
|
|
||||||
// on this site, and a browser already logged in goes straight there,
|
|
||||||
// or to "/" when it is not.
|
|
||||||
func TestLoginPage_CarriesOnlyAPathOnThisSite(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
assert.Contains(
|
|
||||||
t, loginPageGet(h, "/hook/abc", nil).Body.String(),
|
|
||||||
`name="next" value="/hook/abc"`,
|
|
||||||
)
|
|
||||||
assert.Contains(
|
|
||||||
t, loginPageGet(h, "//evil.example/", nil).Body.String(),
|
|
||||||
`name="next" value="/"`,
|
|
||||||
)
|
|
||||||
|
|
||||||
cookies := authenticatedCookies(t, sess, "test-user-id", "testuser")
|
|
||||||
|
|
||||||
cases := []struct{ next, want string }{
|
|
||||||
{"/hook/abc", "/hook/abc"},
|
|
||||||
{"//evil.example/", "/"},
|
|
||||||
{`/\evil.example/`, "/"},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, c := range cases {
|
|
||||||
w := loginPageGet(h, c.next, cookies)
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusSeeOther, w.Code, "next %q", c.next)
|
|
||||||
assert.Equal(
|
|
||||||
t, c.want, w.Header().Get("Location"), "next %q", c.next,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLoginPage_HasNoLinkToItself: the navigation bar on the login
|
|
||||||
// page offers no link to the login page.
|
|
||||||
func TestLoginPage_HasNoLinkToItself(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var h *handlers.Handlers
|
|
||||||
|
|
||||||
app := newTestApp(t, &h)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
w := loginPageGet(h, "", nil)
|
|
||||||
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
assert.NotContains(t, w.Body.String(), `href="/pages/login"`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestLogin_UsernameAtLimitCanLogIn shows that a username of exactly
|
// TestLogin_UsernameAtLimitCanLogIn shows that a username of exactly
|
||||||
// database.MaxUsernameBytes still fits in the session cookie. Past
|
// database.MaxUsernameBytes still fits in the session cookie. Past
|
||||||
// what the cookie can carry, a correct login answers 500.
|
// what the cookie can carry, a correct login answers 500.
|
||||||
|
|||||||
@@ -11,37 +11,72 @@ import (
|
|||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The outcomes of a replay POST, as the notice codes its redirect
|
// replayOutcomeParam is the query parameter the replay POST redirects
|
||||||
// carries. noticeFor holds the line each one shows.
|
// with and the event log page reads its banner from.
|
||||||
|
const replayOutcomeParam = "replay"
|
||||||
|
|
||||||
|
// replayOutcomeCode is the outcome of a replay POST. The redirect
|
||||||
|
// carries one of these fixed codes rather than a message, so nothing a
|
||||||
|
// client submits can reach the rendered page through it.
|
||||||
|
type replayOutcomeCode string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// replayQueued reports that a new delivery was created and handed
|
// replayQueued reports that a new delivery was created and handed
|
||||||
// to the delivery engine.
|
// to the delivery engine.
|
||||||
replayQueued noticeCode = "replay-queued"
|
replayQueued replayOutcomeCode = "queued"
|
||||||
|
|
||||||
// replayTargetDeleted reports a target that once existed and has
|
// replayTargetDeleted reports a target that once existed and has
|
||||||
// since been deleted. Deletes are soft and deliveries carry no
|
// since been deleted. Deletes are soft and deliveries carry no
|
||||||
// foreign key to the target row, so the history survives its
|
// foreign key to the target row, so the history survives its
|
||||||
// target and this is the ordinary case for an old event.
|
// target and this is the ordinary case for an old event.
|
||||||
replayTargetDeleted noticeCode = "replay-target-deleted"
|
replayTargetDeleted replayOutcomeCode = "target-deleted"
|
||||||
|
|
||||||
// replayTargetMissing reports a target id that names no row at
|
// replayTargetMissing reports a target id that names no row at
|
||||||
// all, deleted or otherwise.
|
// all, deleted or otherwise.
|
||||||
replayTargetMissing noticeCode = "replay-target-missing"
|
replayTargetMissing replayOutcomeCode = "target-missing"
|
||||||
|
|
||||||
// replayTargetInactive reports a target the operator has
|
// replayTargetInactive reports a target the operator has
|
||||||
// deactivated. A deactivated target receives no new deliveries, so
|
// deactivated. A deactivated target receives no new deliveries, so
|
||||||
// a replay to it would be a delivery they switched off.
|
// a replay to it would be a delivery they switched off.
|
||||||
replayTargetInactive noticeCode = "replay-target-inactive"
|
replayTargetInactive replayOutcomeCode = "target-inactive"
|
||||||
|
|
||||||
// replayNotTerminal reports a delivery the engine has not finished
|
// replayNotTerminal reports a delivery the engine has not finished
|
||||||
// with.
|
// with.
|
||||||
replayNotTerminal noticeCode = "replay-not-terminal"
|
replayNotTerminal replayOutcomeCode = "not-terminal"
|
||||||
|
|
||||||
// replayInFlight reports that an earlier replay of this event to
|
// replayInFlight reports that an earlier replay of this event to
|
||||||
// this target is still running.
|
// this target is still running.
|
||||||
replayInFlight noticeCode = "replay-in-flight"
|
replayInFlight replayOutcomeCode = "in-flight"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// replayOutcome returns the banner the event log page shows for an
|
||||||
|
// outcome code, and whether the replay was queued. An unrecognised
|
||||||
|
// code yields no banner.
|
||||||
|
func replayOutcome(code string) (string, bool) {
|
||||||
|
switch replayOutcomeCode(code) {
|
||||||
|
case replayQueued:
|
||||||
|
return "Replay queued: a new delivery was created against " +
|
||||||
|
"the target's current configuration.", true
|
||||||
|
case replayTargetDeleted:
|
||||||
|
return "Not replayed: the target this delivery was for has " +
|
||||||
|
"been deleted. Recreate the target, then replay.", false
|
||||||
|
case replayTargetMissing:
|
||||||
|
return "Not replayed: the target this delivery was for no " +
|
||||||
|
"longer exists.", false
|
||||||
|
case replayTargetInactive:
|
||||||
|
return "Not replayed: the target this delivery was for is " +
|
||||||
|
"deactivated. Activate it, then replay.", false
|
||||||
|
case replayNotTerminal:
|
||||||
|
return "Not replayed: this delivery has not finished yet.",
|
||||||
|
false
|
||||||
|
case replayInFlight:
|
||||||
|
return "Not replayed: a delivery of this event to this " +
|
||||||
|
"target is already in flight.", false
|
||||||
|
default:
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// HandleDeliveryReplay re-sends a finished delivery's event to its
|
// HandleDeliveryReplay re-sends a finished delivery's event to its
|
||||||
// target.
|
// target.
|
||||||
//
|
//
|
||||||
@@ -70,7 +105,9 @@ func (h *Handlers) HandleDeliveryReplay() http.HandlerFunc {
|
|||||||
// middleware, which runs before CSRF parses the form.
|
// middleware, which runs before CSRF parses the form.
|
||||||
err := r.ParseForm()
|
err := r.ParseForm()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.renderError(w, r, http.StatusBadRequest)
|
http.Error(
|
||||||
|
w, "Bad request", http.StatusBadRequest,
|
||||||
|
)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -87,14 +124,14 @@ func (h *Handlers) replayDelivery(
|
|||||||
webhook database.Webhook,
|
webhook database.Webhook,
|
||||||
) {
|
) {
|
||||||
if !h.dbMgr.DBExists(webhook.ID) {
|
if !h.dbMgr.DBExists(webhook.ID) {
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
http.NotFound(w, r)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
|
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.serverError(w, r, "failed to get webhook database", err)
|
h.serverError(w, "failed to get webhook database", err)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -105,14 +142,14 @@ func (h *Handlers) replayDelivery(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if !original.Status.Terminal() {
|
if !original.Status.Terminal() {
|
||||||
redirectToEventLog(w, r, webhook, replayNotTerminal)
|
h.finishReplay(w, r, webhook, replayNotTerminal)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
target, code := h.replayTarget(webhook.ID, original.TargetID)
|
target, code := h.replayTarget(webhook.ID, original.TargetID)
|
||||||
if target == nil {
|
if target == nil {
|
||||||
redirectToEventLog(w, r, webhook, code)
|
h.finishReplay(w, r, webhook, code)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -136,7 +173,7 @@ func (h *Handlers) loadReplaySource(
|
|||||||
&original, "id = ?", chi.URLParam(r, "deliveryID"),
|
&original, "id = ?", chi.URLParam(r, "deliveryID"),
|
||||||
).Error
|
).Error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
http.NotFound(w, r)
|
||||||
|
|
||||||
return nil, false
|
return nil, false
|
||||||
}
|
}
|
||||||
@@ -158,14 +195,14 @@ func (h *Handlers) queueReplay(
|
|||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.serverError(
|
h.serverError(
|
||||||
w, r, "failed to count in-flight deliveries", err,
|
w, "failed to count in-flight deliveries", err,
|
||||||
)
|
)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if inFlight > 0 {
|
if inFlight > 0 {
|
||||||
redirectToEventLog(w, r, webhook, replayInFlight)
|
h.finishReplay(w, r, webhook, replayInFlight)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -175,7 +212,7 @@ func (h *Handlers) queueReplay(
|
|||||||
err = webhookDB.
|
err = webhookDB.
|
||||||
First(&event, "id = ?", original.EventID).Error
|
First(&event, "id = ?", original.EventID).Error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.serverError(w, r, "failed to load event for replay", err)
|
h.serverError(w, "failed to load event for replay", err)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -185,7 +222,7 @@ func (h *Handlers) queueReplay(
|
|||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.serverError(
|
h.serverError(
|
||||||
w, r, "failed to create replay delivery", err,
|
w, "failed to create replay delivery", err,
|
||||||
)
|
)
|
||||||
|
|
||||||
return
|
return
|
||||||
@@ -203,7 +240,7 @@ func (h *Handlers) queueReplay(
|
|||||||
"delivery_id", task.DeliveryID,
|
"delivery_id", task.DeliveryID,
|
||||||
)
|
)
|
||||||
|
|
||||||
redirectToEventLog(w, r, webhook, replayQueued)
|
h.finishReplay(w, r, webhook, replayQueued)
|
||||||
}
|
}
|
||||||
|
|
||||||
// replayTarget loads the delivery's target as it stands now.
|
// replayTarget loads the delivery's target as it stands now.
|
||||||
@@ -216,7 +253,7 @@ func (h *Handlers) queueReplay(
|
|||||||
// with the returned code saying why.
|
// with the returned code saying why.
|
||||||
func (h *Handlers) replayTarget(
|
func (h *Handlers) replayTarget(
|
||||||
webhookID, targetID string,
|
webhookID, targetID string,
|
||||||
) (*database.Target, noticeCode) {
|
) (*database.Target, replayOutcomeCode) {
|
||||||
var target database.Target
|
var target database.Target
|
||||||
|
|
||||||
err := h.db.DB().Unscoped().Where(
|
err := h.db.DB().Unscoped().Where(
|
||||||
@@ -326,16 +363,17 @@ func replayBody(body string) *string {
|
|||||||
return &body
|
return &body
|
||||||
}
|
}
|
||||||
|
|
||||||
// redirectToEventLog redirects a replay or resubmit back to the event
|
// finishReplay redirects back to the event log the replay was
|
||||||
// log it was triggered from, carrying the outcome as its notice and
|
// triggered from, carrying the outcome code the page turns into a
|
||||||
// the page number the form submitted.
|
// banner and the page number the form submitted.
|
||||||
func redirectToEventLog(
|
func (h *Handlers) finishReplay(
|
||||||
w http.ResponseWriter,
|
w http.ResponseWriter,
|
||||||
r *http.Request,
|
r *http.Request,
|
||||||
webhook database.Webhook,
|
webhook database.Webhook,
|
||||||
code noticeCode,
|
code replayOutcomeCode,
|
||||||
) {
|
) {
|
||||||
dest := withNotice("/hook/"+webhook.ID+"/events", code)
|
dest := "/source/" + webhook.ID + "/logs?" +
|
||||||
|
replayOutcomeParam + "=" + string(code)
|
||||||
|
|
||||||
// The page is read from the form rather than the query string:
|
// The page is read from the form rather than the query string:
|
||||||
// this is a POST, and its query string is what logs and Referer
|
// this is a POST, and its query string is what logs and Referer
|
||||||
|
|||||||
@@ -138,7 +138,7 @@ func postReplay(
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
req := postRequest(
|
req := postRequest(
|
||||||
"/hook/"+webhookID+"/deliveries/"+
|
"/source/"+webhookID+"/deliveries/"+
|
||||||
deliveryID+"/replay",
|
deliveryID+"/replay",
|
||||||
authenticatedCookies(
|
authenticatedCookies(
|
||||||
t, sess, deleteTestUserID, deleteTestUsername,
|
t, sess, deleteTestUserID, deleteTestUsername,
|
||||||
@@ -212,7 +212,7 @@ func TestHandleDeliveryReplay_AppendsDeliveryAndLeavesOriginal(
|
|||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"/hook/"+wh.ID+"/events?notice=replay-queued",
|
"/source/"+wh.ID+"/logs?replay=queued",
|
||||||
w.Header().Get("Location"),
|
w.Header().Get("Location"),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -362,7 +362,7 @@ func TestHandleDeliveryReplay_RefusesDeletedTarget(t *testing.T) {
|
|||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"/hook/"+wh.ID+"/events?notice=replay-target-deleted",
|
"/source/"+wh.ID+"/logs?replay=target-deleted",
|
||||||
w.Header().Get("Location"),
|
w.Header().Get("Location"),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -390,7 +390,7 @@ func TestHandleDeliveryReplay_RefusesDeletedTarget(t *testing.T) {
|
|||||||
require.Equal(t, http.StatusSeeOther, missing.Code)
|
require.Equal(t, http.StatusSeeOther, missing.Code)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"/hook/"+wh.ID+"/events?notice=replay-target-missing",
|
"/source/"+wh.ID+"/logs?replay=target-missing",
|
||||||
missing.Header().Get("Location"),
|
missing.Header().Get("Location"),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -431,7 +431,7 @@ func TestHandleDeliveryReplay_RefusesWhileEarlierReplayInFlight(
|
|||||||
require.Equal(t, http.StatusSeeOther, first.Code)
|
require.Equal(t, http.StatusSeeOther, first.Code)
|
||||||
require.Equal(
|
require.Equal(
|
||||||
t,
|
t,
|
||||||
"/hook/"+wh.ID+"/events?notice=replay-queued",
|
"/source/"+wh.ID+"/logs?replay=queued",
|
||||||
first.Header().Get("Location"),
|
first.Header().Get("Location"),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -439,7 +439,7 @@ func TestHandleDeliveryReplay_RefusesWhileEarlierReplayInFlight(
|
|||||||
require.Equal(t, http.StatusSeeOther, second.Code)
|
require.Equal(t, http.StatusSeeOther, second.Code)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"/hook/"+wh.ID+"/events?notice=replay-in-flight",
|
"/source/"+wh.ID+"/logs?replay=in-flight",
|
||||||
second.Header().Get("Location"),
|
second.Header().Get("Location"),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -465,7 +465,7 @@ func TestHandleDeliveryReplay_RefusesWhileEarlierReplayInFlight(
|
|||||||
require.Equal(t, http.StatusSeeOther, pending.Code)
|
require.Equal(t, http.StatusSeeOther, pending.Code)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"/hook/"+wh.ID+"/events?notice=replay-not-terminal",
|
"/source/"+wh.ID+"/logs?replay=not-terminal",
|
||||||
pending.Header().Get("Location"),
|
pending.Header().Get("Location"),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -501,7 +501,7 @@ func TestHandleSourceLogs_RendersReplayControlAndBanner(t *testing.T) {
|
|||||||
|
|
||||||
assert.Contains(
|
assert.Contains(
|
||||||
t, body,
|
t, body,
|
||||||
`action="/hook/`+wh.ID+`/deliveries/`+
|
`action="/source/`+wh.ID+`/deliveries/`+
|
||||||
original.ID+`/replay"`,
|
original.ID+`/replay"`,
|
||||||
)
|
)
|
||||||
assert.Contains(t, body, `method="POST"`)
|
assert.Contains(t, body, `method="POST"`)
|
||||||
@@ -509,7 +509,7 @@ func TestHandleSourceLogs_RendersReplayControlAndBanner(t *testing.T) {
|
|||||||
assert.Contains(t, body, ">Replay<")
|
assert.Contains(t, body, ">Replay<")
|
||||||
|
|
||||||
refused := renderSourceLogsPageWithQuery(
|
refused := renderSourceLogsPageWithQuery(
|
||||||
t, h, sess, wh.ID, "?notice=replay-target-deleted",
|
t, h, sess, wh.ID, "?replay=target-deleted",
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Contains(t, refused, "alert-error")
|
assert.Contains(t, refused, "alert-error")
|
||||||
@@ -517,7 +517,7 @@ func TestHandleSourceLogs_RendersReplayControlAndBanner(t *testing.T) {
|
|||||||
|
|
||||||
// An outcome code nobody issued renders no banner at all.
|
// An outcome code nobody issued renders no banner at all.
|
||||||
unknown := renderSourceLogsPageWithQuery(
|
unknown := renderSourceLogsPageWithQuery(
|
||||||
t, h, sess, wh.ID, "?notice=made-up",
|
t, h, sess, wh.ID, "?replay=made-up",
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.NotContains(t, unknown, "alert-error")
|
assert.NotContains(t, unknown, "alert-error")
|
||||||
|
|||||||
@@ -1,53 +0,0 @@
|
|||||||
package handlers_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"html/template"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
|
||||||
)
|
|
||||||
|
|
||||||
// TestErrorPage_RenderFailureKeepsStatus proves that an error page
|
|
||||||
// which cannot render answers with the status it was reporting, as
|
|
||||||
// plain text, and is not attempted again: a page whose own render
|
|
||||||
// fails reaches the error page, and the error page failing as well
|
|
||||||
// ends there with the 500.
|
|
||||||
func TestErrorPage_RenderFailureKeepsStatus(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var h *handlers.Handlers
|
|
||||||
|
|
||||||
app := newTestApp(t, &h)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
// .Status is an int, so asking it for a field fails the render.
|
|
||||||
failing := `{{.Status.Missing}}`
|
|
||||||
h.AddTemplateForTest("error.html", template.Must(
|
|
||||||
template.New("error").Parse(failing),
|
|
||||||
))
|
|
||||||
h.AddTemplateForTest("failing.html", template.Must(
|
|
||||||
template.New("failing").Parse(`{{.Data.Missing}}`),
|
|
||||||
))
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(
|
|
||||||
context.Background(), http.MethodGet, "/", nil,
|
|
||||||
)
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
h.HandleErrorPage(http.StatusNotFound).ServeHTTP(w, req)
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusNotFound, w.Code)
|
|
||||||
assert.Equal(t, "Not Found\n", w.Body.String())
|
|
||||||
|
|
||||||
w = httptest.NewRecorder()
|
|
||||||
h.RenderTemplateForTest(w, req, "failing.html", 0)
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusInternalServerError, w.Code)
|
|
||||||
assert.Equal(t, "Internal Server Error\n", w.Body.String())
|
|
||||||
}
|
|
||||||
@@ -15,12 +15,10 @@ import (
|
|||||||
// eventBodyQuery reads one event's stored body as bytes. The cast
|
// eventBodyQuery reads one event's stored body as bytes. The cast
|
||||||
// to blob is what makes the driver hand back the stored bytes
|
// to blob is what makes the driver hand back the stored bytes
|
||||||
// rather than a string conversion, so Content-Length taken from
|
// rather than a string conversion, so Content-Length taken from
|
||||||
// the result matches what goes on the wire. The retention reaper
|
// the result matches what goes on the wire. The soft-delete
|
||||||
// deletes event rows outright, so a reaped event is simply gone
|
// predicate is spelled out because Raw bypasses GORM's default
|
||||||
// and the query finds no row. The deleted_at predicate repeats
|
// scope, and it is what stops a reaped event still being
|
||||||
// the soft-delete scope GORM adds to its own queries, which Raw
|
// downloadable.
|
||||||
// bypasses; nothing soft-deletes an event, so today it excludes
|
|
||||||
// nothing.
|
|
||||||
const eventBodyQuery = "SELECT cast(body as blob) " +
|
const eventBodyQuery = "SELECT cast(body as blob) " +
|
||||||
"FROM events WHERE id = ? AND webhook_id = ? AND deleted_at IS NULL"
|
"FROM events WHERE id = ? AND webhook_id = ? AND deleted_at IS NULL"
|
||||||
|
|
||||||
@@ -54,7 +52,7 @@ func (h *Handlers) HandleEventBodyDownload() http.HandlerFunc {
|
|||||||
// steered by a client.
|
// steered by a client.
|
||||||
eventID, err := uuid.Parse(chi.URLParam(r, "eventID"))
|
eventID, err := uuid.Parse(chi.URLParam(r, "eventID"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
http.NotFound(w, r)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -105,21 +103,21 @@ func (h *Handlers) serveEventBody(
|
|||||||
eventID string,
|
eventID string,
|
||||||
) {
|
) {
|
||||||
if !h.dbMgr.DBExists(webhook.ID) {
|
if !h.dbMgr.DBExists(webhook.ID) {
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
http.NotFound(w, r)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
|
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.serverError(w, r, "failed to get webhook database", err)
|
h.serverError(w, "failed to get webhook database", err)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
body, found, err := eventBody(webhookDB, webhook.ID, eventID)
|
body, found, err := eventBody(webhookDB, webhook.ID, eventID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.serverError(w, r, "failed to read event body", err)
|
h.serverError(w, "failed to read event body", err)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -132,7 +130,7 @@ func (h *Handlers) serveEventBody(
|
|||||||
// row and the whole body is served, or it does not and the
|
// row and the whole body is served, or it does not and the
|
||||||
// response is a clean 404.
|
// response is a clean 404.
|
||||||
if !found {
|
if !found {
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
http.NotFound(w, r)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -64,8 +64,8 @@ func fetchEventBody(
|
|||||||
req := httptest.NewRequestWithContext(
|
req := httptest.NewRequestWithContext(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
http.MethodGet,
|
http.MethodGet,
|
||||||
"/hook/"+url.PathEscape(sourceID)+
|
"/source/"+url.PathEscape(sourceID)+
|
||||||
"/events/"+url.PathEscape(eventID)+"/body",
|
"/logs/"+url.PathEscape(eventID)+"/body",
|
||||||
nil,
|
nil,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -405,11 +405,10 @@ func TestHandleEventBodyDownload_UnknownEvent404s(t *testing.T) {
|
|||||||
// route. The body is read in one query before any header is
|
// route. The body is read in one query before any header is
|
||||||
// written, so a reaped event cannot produce a partial download:
|
// written, so a reaped event cannot produce a partial download:
|
||||||
// it is a clean 404 with no Content-Length and no
|
// it is a clean 404 with no Content-Length and no
|
||||||
// Content-Disposition. The reaper deletes event rows outright,
|
// Content-Disposition. Both removals the codebase performs are
|
||||||
// which is the "hard deleted" case. The "soft deleted" case
|
// covered — the reaper hard-deletes, and a soft-deleted row is
|
||||||
// covers a row no code produces today: it only pins the query's
|
// excluded by the query's own deleted_at predicate rather than
|
||||||
// own deleted_at predicate, the soft-delete condition Raw would
|
// by GORM's default scope, which Raw bypasses.
|
||||||
// otherwise skip.
|
|
||||||
func TestHandleEventBodyDownload_ReapedEvent404s(t *testing.T) {
|
func TestHandleEventBodyDownload_ReapedEvent404s(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -491,7 +490,7 @@ func TestHandleSourceLogs_TruncationMarkerLinksToDownload(
|
|||||||
page := renderSourceLogsPage(t, h, sess, big.ID)
|
page := renderSourceLogsPage(t, h, sess, big.ID)
|
||||||
assert.Contains(
|
assert.Contains(
|
||||||
t, page,
|
t, page,
|
||||||
"/hook/"+big.ID+"/events/"+bigEvt.ID+"/body",
|
"/source/"+big.ID+"/logs/"+bigEvt.ID+"/body",
|
||||||
)
|
)
|
||||||
|
|
||||||
small := seedWebhook(t, db)
|
small := seedWebhook(t, db)
|
||||||
@@ -502,6 +501,6 @@ func TestHandleSourceLogs_TruncationMarkerLinksToDownload(
|
|||||||
page = renderSourceLogsPage(t, h, sess, small.ID)
|
page = renderSourceLogsPage(t, h, sess, small.ID)
|
||||||
assert.NotContains(
|
assert.NotContains(
|
||||||
t, page,
|
t, page,
|
||||||
"/hook/"+small.ID+"/events/"+smallEvt.ID+"/body",
|
"/source/"+small.ID+"/logs/"+smallEvt.ID+"/body",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package handlers
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
"github.com/go-chi/chi"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
@@ -10,19 +11,43 @@ import (
|
|||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The outcomes of a resubmit POST, as the notice codes its redirect
|
// resubmitOutcomeParam is the query parameter the resubmit POST
|
||||||
// carries. noticeFor holds the line each one shows.
|
// redirects with and the event log page reads its banner from.
|
||||||
|
const resubmitOutcomeParam = "resubmit"
|
||||||
|
|
||||||
|
// resubmitOutcomeCode is the outcome of a resubmit POST. The redirect
|
||||||
|
// carries one of these fixed codes rather than a message, so nothing a
|
||||||
|
// client submits can reach the rendered page through it.
|
||||||
|
type resubmitOutcomeCode string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// resubmitQueued reports that a new event was stored and its
|
// resubmitQueued reports that a new event was stored and its
|
||||||
// deliveries handed to the delivery engine.
|
// deliveries handed to the delivery engine.
|
||||||
resubmitQueued noticeCode = "resubmit-queued"
|
resubmitQueued resubmitOutcomeCode = "queued"
|
||||||
|
|
||||||
// resubmitNoTargets reports a source with no active targets. The
|
// resubmitNoTargets reports a source with no active targets. The
|
||||||
// new event is stored either way, exactly as a received event
|
// new event is stored either way, exactly as a received event
|
||||||
// with no targets is.
|
// with no targets is.
|
||||||
resubmitNoTargets noticeCode = "resubmit-no-targets"
|
resubmitNoTargets resubmitOutcomeCode = "no-targets"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// resubmitOutcome returns the banner the event log page shows for an
|
||||||
|
// outcome code, and whether the resubmit was queued. An unrecognised
|
||||||
|
// code yields no banner.
|
||||||
|
func resubmitOutcome(code string) (string, bool) {
|
||||||
|
switch resubmitOutcomeCode(code) {
|
||||||
|
case resubmitQueued:
|
||||||
|
return "Resubmitted: a new event was created from the stored " +
|
||||||
|
"one and queued to every active target.", true
|
||||||
|
case resubmitNoTargets:
|
||||||
|
return "Resubmitted: a new event was created, but this " +
|
||||||
|
"source has no active targets, so nothing was queued.",
|
||||||
|
true
|
||||||
|
default:
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// resubmitSource is the stored event a resubmit copies. Its body is
|
// resubmitSource is the stored event a resubmit copies. Its body is
|
||||||
// read as bytes rather than as a string so the copy is byte-identical
|
// read as bytes rather than as a string so the copy is byte-identical
|
||||||
// to what was received, whatever the payload's encoding.
|
// to what was received, whatever the payload's encoding.
|
||||||
@@ -74,7 +99,7 @@ func (h *Handlers) HandleEventResubmit() http.HandlerFunc {
|
|||||||
// middleware, which runs before CSRF parses the form.
|
// middleware, which runs before CSRF parses the form.
|
||||||
err := r.ParseForm()
|
err := r.ParseForm()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.renderError(w, r, http.StatusBadRequest)
|
http.Error(w, "Bad request", http.StatusBadRequest)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -95,20 +120,20 @@ func (h *Handlers) resubmitEvent(
|
|||||||
// alphabet rather than from the request.
|
// alphabet rather than from the request.
|
||||||
eventID, err := uuid.Parse(chi.URLParam(r, "eventID"))
|
eventID, err := uuid.Parse(chi.URLParam(r, "eventID"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
http.NotFound(w, r)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if !h.dbMgr.DBExists(webhook.ID) {
|
if !h.dbMgr.DBExists(webhook.ID) {
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
http.NotFound(w, r)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
|
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.serverError(w, r, "failed to get webhook database", err)
|
h.serverError(w, "failed to get webhook database", err)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -122,7 +147,7 @@ func (h *Handlers) resubmitEvent(
|
|||||||
webhookDB, webhook.ID, eventID.String(),
|
webhookDB, webhook.ID, eventID.String(),
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.serverError(w, r, "failed to load event to resubmit", err)
|
h.serverError(w, "failed to load event to resubmit", err)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -130,7 +155,7 @@ func (h *Handlers) resubmitEvent(
|
|||||||
// A miss is a 404 whether the event was reaped, belongs to
|
// A miss is a 404 whether the event was reaped, belongs to
|
||||||
// another webhook, or never existed.
|
// another webhook, or never existed.
|
||||||
if !found {
|
if !found {
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
http.NotFound(w, r)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -145,9 +170,8 @@ func (h *Handlers) resubmitEvent(
|
|||||||
// per-webhook database files — a sibling webhook's event is not in the
|
// per-webhook database files — a sibling webhook's event is not in the
|
||||||
// database being queried at all — and is there so the scoping survives
|
// database being queried at all — and is there so the scoping survives
|
||||||
// any future change that puts more than one webhook's events in one
|
// any future change that puts more than one webhook's events in one
|
||||||
// file. A reaped event is not found because the retention reaper
|
// file. Going through Model applies GORM's soft-delete scope, which is
|
||||||
// deletes its row outright rather than marking it deleted; see
|
// what stops a reaped event being resubmitted.
|
||||||
// deleteEvents in internal/database/retention.go.
|
|
||||||
func loadResubmitSource(
|
func loadResubmitSource(
|
||||||
webhookDB *gorm.DB,
|
webhookDB *gorm.DB,
|
||||||
webhookID, eventID string,
|
webhookID, eventID string,
|
||||||
@@ -183,7 +207,7 @@ func (h *Handlers) queueResubmit(
|
|||||||
// inactive one is skipped rather than refused.
|
// inactive one is skipped rather than refused.
|
||||||
targets, err := h.loadActiveTargets(webhook.ID)
|
targets, err := h.loadActiveTargets(webhook.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.serverError(w, r, "failed to query targets", err)
|
h.serverError(w, "failed to query targets", err)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -201,7 +225,7 @@ func (h *Handlers) queueResubmit(
|
|||||||
targets,
|
targets,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.serverError(w, r, "failed to store resubmitted event", err)
|
h.serverError(w, "failed to store resubmitted event", err)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -221,5 +245,29 @@ func (h *Handlers) queueResubmit(
|
|||||||
code = resubmitNoTargets
|
code = resubmitNoTargets
|
||||||
}
|
}
|
||||||
|
|
||||||
redirectToEventLog(w, r, webhook, code)
|
h.finishResubmit(w, r, webhook, code)
|
||||||
|
}
|
||||||
|
|
||||||
|
// finishResubmit redirects back to the event log the resubmit was
|
||||||
|
// triggered from, carrying the outcome code the page turns into a
|
||||||
|
// banner and the page number the form submitted.
|
||||||
|
func (h *Handlers) finishResubmit(
|
||||||
|
w http.ResponseWriter,
|
||||||
|
r *http.Request,
|
||||||
|
webhook database.Webhook,
|
||||||
|
code resubmitOutcomeCode,
|
||||||
|
) {
|
||||||
|
dest := "/source/" + webhook.ID + "/logs?" +
|
||||||
|
resubmitOutcomeParam + "=" + string(code)
|
||||||
|
|
||||||
|
// The page is read from the form rather than the query string:
|
||||||
|
// this is a POST, and its query string is what logs and Referer
|
||||||
|
// headers record.
|
||||||
|
if page := pageOrFirst(
|
||||||
|
r.PostFormValue("page"),
|
||||||
|
); page > 1 {
|
||||||
|
dest += "&page=" + strconv.Itoa(page)
|
||||||
|
}
|
||||||
|
|
||||||
|
http.Redirect(w, r, dest, http.StatusSeeOther)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ func postResubmit(
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
req := postRequest(
|
req := postRequest(
|
||||||
"/hook/"+webhookID+"/events/"+eventID+"/resubmit",
|
"/source/"+webhookID+"/events/"+eventID+"/resubmit",
|
||||||
authenticatedCookies(
|
authenticatedCookies(
|
||||||
t, sess, deleteTestUserID, deleteTestUsername,
|
t, sess, deleteTestUserID, deleteTestUsername,
|
||||||
),
|
),
|
||||||
@@ -154,7 +154,7 @@ func TestHandleEventResubmit_DeliversToTargetCreatedAfterTheEvent(
|
|||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"/hook/"+wh.ID+"/events?notice=resubmit-queued",
|
"/source/"+wh.ID+"/logs?resubmit=queued",
|
||||||
w.Header().Get("Location"),
|
w.Header().Get("Location"),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -204,7 +204,6 @@ func assertEventCopy(
|
|||||||
assert.Equal(t, original.Method, fresh.Method)
|
assert.Equal(t, original.Method, fresh.Method)
|
||||||
assert.Equal(t, original.Headers, fresh.Headers)
|
assert.Equal(t, original.Headers, fresh.Headers)
|
||||||
assert.Equal(t, original.Body, fresh.Body)
|
assert.Equal(t, original.Body, fresh.Body)
|
||||||
assert.Equal(t, int64(len(original.Body)), fresh.BodyBytes)
|
|
||||||
assert.Equal(t, original.ContentType, fresh.ContentType)
|
assert.Equal(t, original.ContentType, fresh.ContentType)
|
||||||
assert.Equal(t, original.EntrypointID, fresh.EntrypointID)
|
assert.Equal(t, original.EntrypointID, fresh.EntrypointID)
|
||||||
assert.Equal(t, original.WebhookID, fresh.WebhookID)
|
assert.Equal(t, original.WebhookID, fresh.WebhookID)
|
||||||
@@ -282,7 +281,7 @@ func TestHandleEventResubmit_IsRepeatable(t *testing.T) {
|
|||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"/hook/"+wh.ID+"/events?notice=resubmit-queued",
|
"/source/"+wh.ID+"/logs?resubmit=queued",
|
||||||
w.Header().Get("Location"),
|
w.Header().Get("Location"),
|
||||||
"a resubmit must not be refused while an earlier "+
|
"a resubmit must not be refused while an earlier "+
|
||||||
"one is in flight",
|
"one is in flight",
|
||||||
@@ -436,7 +435,7 @@ func TestHandleEventResubmit_SkipsInactiveTarget(t *testing.T) {
|
|||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"/hook/"+wh.ID+"/events?notice=resubmit-queued",
|
"/source/"+wh.ID+"/logs?resubmit=queued",
|
||||||
w.Header().Get("Location"),
|
w.Header().Get("Location"),
|
||||||
"an inactive target is skipped, not an error",
|
"an inactive target is skipped, not an error",
|
||||||
)
|
)
|
||||||
@@ -482,7 +481,7 @@ func TestHandleEventResubmit_NoActiveTargetsStillStoresEvent(
|
|||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"/hook/"+wh.ID+"/events?notice=resubmit-no-targets",
|
"/source/"+wh.ID+"/logs?resubmit=no-targets",
|
||||||
w.Header().Get("Location"),
|
w.Header().Get("Location"),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -598,7 +597,7 @@ func TestHandleSourceLogs_ShowsResubmitProvenance(t *testing.T) {
|
|||||||
)
|
)
|
||||||
assert.Contains(
|
assert.Contains(
|
||||||
t, body,
|
t, body,
|
||||||
"/hook/"+wh.ID+"/events/"+original.ID+"/resubmit",
|
"/source/"+wh.ID+"/events/"+original.ID+"/resubmit",
|
||||||
"the log must offer the resubmit action per event",
|
"the log must offer the resubmit action per event",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,11 +1,9 @@
|
|||||||
package handlers
|
package handlers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"html/template"
|
"html/template"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
@@ -36,15 +34,6 @@ const MaxRenderedAttemptsForTest = maxRenderedAttempts
|
|||||||
// the handlers enforce rather than a number copied beside it.
|
// the handlers enforce rather than a number copied beside it.
|
||||||
const MaxTargetRetriesForTest = maxTargetRetries
|
const MaxTargetRetriesForTest = maxTargetRetries
|
||||||
|
|
||||||
// EventDBLeftMsgForTest and SidecarLeftMsgForTest expose the two
|
|
||||||
// messages the webhook delete handler logs when a file of the event
|
|
||||||
// database is left on disk, so a test checking that one is absent
|
|
||||||
// checks for the handler's own wording.
|
|
||||||
const (
|
|
||||||
EventDBLeftMsgForTest = eventDBLeftMsg
|
|
||||||
SidecarLeftMsgForTest = sidecarLeftMsg
|
|
||||||
)
|
|
||||||
|
|
||||||
// PageOrFirstForTest exposes pageOrFirst for use in the handlers_test
|
// PageOrFirstForTest exposes pageOrFirst for use in the handlers_test
|
||||||
// package.
|
// package.
|
||||||
func PageOrFirstForTest(s string) int {
|
func PageOrFirstForTest(s string) int {
|
||||||
@@ -76,7 +65,7 @@ func (s *Handlers) LoadEventLogViewsForTest(
|
|||||||
page int,
|
page int,
|
||||||
) []EventLogView {
|
) []EventLogView {
|
||||||
views, _, _ := s.loadEventsWithDeliveries(
|
views, _, _ := s.loadEventsWithDeliveries(
|
||||||
w, newRequestForTest(), webhook, nil, page,
|
w, webhook, nil, page,
|
||||||
)
|
)
|
||||||
|
|
||||||
return views
|
return views
|
||||||
@@ -105,14 +94,6 @@ func FinishedByTargetForTest(
|
|||||||
return finishedByTarget(webhookDB, since)
|
return finishedByTarget(webhookDB, since)
|
||||||
}
|
}
|
||||||
|
|
||||||
// newRequestForTest is the request the helpers here pass on for
|
|
||||||
// callers that have none: it is used only to render the error page.
|
|
||||||
func newRequestForTest() *http.Request {
|
|
||||||
return httptest.NewRequestWithContext(
|
|
||||||
context.Background(), http.MethodGet, "/", nil,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// AddTemplateForTest registers a template under a page name so that
|
// AddTemplateForTest registers a template under a page name so that
|
||||||
// the handlers_test package can drive the render path with a
|
// the handlers_test package can drive the render path with a
|
||||||
// template of its own.
|
// template of its own.
|
||||||
@@ -166,5 +147,5 @@ func (s *Handlers) BuildDatabaseTargetConfigForTest(
|
|||||||
w http.ResponseWriter,
|
w http.ResponseWriter,
|
||||||
expiry string,
|
expiry string,
|
||||||
) (string, error) {
|
) (string, error) {
|
||||||
return s.buildDatabaseTargetConfig(w, newRequestForTest(), expiry)
|
return s.buildDatabaseTargetConfig(w, expiry)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -306,7 +306,7 @@ func postWebhook(
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(
|
req := httptest.NewRequestWithContext(
|
||||||
context.Background(), http.MethodPost, "/h/x",
|
context.Background(), http.MethodPost, "/webhook/x",
|
||||||
strings.NewReader("{}"),
|
strings.NewReader("{}"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
+37
-153
@@ -10,12 +10,9 @@ import (
|
|||||||
"html/template"
|
"html/template"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/http"
|
"net/http"
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
|
||||||
"go.uber.org/fx"
|
"go.uber.org/fx"
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
"sneak.berlin/go/webhooker/internal/delivery"
|
||||||
"sneak.berlin/go/webhooker/internal/globals"
|
"sneak.berlin/go/webhooker/internal/globals"
|
||||||
@@ -31,7 +28,7 @@ const (
|
|||||||
// maxBodyShift is the bit shift for 1 MB body limit.
|
// maxBodyShift is the bit shift for 1 MB body limit.
|
||||||
maxBodyShift = 20
|
maxBodyShift = 20
|
||||||
// recentEventLimit is the number of recent events to show.
|
// recentEventLimit is the number of recent events to show.
|
||||||
recentEventLimit = 50
|
recentEventLimit = 20
|
||||||
// paginationPerPage is the number of items per page.
|
// paginationPerPage is the number of items per page.
|
||||||
paginationPerPage = 25
|
paginationPerPage = 25
|
||||||
|
|
||||||
@@ -39,9 +36,6 @@ const (
|
|||||||
tmplKeyError = "Error"
|
tmplKeyError = "Error"
|
||||||
// tmplKeyWebhook is the template data key for a webhook.
|
// tmplKeyWebhook is the template data key for a webhook.
|
||||||
tmplKeyWebhook = "Webhook"
|
tmplKeyWebhook = "Webhook"
|
||||||
// tmplKeyNext is the template data key for the page to return
|
|
||||||
// to after login.
|
|
||||||
tmplKeyNext = "Next"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// errInvalidPassword is returned when a password does not match.
|
// errInvalidPassword is returned when a password does not match.
|
||||||
@@ -59,17 +53,14 @@ type HandlersParams struct {
|
|||||||
|
|
||||||
Logger *logger.Logger
|
Logger *logger.Logger
|
||||||
Globals *globals.Globals
|
Globals *globals.Globals
|
||||||
Config *config.Config
|
|
||||||
Database *database.Database
|
Database *database.Database
|
||||||
WebhookDBMgr *database.WebhookDBManager
|
WebhookDBMgr *database.WebhookDBManager
|
||||||
Healthcheck *healthcheck.Healthcheck
|
Healthcheck *healthcheck.Healthcheck
|
||||||
Session *session.Session
|
Session *session.Session
|
||||||
Middleware *middleware.Middleware
|
Middleware *middleware.Middleware
|
||||||
Notifier delivery.Notifier
|
Notifier delivery.Notifier
|
||||||
Archives delivery.Archives
|
Evictor delivery.WebhookEvictor
|
||||||
SSRFGuard *delivery.Guard
|
SSRFGuard *delivery.Guard
|
||||||
Metrics *metrics.Set
|
|
||||||
Registry *prometheus.Registry
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Handlers provides HTTP handler methods for all application
|
// Handlers provides HTTP handler methods for all application
|
||||||
@@ -83,7 +74,7 @@ type Handlers struct {
|
|||||||
session *session.Session
|
session *session.Session
|
||||||
mw *middleware.Middleware
|
mw *middleware.Middleware
|
||||||
notifier delivery.Notifier
|
notifier delivery.Notifier
|
||||||
archives delivery.Archives
|
evictor delivery.WebhookEvictor
|
||||||
mtr *metrics.Set
|
mtr *metrics.Set
|
||||||
templates map[string]*template.Template
|
templates map[string]*template.Template
|
||||||
|
|
||||||
@@ -92,15 +83,6 @@ type Handlers struct {
|
|||||||
// is one delivery will actually attempt.
|
// is one delivery will actually attempt.
|
||||||
ssrf *delivery.Guard
|
ssrf *delivery.Guard
|
||||||
|
|
||||||
// renameMu makes the webhook edit, the target edit and target
|
|
||||||
// creation run one at a time, each held from loading the stored
|
|
||||||
// names through the archive rename, the save and any move back.
|
|
||||||
// Interleaved, one could rename an archive between another's
|
|
||||||
// rename and save, leaving the file named for one edit and the
|
|
||||||
// stored names from the other. An archive download holds it while
|
|
||||||
// it reads the stored names and opens the file they give.
|
|
||||||
renameMu sync.Mutex
|
|
||||||
|
|
||||||
// dummyVerifications counts the equivalent-cost verifications
|
// dummyVerifications counts the equivalent-cost verifications
|
||||||
// charged for usernames that do not exist. It exists so a test
|
// charged for usernames that do not exist. It exists so a test
|
||||||
// can prove that path runs without measuring wall-clock time.
|
// can prove that path runs without measuring wall-clock time.
|
||||||
@@ -109,10 +91,10 @@ type Handlers struct {
|
|||||||
|
|
||||||
// parsePageTemplate parses a page-specific template set from the
|
// parsePageTemplate parses a page-specific template set from the
|
||||||
// embedded FS. Each page template is combined with the shared
|
// embedded FS. Each page template is combined with the shared
|
||||||
// base, htmlheader, navbar and notice templates, and with any further
|
// base, htmlheader, and navbar templates, and with any further files
|
||||||
// files the page includes. The page file must be listed first so that
|
// the page includes. The page file must be listed first so that its
|
||||||
// its root action ({{template "base" .}}) becomes the template set's
|
// root action ({{template "base" .}}) becomes the template set's entry
|
||||||
// entry point.
|
// point.
|
||||||
func parsePageTemplate(
|
func parsePageTemplate(
|
||||||
pageFile string, included ...string,
|
pageFile string, included ...string,
|
||||||
) *template.Template {
|
) *template.Template {
|
||||||
@@ -121,7 +103,6 @@ func parsePageTemplate(
|
|||||||
"base.html",
|
"base.html",
|
||||||
"htmlheader.html",
|
"htmlheader.html",
|
||||||
"navbar.html",
|
"navbar.html",
|
||||||
"notice.html",
|
|
||||||
}, included...)
|
}, included...)
|
||||||
|
|
||||||
return template.Must(
|
return template.Must(
|
||||||
@@ -144,22 +125,20 @@ func New(
|
|||||||
s.session = params.Session
|
s.session = params.Session
|
||||||
s.mw = params.Middleware
|
s.mw = params.Middleware
|
||||||
s.notifier = params.Notifier
|
s.notifier = params.Notifier
|
||||||
s.archives = params.Archives
|
s.evictor = params.Evictor
|
||||||
s.mtr = params.Metrics
|
s.mtr = metrics.Default()
|
||||||
s.ssrf = params.SSRFGuard
|
s.ssrf = params.SSRFGuard
|
||||||
|
|
||||||
// Parse all page templates once at startup
|
// Parse all page templates once at startup
|
||||||
s.templates = map[string]*template.Template{
|
s.templates = map[string]*template.Template{
|
||||||
"login.html": parsePageTemplate("login.html"),
|
"login.html": parsePageTemplate("login.html"),
|
||||||
"profile.html": parsePageTemplate("profile.html"),
|
"profile.html": parsePageTemplate("profile.html"),
|
||||||
"settings.html": parsePageTemplate("settings.html"),
|
|
||||||
"sources_list.html": parsePageTemplate("sources_list.html"),
|
"sources_list.html": parsePageTemplate("sources_list.html"),
|
||||||
"sources_new.html": parsePageTemplate("sources_new.html"),
|
"sources_new.html": parsePageTemplate("sources_new.html"),
|
||||||
"source_detail.html": parsePageTemplate("source_detail.html", "webhook_stats.html"),
|
"source_detail.html": parsePageTemplate("source_detail.html", "webhook_stats.html"),
|
||||||
"source_edit.html": parsePageTemplate("source_edit.html"),
|
"source_edit.html": parsePageTemplate("source_edit.html"),
|
||||||
"source_logs.html": parsePageTemplate("source_logs.html"),
|
"source_logs.html": parsePageTemplate("source_logs.html"),
|
||||||
"target_edit.html": parsePageTemplate("target_edit.html"),
|
"target_edit.html": parsePageTemplate("target_edit.html"),
|
||||||
"error.html": parsePageTemplate("error.html"),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
lc.Append(fx.Hook{
|
lc.Append(fx.Hook{
|
||||||
@@ -171,16 +150,6 @@ func New(
|
|||||||
return s, nil
|
return s, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// HandleErrorPage returns a handler that answers every request with
|
|
||||||
// the error page for status. The router uses it for unknown paths, the
|
|
||||||
// CSRF middleware for a refused form, and each admin page route
|
|
||||||
// group's recoverer for a panic.
|
|
||||||
func (s *Handlers) HandleErrorPage(status int) http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
s.renderError(w, r, status)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Handlers) respondJSON(
|
func (s *Handlers) respondJSON(
|
||||||
w http.ResponseWriter,
|
w http.ResponseWriter,
|
||||||
_ *http.Request,
|
_ *http.Request,
|
||||||
@@ -198,78 +167,15 @@ func (s *Handlers) respondJSON(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// serverError logs an error and answers with the 500 error page.
|
// serverError logs an error and sends a 500 response.
|
||||||
func (s *Handlers) serverError(
|
func (s *Handlers) serverError(
|
||||||
w http.ResponseWriter, r *http.Request, msg string, err error,
|
w http.ResponseWriter, msg string, err error,
|
||||||
) {
|
) {
|
||||||
s.log.Error(msg, "error", err)
|
s.log.Error(msg, "error", err)
|
||||||
s.renderError(w, r, http.StatusInternalServerError)
|
http.Error(
|
||||||
}
|
w, "Internal server error",
|
||||||
|
http.StatusInternalServerError,
|
||||||
// renderError answers with status and the error page: the normal
|
)
|
||||||
// layout, one fixed line explaining the status, and a link back to the
|
|
||||||
// webhook list, or to sign-in when nobody is signed in.
|
|
||||||
//
|
|
||||||
// It renders the page itself rather than through renderTemplate,
|
|
||||||
// whose own failure comes here. If the error page cannot render
|
|
||||||
// either, the answer is the same status in plain text: never a second
|
|
||||||
// attempt, and never a different status.
|
|
||||||
func (s *Handlers) renderError(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
r *http.Request,
|
|
||||||
status int,
|
|
||||||
) {
|
|
||||||
// The page names the signed-in user, and some error pages are
|
|
||||||
// served outside the routes where NoCache runs.
|
|
||||||
w.Header().Set("Cache-Control", "no-store")
|
|
||||||
|
|
||||||
// No notice: one would say an action worked above a page saying
|
|
||||||
// the request failed.
|
|
||||||
data := s.pageData(r, map[string]any{
|
|
||||||
"Status": status,
|
|
||||||
"StatusText": http.StatusText(status),
|
|
||||||
"Message": errorPageText(status),
|
|
||||||
}, nil)
|
|
||||||
|
|
||||||
var buf bytes.Buffer
|
|
||||||
|
|
||||||
err := s.templates["error.html"].Execute(&buf, data)
|
|
||||||
if err != nil {
|
|
||||||
s.log.Error("failed to render error page", "error", err)
|
|
||||||
http.Error(w, http.StatusText(status), status)
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
||||||
w.WriteHeader(status)
|
|
||||||
|
|
||||||
_, err = buf.WriteTo(w)
|
|
||||||
if err != nil {
|
|
||||||
s.log.Error("failed to write error page", "error", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// errorPageText is the line the error page shows for status. It is
|
|
||||||
// fixed per status, so the page tells the reader no more than the
|
|
||||||
// plain-text answers it replaced did.
|
|
||||||
func errorPageText(status int) string {
|
|
||||||
switch status {
|
|
||||||
case http.StatusBadRequest:
|
|
||||||
return "The request could not be read."
|
|
||||||
case http.StatusForbidden:
|
|
||||||
return "The request was refused. If it came from a form " +
|
|
||||||
"left open for a long time, reload the page and try " +
|
|
||||||
"again."
|
|
||||||
case http.StatusNotFound:
|
|
||||||
return "There is nothing here. It may have been deleted, " +
|
|
||||||
"or the address may be wrong."
|
|
||||||
case http.StatusServiceUnavailable:
|
|
||||||
return "The server is busy. Please try again in a moment."
|
|
||||||
default: // http.StatusInternalServerError
|
|
||||||
return "Something went wrong on the server. Please try " +
|
|
||||||
"again."
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// UserInfo represents user information for templates
|
// UserInfo represents user information for templates
|
||||||
@@ -283,7 +189,6 @@ type templateDataWrapper struct {
|
|||||||
User *UserInfo
|
User *UserInfo
|
||||||
CSRFToken string
|
CSRFToken string
|
||||||
Version string
|
Version string
|
||||||
Notice *notice
|
|
||||||
Data any
|
Data any
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -310,26 +215,12 @@ func (s *Handlers) getUserInfo(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// renderTemplate renders a pre-parsed template with common
|
// renderTemplate renders a pre-parsed template with common
|
||||||
// data and answers 200.
|
// data
|
||||||
func (s *Handlers) renderTemplate(
|
func (s *Handlers) renderTemplate(
|
||||||
w http.ResponseWriter,
|
w http.ResponseWriter,
|
||||||
r *http.Request,
|
r *http.Request,
|
||||||
pageTemplate string,
|
pageTemplate string,
|
||||||
data any,
|
data any,
|
||||||
) {
|
|
||||||
s.renderTemplateStatus(w, r, pageTemplate, data, http.StatusOK)
|
|
||||||
}
|
|
||||||
|
|
||||||
// renderTemplateStatus is renderTemplate answering with status, for a
|
|
||||||
// form shown again with an error. Call it instead of WriteHeader
|
|
||||||
// followed by renderTemplate: the status is written only once the page
|
|
||||||
// has rendered, so a failed render can still answer 500.
|
|
||||||
func (s *Handlers) renderTemplateStatus(
|
|
||||||
w http.ResponseWriter,
|
|
||||||
r *http.Request,
|
|
||||||
pageTemplate string,
|
|
||||||
data any,
|
|
||||||
status int,
|
|
||||||
) {
|
) {
|
||||||
tmpl, ok := s.templates[pageTemplate]
|
tmpl, ok := s.templates[pageTemplate]
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -337,22 +228,14 @@ func (s *Handlers) renderTemplateStatus(
|
|||||||
"template not found",
|
"template not found",
|
||||||
"template", pageTemplate,
|
"template", pageTemplate,
|
||||||
)
|
)
|
||||||
s.renderError(w, r, http.StatusInternalServerError)
|
http.Error(
|
||||||
|
w, "Internal server error",
|
||||||
|
http.StatusInternalServerError,
|
||||||
|
)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
s.executeTemplate(
|
|
||||||
w, r, tmpl, s.pageData(r, data, noticeFor(r)), status,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// pageData adds the fields the shared layout renders to a page's own
|
|
||||||
// data. The layout shows the notice, when there is one, above the
|
|
||||||
// page.
|
|
||||||
func (s *Handlers) pageData(
|
|
||||||
r *http.Request, data any, pageNotice *notice,
|
|
||||||
) any {
|
|
||||||
userInfo := s.getUserInfo(r)
|
userInfo := s.getUserInfo(r)
|
||||||
csrfToken := middleware.CSRFToken(r)
|
csrfToken := middleware.CSRFToken(r)
|
||||||
|
|
||||||
@@ -366,34 +249,33 @@ func (s *Handlers) pageData(
|
|||||||
m["User"] = userInfo
|
m["User"] = userInfo
|
||||||
m["CSRFToken"] = csrfToken
|
m["CSRFToken"] = csrfToken
|
||||||
m["Version"] = version
|
m["Version"] = version
|
||||||
m["Notice"] = pageNotice
|
s.executeTemplate(w, tmpl, m)
|
||||||
|
|
||||||
return m
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
return templateDataWrapper{
|
wrapper := templateDataWrapper{
|
||||||
User: userInfo,
|
User: userInfo,
|
||||||
CSRFToken: csrfToken,
|
CSRFToken: csrfToken,
|
||||||
Version: version,
|
Version: version,
|
||||||
Notice: pageNotice,
|
|
||||||
Data: data,
|
Data: data,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
s.executeTemplate(w, tmpl, wrapper)
|
||||||
}
|
}
|
||||||
|
|
||||||
// executeTemplate renders the template into a buffer and writes status
|
// executeTemplate renders the template into a buffer and writes to
|
||||||
// and the page to the response only once rendering has fully
|
// the response only once rendering has fully succeeded. Executing
|
||||||
// succeeded. Executing straight into the ResponseWriter commits a
|
// straight into the ResponseWriter commits a partial body and a 200
|
||||||
// partial body and the status before a mid-render error can be
|
// status before a mid-render error can be reported, leaving no way
|
||||||
// reported, leaving no way to serve a 500. Buffering makes a page's
|
// to serve a 500. Buffering makes a page's rendered size resident
|
||||||
// rendered size resident memory per concurrent viewer, so every page
|
// memory per concurrent viewer, so every page owes it a bound: the
|
||||||
// owes it a bound: the event log caps each stored body at
|
// event log caps each stored body at maxRenderedBodyBytes for exactly
|
||||||
// maxRenderedBodyBytes for exactly this reason.
|
// this reason.
|
||||||
func (s *Handlers) executeTemplate(
|
func (s *Handlers) executeTemplate(
|
||||||
w http.ResponseWriter,
|
w http.ResponseWriter,
|
||||||
r *http.Request,
|
|
||||||
tmpl *template.Template,
|
tmpl *template.Template,
|
||||||
data any,
|
data any,
|
||||||
status int,
|
|
||||||
) {
|
) {
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
|
|
||||||
@@ -402,13 +284,15 @@ func (s *Handlers) executeTemplate(
|
|||||||
s.log.Error(
|
s.log.Error(
|
||||||
"failed to execute template", "error", err,
|
"failed to execute template", "error", err,
|
||||||
)
|
)
|
||||||
s.renderError(w, r, http.StatusInternalServerError)
|
http.Error(
|
||||||
|
w, "Internal server error",
|
||||||
|
http.StatusInternalServerError,
|
||||||
|
)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||||
w.WriteHeader(status)
|
|
||||||
|
|
||||||
_, err = buf.WriteTo(w)
|
_, err = buf.WriteTo(w)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package handlers_test
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"html/template"
|
"html/template"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
@@ -21,7 +20,6 @@ import (
|
|||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
"sneak.berlin/go/webhooker/internal/handlers"
|
||||||
"sneak.berlin/go/webhooker/internal/healthcheck"
|
"sneak.berlin/go/webhooker/internal/healthcheck"
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
"sneak.berlin/go/webhooker/internal/logger"
|
||||||
"sneak.berlin/go/webhooker/internal/metrics"
|
|
||||||
"sneak.berlin/go/webhooker/internal/middleware"
|
"sneak.berlin/go/webhooker/internal/middleware"
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
"sneak.berlin/go/webhooker/internal/session"
|
||||||
)
|
)
|
||||||
@@ -53,103 +51,23 @@ func (n *recordingNotifier) Tasks() []delivery.Task {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// recordingArchives is a delivery.Archives that records what it
|
// recordingEvictor is a delivery.WebhookEvictor that records
|
||||||
// was asked to do, so a test can prove that a deletion or rename
|
// the webhook ids it was asked to evict, so a test can prove
|
||||||
// path reached the delivery engine. After FailRenames, every
|
// that a deletion path reached the delivery engine.
|
||||||
// rename of that target fails with the given error. After
|
type recordingEvictor struct {
|
||||||
// BlockNextRename, the next rename is recorded and then waits.
|
mu sync.Mutex
|
||||||
type recordingArchives struct {
|
evicted []string
|
||||||
mu sync.Mutex
|
|
||||||
evicted []string
|
|
||||||
evictedTargets []string
|
|
||||||
renames []archiveRename
|
|
||||||
renameErrs map[string]error
|
|
||||||
entered chan struct{}
|
|
||||||
release chan struct{}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// errInjectedRename is the failure a test hands FailRenames.
|
func (r *recordingEvictor) EvictWebhook(webhookID string) {
|
||||||
var errInjectedRename = errors.New("injected rename failure")
|
|
||||||
|
|
||||||
// errNameTaken is what the delivery engine returns when a file
|
|
||||||
// already has an archive's new name, here archive-taken.db.
|
|
||||||
var errNameTaken = fmt.Errorf(
|
|
||||||
"%w: archive-taken.db", delivery.ErrArchiveNameTaken,
|
|
||||||
)
|
|
||||||
|
|
||||||
// archiveRename is one recorded Rename call.
|
|
||||||
type archiveRename struct {
|
|
||||||
TargetID string
|
|
||||||
WebhookName string
|
|
||||||
TargetName string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *recordingArchives) EvictWebhook(webhookID string) {
|
|
||||||
r.mu.Lock()
|
r.mu.Lock()
|
||||||
defer r.mu.Unlock()
|
defer r.mu.Unlock()
|
||||||
|
|
||||||
r.evicted = append(r.evicted, webhookID)
|
r.evicted = append(r.evicted, webhookID)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *recordingArchives) EvictTarget(targetID string) {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
|
|
||||||
r.evictedTargets = append(r.evictedTargets, targetID)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *recordingArchives) Rename(
|
|
||||||
targetID, webhookName, targetName string,
|
|
||||||
) error {
|
|
||||||
r.mu.Lock()
|
|
||||||
|
|
||||||
r.renames = append(r.renames, archiveRename{
|
|
||||||
TargetID: targetID,
|
|
||||||
WebhookName: webhookName,
|
|
||||||
TargetName: targetName,
|
|
||||||
})
|
|
||||||
err := r.renameErrs[targetID]
|
|
||||||
entered, release := r.entered, r.release
|
|
||||||
r.entered, r.release = nil, nil
|
|
||||||
|
|
||||||
r.mu.Unlock()
|
|
||||||
|
|
||||||
if entered != nil {
|
|
||||||
close(entered)
|
|
||||||
<-release
|
|
||||||
}
|
|
||||||
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// BlockNextRename makes the next rename, once recorded, wait until
|
|
||||||
// the returned release is called. The returned channel is closed
|
|
||||||
// when that rename starts waiting.
|
|
||||||
func (r *recordingArchives) BlockNextRename() (<-chan struct{}, func()) {
|
|
||||||
entered := make(chan struct{})
|
|
||||||
release := make(chan struct{})
|
|
||||||
|
|
||||||
r.mu.Lock()
|
|
||||||
r.entered, r.release = entered, release
|
|
||||||
r.mu.Unlock()
|
|
||||||
|
|
||||||
return entered, func() { close(release) }
|
|
||||||
}
|
|
||||||
|
|
||||||
// FailRenames makes every later rename of targetID fail with err.
|
|
||||||
func (r *recordingArchives) FailRenames(targetID string, err error) {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
|
|
||||||
if r.renameErrs == nil {
|
|
||||||
r.renameErrs = map[string]error{}
|
|
||||||
}
|
|
||||||
|
|
||||||
r.renameErrs[targetID] = err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Evicted returns a copy of the recorded webhook ids.
|
// Evicted returns a copy of the recorded webhook ids.
|
||||||
func (r *recordingArchives) Evicted() []string {
|
func (r *recordingEvictor) Evicted() []string {
|
||||||
r.mu.Lock()
|
r.mu.Lock()
|
||||||
defer r.mu.Unlock()
|
defer r.mu.Unlock()
|
||||||
|
|
||||||
@@ -159,62 +77,22 @@ func (r *recordingArchives) Evicted() []string {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// EvictedTargets returns a copy of the recorded target ids.
|
|
||||||
func (r *recordingArchives) EvictedTargets() []string {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
|
|
||||||
out := make([]string, len(r.evictedTargets))
|
|
||||||
copy(out, r.evictedTargets)
|
|
||||||
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// Renames returns a copy of the recorded renames.
|
|
||||||
func (r *recordingArchives) Renames() []archiveRename {
|
|
||||||
r.mu.Lock()
|
|
||||||
defer r.mu.Unlock()
|
|
||||||
|
|
||||||
out := make([]archiveRename, len(r.renames))
|
|
||||||
copy(out, r.renames)
|
|
||||||
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// newTestApp returns an app whose RequireStart fails the test when
|
|
||||||
// starting takes longer than fx's default start timeout of 15s. That
|
|
||||||
// limit catches a start that hangs, not a busy host: measured with make
|
|
||||||
// test on 2026-10-02 at host load 58-69 on 48 cores, the slowest of this
|
|
||||||
// package's starts took 0.49s.
|
|
||||||
func newTestApp(
|
func newTestApp(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
targets ...any,
|
targets ...any,
|
||||||
) *fxtest.App {
|
) *fxtest.App {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
return newTestAppWithConfig(
|
|
||||||
t, &config.Config{DataDir: t.TempDir()}, targets...,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// newTestAppWithConfig is newTestApp over a caller-supplied Config.
|
|
||||||
func newTestAppWithConfig(
|
|
||||||
t *testing.T,
|
|
||||||
cfg *config.Config,
|
|
||||||
targets ...any,
|
|
||||||
) *fxtest.App {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return fxtest.New(
|
return fxtest.New(
|
||||||
t,
|
t,
|
||||||
// fx's own log is discarded, not sent to t.Logf: a hook still
|
|
||||||
// running after a start or stop timeout would write there after
|
|
||||||
// the test has returned.
|
|
||||||
fx.NopLogger,
|
|
||||||
fx.Provide(
|
fx.Provide(
|
||||||
globals.New,
|
globals.New,
|
||||||
logger.New,
|
logger.New,
|
||||||
func() *config.Config { return cfg },
|
func() *config.Config {
|
||||||
|
return &config.Config{
|
||||||
|
DataDir: t.TempDir(),
|
||||||
|
}
|
||||||
|
},
|
||||||
database.New,
|
database.New,
|
||||||
database.NewWebhookDBManager,
|
database.NewWebhookDBManager,
|
||||||
healthcheck.New,
|
healthcheck.New,
|
||||||
@@ -225,14 +103,12 @@ func newTestAppWithConfig(
|
|||||||
func(n *recordingNotifier) delivery.Notifier {
|
func(n *recordingNotifier) delivery.Notifier {
|
||||||
return n
|
return n
|
||||||
},
|
},
|
||||||
func() *recordingArchives {
|
func() *recordingEvictor {
|
||||||
return &recordingArchives{}
|
return &recordingEvictor{}
|
||||||
},
|
},
|
||||||
func(r *recordingArchives) delivery.Archives {
|
func(r *recordingEvictor) delivery.WebhookEvictor {
|
||||||
return r
|
return r
|
||||||
},
|
},
|
||||||
metrics.NewRegistry,
|
|
||||||
metrics.New,
|
|
||||||
middleware.New,
|
middleware.New,
|
||||||
delivery.NewGuard,
|
delivery.NewGuard,
|
||||||
handlers.New,
|
handlers.New,
|
||||||
@@ -300,7 +176,7 @@ func TestHandleIndex_Authenticated(t *testing.T) {
|
|||||||
|
|
||||||
assert.Equal(t, http.StatusSeeOther, w2.Code)
|
assert.Equal(t, http.StatusSeeOther, w2.Code)
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, "/hooks", w2.Header().Get("Location"),
|
t, "/sources", w2.Header().Get("Location"),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -431,14 +307,10 @@ func TestRenderTemplateMidRenderErrorSendsNoPartialBody(t *testing.T) {
|
|||||||
t, http.StatusInternalServerError, w.Code,
|
t, http.StatusInternalServerError, w.Code,
|
||||||
"a failed render must report a 500",
|
"a failed render must report a 500",
|
||||||
)
|
)
|
||||||
assert.NotContains(
|
assert.Equal(
|
||||||
t, w.Body.String(), partialPageMarker,
|
t, "Internal server error\n", w.Body.String(),
|
||||||
"the response must carry no part of the aborted page",
|
"the response must carry no part of the aborted page",
|
||||||
)
|
)
|
||||||
assert.Contains(
|
|
||||||
t, w.Body.String(), "500 Internal Server Error",
|
|
||||||
"a failed render must answer with the error page",
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildDatabaseTargetConfig_Valid(t *testing.T) {
|
func TestBuildDatabaseTargetConfig_Valid(t *testing.T) {
|
||||||
|
|||||||
@@ -5,13 +5,13 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// HandleIndex returns a handler for the root path that redirects
|
// HandleIndex returns a handler for the root path that redirects
|
||||||
// based on authentication state: authenticated users go to /hooks
|
// based on authentication state: authenticated users go to /sources
|
||||||
// (the dashboard), unauthenticated users go to the login page.
|
// (the dashboard), unauthenticated users go to the login page.
|
||||||
func (s *Handlers) HandleIndex() http.HandlerFunc {
|
func (s *Handlers) HandleIndex() http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
sess, err := s.session.Get(r)
|
sess, err := s.session.Get(r)
|
||||||
if err == nil && s.session.IsAuthenticated(sess) {
|
if err == nil && s.session.IsAuthenticated(sess) {
|
||||||
http.Redirect(w, r, "/hooks", http.StatusSeeOther)
|
http.Redirect(w, r, "/sources", http.StatusSeeOther)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ package handlers_test
|
|||||||
// this package reach a value an UNAUTHENTICATED client picks outright
|
// this package reach a value an UNAUTHENTICATED client picks outright
|
||||||
// and of a length it picks outright:
|
// and of a length it picks outright:
|
||||||
//
|
//
|
||||||
// - the unknown-entrypoint DEBUG line on /h/{uuid}, whose
|
// - the unknown-entrypoint DEBUG line on /webhook/{uuid}, whose
|
||||||
// path segment matched no stored entrypoint and so is bounded by
|
// path segment matched no stored entrypoint and so is bounded by
|
||||||
// nothing;
|
// nothing;
|
||||||
// - the failed-login DEBUG lines, whose username is a form field.
|
// - the failed-login DEBUG lines, whose username is a form field.
|
||||||
@@ -190,12 +190,12 @@ func assertNoClientText(t *testing.T, buf *bytes.Buffer) {
|
|||||||
// route pattern.
|
// route pattern.
|
||||||
func receiverRouter(h *handlers.Handlers) *chi.Mux {
|
func receiverRouter(h *handlers.Handlers) *chi.Mux {
|
||||||
router := chi.NewRouter()
|
router := chi.NewRouter()
|
||||||
router.Post("/h/{uuid}", h.HandleWebhook())
|
router.Post("/webhook/{uuid}", h.HandleWebhook())
|
||||||
|
|
||||||
return router
|
return router
|
||||||
}
|
}
|
||||||
|
|
||||||
// postReceiver sends one POST at /h/<segment>.
|
// postReceiver sends one POST at /webhook/<segment>.
|
||||||
//
|
//
|
||||||
// RawPath is cleared after parsing so chi routes on the decoded path
|
// RawPath is cleared after parsing so chi routes on the decoded path
|
||||||
// and the handler sees the raw bytes rather than their percent-escaped
|
// and the handler sees the raw bytes rather than their percent-escaped
|
||||||
@@ -210,7 +210,7 @@ func postReceiver(
|
|||||||
req := httptest.NewRequestWithContext(
|
req := httptest.NewRequestWithContext(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
http.MethodPost,
|
http.MethodPost,
|
||||||
"/h/"+url.PathEscape(segment),
|
"/webhook/"+url.PathEscape(segment),
|
||||||
strings.NewReader(""),
|
strings.NewReader(""),
|
||||||
)
|
)
|
||||||
req.URL.RawPath = ""
|
req.URL.RawPath = ""
|
||||||
@@ -507,7 +507,7 @@ func TestVerificationCapacity_LogLineDoesNotTrackPathSize(
|
|||||||
http.StatusServiceUnavailable,
|
http.StatusServiceUnavailable,
|
||||||
postLoginAtPath(
|
postLoginAtPath(
|
||||||
t, h,
|
t, h,
|
||||||
"/hook/"+url.PathEscape(
|
"/source/"+url.PathEscape(
|
||||||
oversizedFill(fill),
|
oversizedFill(fill),
|
||||||
)+"/login",
|
)+"/login",
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -1,21 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
|
|
||||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
|
||||||
)
|
|
||||||
|
|
||||||
// HandleMetrics returns the Prometheus scrape handler for the
|
|
||||||
// registry built by metrics.NewRegistry, which the HTTP, delivery, Go
|
|
||||||
// runtime and process collectors register on. It is what
|
|
||||||
// promhttp.Handler builds for the global default registry, including
|
|
||||||
// the promhttp_metric_handler_* series that count scrapes, pointed at
|
|
||||||
// that registry instead.
|
|
||||||
func (s *Handlers) HandleMetrics() http.HandlerFunc {
|
|
||||||
reg := s.params.Registry
|
|
||||||
|
|
||||||
return promhttp.InstrumentMetricHandler(
|
|
||||||
reg, promhttp.HandlerFor(reg, promhttp.HandlerOpts{}),
|
|
||||||
).ServeHTTP
|
|
||||||
}
|
|
||||||
@@ -1,109 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import "net/http"
|
|
||||||
|
|
||||||
// noticeParam is the query parameter an action's redirect carries its
|
|
||||||
// notice code in.
|
|
||||||
const noticeParam = "notice"
|
|
||||||
|
|
||||||
// noticeCode names one of the fixed lines noticeFor knows. An action
|
|
||||||
// redirects with the code rather than the line, so nothing a client
|
|
||||||
// puts in the URL reaches the page: a code noticeFor does not know
|
|
||||||
// shows nothing.
|
|
||||||
type noticeCode string
|
|
||||||
|
|
||||||
// The codes of the actions on the webhook pages and of signing out.
|
|
||||||
// Replay's codes, with the reasons a replay can be refused, and
|
|
||||||
// resubmit's codes are defined beside those actions.
|
|
||||||
const (
|
|
||||||
webhookCreated noticeCode = "webhook-created"
|
|
||||||
webhookSaved noticeCode = "webhook-saved"
|
|
||||||
webhookDeleted noticeCode = "webhook-deleted"
|
|
||||||
entrypointAdded noticeCode = "entrypoint-added"
|
|
||||||
entrypointDeleted noticeCode = "entrypoint-deleted"
|
|
||||||
entrypointActivated noticeCode = "entrypoint-activated"
|
|
||||||
entrypointDeactivated noticeCode = "entrypoint-deactivated"
|
|
||||||
targetAdded noticeCode = "target-added"
|
|
||||||
targetSaved noticeCode = "target-saved"
|
|
||||||
targetDeleted noticeCode = "target-deleted"
|
|
||||||
targetActivated noticeCode = "target-activated"
|
|
||||||
targetDeactivated noticeCode = "target-deactivated"
|
|
||||||
signedOut noticeCode = "signed-out"
|
|
||||||
)
|
|
||||||
|
|
||||||
// notice is the line templates/notice.html shows above a page to say
|
|
||||||
// what an action did.
|
|
||||||
type notice struct {
|
|
||||||
Text string
|
|
||||||
|
|
||||||
// Failed shows the line as an error: the action was refused.
|
|
||||||
Failed bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// noticeFor returns the notice the request's URL names, or nil when it
|
|
||||||
// names none or an unknown code.
|
|
||||||
func noticeFor(r *http.Request) *notice {
|
|
||||||
n, ok := map[noticeCode]notice{
|
|
||||||
webhookCreated: {Text: "Webhook created."},
|
|
||||||
webhookSaved: {Text: "Webhook saved."},
|
|
||||||
webhookDeleted: {Text: "Webhook deleted."},
|
|
||||||
entrypointAdded: {Text: "Entrypoint added."},
|
|
||||||
entrypointDeleted: {Text: "Entrypoint deleted."},
|
|
||||||
entrypointActivated: {Text: "Entrypoint activated."},
|
|
||||||
entrypointDeactivated: {Text: "Entrypoint deactivated."},
|
|
||||||
targetAdded: {Text: "Target added."},
|
|
||||||
targetSaved: {Text: "Target saved."},
|
|
||||||
targetDeleted: {Text: "Target deleted."},
|
|
||||||
targetActivated: {Text: "Target activated."},
|
|
||||||
targetDeactivated: {Text: "Target deactivated."},
|
|
||||||
signedOut: {Text: "Signed out."},
|
|
||||||
|
|
||||||
replayQueued: {
|
|
||||||
Text: "Replay queued: a new delivery was created " +
|
|
||||||
"against the target's current configuration.",
|
|
||||||
},
|
|
||||||
replayTargetDeleted: {
|
|
||||||
Text: "Not replayed: the target this delivery was for " +
|
|
||||||
"has been deleted. Recreate the target, then replay.",
|
|
||||||
Failed: true,
|
|
||||||
},
|
|
||||||
replayTargetMissing: {
|
|
||||||
Text: "Not replayed: the target this delivery was for " +
|
|
||||||
"no longer exists.",
|
|
||||||
Failed: true,
|
|
||||||
},
|
|
||||||
replayTargetInactive: {
|
|
||||||
Text: "Not replayed: the target this delivery was for " +
|
|
||||||
"is deactivated. Activate it, then replay.",
|
|
||||||
Failed: true,
|
|
||||||
},
|
|
||||||
replayNotTerminal: {
|
|
||||||
Text: "Not replayed: this delivery has not finished yet.",
|
|
||||||
Failed: true,
|
|
||||||
},
|
|
||||||
replayInFlight: {
|
|
||||||
Text: "Not replayed: a delivery of this event to this " +
|
|
||||||
"target is already in flight.",
|
|
||||||
Failed: true,
|
|
||||||
},
|
|
||||||
|
|
||||||
resubmitQueued: {
|
|
||||||
Text: "Resubmitted: a new event was created from the " +
|
|
||||||
"stored one and queued to every active target.",
|
|
||||||
},
|
|
||||||
resubmitNoTargets: {
|
|
||||||
Text: "Resubmitted: a new event was created, but this " +
|
|
||||||
"source has no active targets, so nothing was queued.",
|
|
||||||
},
|
|
||||||
}[noticeCode(r.URL.Query().Get(noticeParam))]
|
|
||||||
if !ok {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return &n
|
|
||||||
}
|
|
||||||
|
|
||||||
// withNotice returns path with code added as its notice.
|
|
||||||
func withNotice(path string, code noticeCode) string {
|
|
||||||
return path + "?" + noticeParam + "=" + string(code)
|
|
||||||
}
|
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
package handlers
|
package handlers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
"github.com/go-chi/chi"
|
||||||
@@ -36,14 +37,14 @@ func (h *Handlers) HandlePasswordChange() http.HandlerFunc {
|
|||||||
err := r.ParseForm()
|
err := r.ParseForm()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.log.Error("failed to parse form", "error", err)
|
h.log.Error("failed to parse form", "error", err)
|
||||||
h.renderError(w, r, http.StatusBadRequest)
|
http.Error(w, "Bad request", http.StatusBadRequest)
|
||||||
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
successMessage, errorMessage, handled := h.applyPasswordChange(
|
successMessage, errorMessage, handled := h.applyPasswordChange(
|
||||||
|
r.Context(),
|
||||||
w,
|
w,
|
||||||
r,
|
|
||||||
sessionUsername,
|
sessionUsername,
|
||||||
// PostFormValue, not FormValue: the credential must
|
// PostFormValue, not FormValue: the credential must
|
||||||
// come from the body, never from the query string.
|
// come from the body, never from the query string.
|
||||||
@@ -65,12 +66,12 @@ func (h *Handlers) HandlePasswordChange() http.HandlerFunc {
|
|||||||
// applyPasswordChange verifies the current password and, on success,
|
// applyPasswordChange verifies the current password and, on success,
|
||||||
// persists a fresh hash for the user, reusing the same helpers that
|
// persists a fresh hash for the user, reusing the same helpers that
|
||||||
// bootstrap the admin user. It returns the success and error messages
|
// bootstrap the admin user. It returns the success and error messages
|
||||||
// to display on the profile page. On an internal failure it writes the
|
// to display on the profile page. On an internal failure it writes a
|
||||||
// error page itself and returns handled=false, signalling the caller
|
// 500 response itself and returns handled=false, signalling the caller
|
||||||
// to stop without re-rendering the page.
|
// to stop without re-rendering the page.
|
||||||
func (h *Handlers) applyPasswordChange(
|
func (h *Handlers) applyPasswordChange(
|
||||||
|
ctx context.Context,
|
||||||
w http.ResponseWriter,
|
w http.ResponseWriter,
|
||||||
r *http.Request,
|
|
||||||
username, currentPassword, newPassword, confirmPassword string,
|
username, currentPassword, newPassword, confirmPassword string,
|
||||||
) (string, string, bool) {
|
) (string, string, bool) {
|
||||||
// This endpoint verifies one password and hashes another, at
|
// This endpoint verifies one password and hashes another, at
|
||||||
@@ -78,10 +79,15 @@ func (h *Handlers) applyPasswordChange(
|
|||||||
// endpoint uses. The bound is per hash, not per endpoint: leaving
|
// endpoint uses. The bound is per hash, not per endpoint: leaving
|
||||||
// this path outside it would leave a hole in it. The slot is held
|
// this path outside it would leave a hole in it. The slot is held
|
||||||
// across both hashes.
|
// across both hashes.
|
||||||
release, ok := h.mw.BeginPasswordVerification(r.Context())
|
release, ok := h.mw.BeginPasswordVerification(ctx)
|
||||||
if !ok {
|
if !ok {
|
||||||
h.log.Warn("password verification capacity exhausted")
|
h.log.Warn("password verification capacity exhausted")
|
||||||
h.renderError(w, r, http.StatusServiceUnavailable)
|
http.Error(
|
||||||
|
w,
|
||||||
|
"The server is busy verifying credentials. "+
|
||||||
|
"Please try again.",
|
||||||
|
http.StatusServiceUnavailable,
|
||||||
|
)
|
||||||
|
|
||||||
return "", "", false
|
return "", "", false
|
||||||
}
|
}
|
||||||
@@ -97,7 +103,7 @@ func (h *Handlers) applyPasswordChange(
|
|||||||
).First(&user).Error
|
).First(&user).Error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.serverError(
|
h.serverError(
|
||||||
w, r, "failed to load user for password change", err,
|
w, "failed to load user for password change", err,
|
||||||
)
|
)
|
||||||
|
|
||||||
return "", "", false
|
return "", "", false
|
||||||
@@ -107,7 +113,7 @@ func (h *Handlers) applyPasswordChange(
|
|||||||
currentPassword, user.Password,
|
currentPassword, user.Password,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.serverError(w, r, "failed to verify password", err)
|
h.serverError(w, "failed to verify password", err)
|
||||||
|
|
||||||
return "", "", false
|
return "", "", false
|
||||||
}
|
}
|
||||||
@@ -126,7 +132,7 @@ func (h *Handlers) applyPasswordChange(
|
|||||||
|
|
||||||
hashedPassword, err := database.HashPassword(newPassword)
|
hashedPassword, err := database.HashPassword(newPassword)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.serverError(w, r, "failed to hash new password", err)
|
h.serverError(w, "failed to hash new password", err)
|
||||||
|
|
||||||
return "", "", false
|
return "", "", false
|
||||||
}
|
}
|
||||||
@@ -135,7 +141,7 @@ func (h *Handlers) applyPasswordChange(
|
|||||||
"password", hashedPassword,
|
"password", hashedPassword,
|
||||||
).Error
|
).Error
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.serverError(w, r, "failed to update password", err)
|
h.serverError(w, "failed to update password", err)
|
||||||
|
|
||||||
return "", "", false
|
return "", "", false
|
||||||
}
|
}
|
||||||
@@ -156,7 +162,7 @@ func (h *Handlers) profileOwnerOrDeny(
|
|||||||
) (string, string, bool) {
|
) (string, string, bool) {
|
||||||
requestedUsername := chi.URLParam(r, "username")
|
requestedUsername := chi.URLParam(r, "username")
|
||||||
if requestedUsername == "" {
|
if requestedUsername == "" {
|
||||||
h.renderError(w, r, http.StatusNotFound)
|
http.NotFound(w, r)
|
||||||
|
|
||||||
return "", "", false
|
return "", "", false
|
||||||
}
|
}
|
||||||
@@ -166,7 +172,7 @@ func (h *Handlers) profileOwnerOrDeny(
|
|||||||
// unexpected retrieval error.
|
// unexpected retrieval error.
|
||||||
sess, err := h.session.Get(r)
|
sess, err := h.session.Get(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.serverError(w, r, "failed to get session", err)
|
h.serverError(w, "failed to get session", err)
|
||||||
|
|
||||||
return "", "", false
|
return "", "", false
|
||||||
}
|
}
|
||||||
@@ -174,7 +180,10 @@ func (h *Handlers) profileOwnerOrDeny(
|
|||||||
sessionUsername, ok := h.session.GetUsername(sess)
|
sessionUsername, ok := h.session.GetUsername(sess)
|
||||||
if !ok {
|
if !ok {
|
||||||
h.log.Error("authenticated session missing username")
|
h.log.Error("authenticated session missing username")
|
||||||
h.renderError(w, r, http.StatusInternalServerError)
|
http.Error(
|
||||||
|
w, "Internal server error",
|
||||||
|
http.StatusInternalServerError,
|
||||||
|
)
|
||||||
|
|
||||||
return "", "", false
|
return "", "", false
|
||||||
}
|
}
|
||||||
@@ -182,14 +191,17 @@ func (h *Handlers) profileOwnerOrDeny(
|
|||||||
sessionUserID, ok := h.session.GetUserID(sess)
|
sessionUserID, ok := h.session.GetUserID(sess)
|
||||||
if !ok {
|
if !ok {
|
||||||
h.log.Error("authenticated session missing user ID")
|
h.log.Error("authenticated session missing user ID")
|
||||||
h.renderError(w, r, http.StatusInternalServerError)
|
http.Error(
|
||||||
|
w, "Internal server error",
|
||||||
|
http.StatusInternalServerError,
|
||||||
|
)
|
||||||
|
|
||||||
return "", "", false
|
return "", "", false
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only allow users to act on their own profile.
|
// Only allow users to act on their own profile.
|
||||||
if requestedUsername != sessionUsername {
|
if requestedUsername != sessionUsername {
|
||||||
h.renderError(w, r, http.StatusForbidden)
|
http.Error(w, "Forbidden", http.StatusForbidden)
|
||||||
|
|
||||||
return "", "", false
|
return "", "", false
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -128,9 +128,7 @@ func TestUserRoute_Unauthenticated_RedirectedByMiddleware(t *testing.T) {
|
|||||||
|
|
||||||
var sess *session.Session
|
var sess *session.Session
|
||||||
|
|
||||||
var h *handlers.Handlers
|
app := newTestApp(t, &log, &cfg, &sess)
|
||||||
|
|
||||||
app := newTestApp(t, &log, &cfg, &sess, &h)
|
|
||||||
app.RequireStart()
|
app.RequireStart()
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
t.Cleanup(app.RequireStop)
|
||||||
@@ -141,7 +139,7 @@ func TestUserRoute_Unauthenticated_RedirectedByMiddleware(t *testing.T) {
|
|||||||
|
|
||||||
router := chi.NewRouter()
|
router := chi.NewRouter()
|
||||||
router.Route("/user/{username}", func(r chi.Router) {
|
router.Route("/user/{username}", func(r chi.Router) {
|
||||||
r.Use(mw.CSRF(h.HandleErrorPage(http.StatusForbidden)))
|
r.Use(mw.CSRF())
|
||||||
r.Use(mw.RequireAuth())
|
r.Use(mw.RequireAuth())
|
||||||
r.Get("/", func(w http.ResponseWriter, _ *http.Request) {
|
r.Get("/", func(w http.ResponseWriter, _ *http.Request) {
|
||||||
handlerReached = true
|
handlerReached = true
|
||||||
@@ -162,10 +160,7 @@ func TestUserRoute_Unauthenticated_RedirectedByMiddleware(t *testing.T) {
|
|||||||
"handler must not be reached for unauthenticated request",
|
"handler must not be reached for unauthenticated request",
|
||||||
)
|
)
|
||||||
assert.Equal(t, http.StatusSeeOther, w.Code)
|
assert.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Equal(
|
assert.Equal(t, "/pages/login", w.Header().Get("Location"))
|
||||||
t, "/pages/login?next=%2Fuser%2Ftestuser",
|
|
||||||
w.Header().Get("Location"),
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// passwordChangeRequest builds a POST request to the password-change
|
// passwordChangeRequest builds a POST request to the password-change
|
||||||
|
|||||||
@@ -1,293 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"slices"
|
|
||||||
"strconv"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/dustin/go-humanize"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
)
|
|
||||||
|
|
||||||
// recentEventColumns is the recent events list's projection. It
|
|
||||||
// leaves out the body, for the reason maxRenderedBodyBytes gives,
|
|
||||||
// and reads its size from body_bytes, recorded when the event was
|
|
||||||
// stored.
|
|
||||||
const recentEventColumns = "id, created_at, method, content_type, " +
|
|
||||||
"resubmitted_from_id, body_bytes"
|
|
||||||
|
|
||||||
// recentAttemptColumns is the part of a recorded attempt the list
|
|
||||||
// uses. The event log's deliveryResultColumns also reads response
|
|
||||||
// bodies, which the list does not show.
|
|
||||||
const recentAttemptColumns = "delivery_id, status_code, created_at"
|
|
||||||
|
|
||||||
// RecentEventView is one row of the recent events list on a
|
|
||||||
// webhook's page.
|
|
||||||
type RecentEventView struct {
|
|
||||||
Method string
|
|
||||||
ContentType string
|
|
||||||
|
|
||||||
// ResubmittedFromID names the event this one was copied from,
|
|
||||||
// empty for an event that arrived on the receiver.
|
|
||||||
ResubmittedFromID string
|
|
||||||
|
|
||||||
// Received is how long ago the event arrived, and ReceivedUTC
|
|
||||||
// the full timestamp the page shows on hover.
|
|
||||||
Received string
|
|
||||||
ReceivedUTC string
|
|
||||||
|
|
||||||
// Size is the size of the stored body.
|
|
||||||
Size string
|
|
||||||
|
|
||||||
// ProcessingTime is how long the event's slowest delivery
|
|
||||||
// took; see processingTime.
|
|
||||||
ProcessingTime string
|
|
||||||
|
|
||||||
// Status is what the webhook's HTTP target answered, and
|
|
||||||
// StatusClass its colour; see targetStatus. Both are empty
|
|
||||||
// unless the webhook has exactly one HTTP target.
|
|
||||||
Status string
|
|
||||||
StatusClass string
|
|
||||||
}
|
|
||||||
|
|
||||||
// recentEventRow is one row of recentEventColumns.
|
|
||||||
type recentEventRow struct {
|
|
||||||
ID string
|
|
||||||
CreatedAt time.Time
|
|
||||||
Method string
|
|
||||||
ContentType string
|
|
||||||
ResubmittedFromID *string
|
|
||||||
BodyBytes uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
// recentAttemptRow is one row of recentAttemptColumns. CreatedAt is
|
|
||||||
// when the attempt's result was recorded, which is when the attempt
|
|
||||||
// finished.
|
|
||||||
type recentAttemptRow struct {
|
|
||||||
DeliveryID string
|
|
||||||
StatusCode int
|
|
||||||
CreatedAt time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// singleHTTPTargetID returns the ID of the webhook's HTTP target
|
|
||||||
// when it has exactly one, and "" when it has none or several.
|
|
||||||
func singleHTTPTargetID(targets []database.Target) string {
|
|
||||||
id := ""
|
|
||||||
count := 0
|
|
||||||
|
|
||||||
for i := range targets {
|
|
||||||
if targets[i].Type == database.TargetTypeHTTP {
|
|
||||||
id = targets[i].ID
|
|
||||||
count++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if count != 1 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
return id
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadRecentEvents loads the webhook's recentEventLimit newest
|
|
||||||
// events for its page, newest first. statusTargetID is the
|
|
||||||
// webhook's only HTTP target, or "" when the list shows no status.
|
|
||||||
func loadRecentEvents(
|
|
||||||
webhookDB *gorm.DB, webhookID, statusTargetID string,
|
|
||||||
) ([]RecentEventView, error) {
|
|
||||||
var rows []recentEventRow
|
|
||||||
|
|
||||||
err := webhookDB.Model(&database.Event{}).
|
|
||||||
Select(recentEventColumns).
|
|
||||||
Where("webhook_id = ?", webhookID).
|
|
||||||
Order("created_at DESC").
|
|
||||||
Limit(recentEventLimit).
|
|
||||||
Find(&rows).Error
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
eventIDs := make([]string, len(rows))
|
|
||||||
for i := range rows {
|
|
||||||
eventIDs[i] = rows[i].ID
|
|
||||||
}
|
|
||||||
|
|
||||||
// Oldest first, so an event's last delivery to a target is its
|
|
||||||
// newest: a replay adds a delivery rather than changing the
|
|
||||||
// earlier one.
|
|
||||||
var deliveries []database.Delivery
|
|
||||||
|
|
||||||
err = webhookDB.
|
|
||||||
Select("id, event_id, target_id, status, created_at").
|
|
||||||
Where("event_id IN ?", eventIDs).
|
|
||||||
Order("created_at ASC").
|
|
||||||
Find(&deliveries).Error
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
byEvent := make(map[string][]database.Delivery, len(rows))
|
|
||||||
deliveryIDs := make([]string, len(deliveries))
|
|
||||||
|
|
||||||
for i := range deliveries {
|
|
||||||
eventID := deliveries[i].EventID
|
|
||||||
byEvent[eventID] = append(byEvent[eventID], deliveries[i])
|
|
||||||
deliveryIDs[i] = deliveries[i].ID
|
|
||||||
}
|
|
||||||
|
|
||||||
attempts, err := loadRecentAttempts(webhookDB, deliveryIDs)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
views := make([]RecentEventView, len(rows))
|
|
||||||
for i := range rows {
|
|
||||||
views[i] = rows[i].view(
|
|
||||||
byEvent[rows[i].ID], attempts, statusTargetID,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return views, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// loadRecentAttempts loads the recorded attempts of the listed
|
|
||||||
// events' deliveries, keyed by delivery ID, each delivery's in
|
|
||||||
// attempt order. The IDs go in chunks for the reason
|
|
||||||
// deliveryIDChunkSize gives.
|
|
||||||
func loadRecentAttempts(
|
|
||||||
webhookDB *gorm.DB, deliveryIDs []string,
|
|
||||||
) (map[string][]recentAttemptRow, error) {
|
|
||||||
byDelivery := make(map[string][]recentAttemptRow)
|
|
||||||
|
|
||||||
for chunk := range slices.Chunk(deliveryIDs, deliveryIDChunkSize) {
|
|
||||||
var rows []recentAttemptRow
|
|
||||||
|
|
||||||
err := webhookDB.Model(&database.DeliveryResult{}).
|
|
||||||
Select(recentAttemptColumns).
|
|
||||||
Where("delivery_id IN ?", chunk).
|
|
||||||
Order("attempt_num ASC").
|
|
||||||
Find(&rows).Error
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := range rows {
|
|
||||||
id := rows[i].DeliveryID
|
|
||||||
byDelivery[id] = append(byDelivery[id], rows[i])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return byDelivery, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// view projects a loaded row for rendering. deliveries is the
|
|
||||||
// event's deliveries, oldest first, and attempts their recorded
|
|
||||||
// attempts keyed by delivery ID.
|
|
||||||
func (r *recentEventRow) view(
|
|
||||||
deliveries []database.Delivery,
|
|
||||||
attempts map[string][]recentAttemptRow,
|
|
||||||
statusTargetID string,
|
|
||||||
) RecentEventView {
|
|
||||||
v := RecentEventView{
|
|
||||||
Method: r.Method,
|
|
||||||
ContentType: r.ContentType,
|
|
||||||
Received: humanize.Time(r.CreatedAt),
|
|
||||||
ReceivedUTC: r.CreatedAt.UTC().Format(time.DateTime) + " UTC",
|
|
||||||
Size: humanize.Bytes(r.BodyBytes),
|
|
||||||
ProcessingTime: processingTime(deliveries, attempts),
|
|
||||||
}
|
|
||||||
|
|
||||||
if r.ResubmittedFromID != nil {
|
|
||||||
v.ResubmittedFromID = *r.ResubmittedFromID
|
|
||||||
}
|
|
||||||
|
|
||||||
if statusTargetID != "" {
|
|
||||||
v.Status, v.StatusClass = targetStatus(
|
|
||||||
deliveries, attempts, statusTargetID,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
return v
|
|
||||||
}
|
|
||||||
|
|
||||||
// processingTime is how long the event's slowest delivery took,
|
|
||||||
// from being queued to its last recorded attempt, time spent
|
|
||||||
// waiting between retries included. A delivery is queued when its
|
|
||||||
// event is received, or when an operator replays it, so a replay
|
|
||||||
// is timed from the replay rather than from the event's arrival.
|
|
||||||
// It is "in progress" while any delivery is pending or retrying,
|
|
||||||
// and empty for an event with no deliveries.
|
|
||||||
func processingTime(
|
|
||||||
deliveries []database.Delivery,
|
|
||||||
attempts map[string][]recentAttemptRow,
|
|
||||||
) string {
|
|
||||||
if len(deliveries) == 0 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
var slowest time.Duration
|
|
||||||
|
|
||||||
for i := range deliveries {
|
|
||||||
if !deliveries[i].Status.Terminal() {
|
|
||||||
return "in progress"
|
|
||||||
}
|
|
||||||
|
|
||||||
tries := attempts[deliveries[i].ID]
|
|
||||||
if len(tries) == 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
last := tries[len(tries)-1].CreatedAt
|
|
||||||
slowest = max(slowest, last.Sub(deliveries[i].CreatedAt))
|
|
||||||
}
|
|
||||||
|
|
||||||
return slowest.Round(time.Millisecond).String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// targetStatus is what the target answered for the event, and the
|
|
||||||
// colour to show it in: the HTTP status code of the last attempt of
|
|
||||||
// the event's newest delivery to the target. Without a code it is
|
|
||||||
// "no response" when that attempt failed before a response
|
|
||||||
// arrived, the delivery's status ("pending") before any attempt,
|
|
||||||
// and "not sent" when the event has no delivery to the target.
|
|
||||||
func targetStatus(
|
|
||||||
deliveries []database.Delivery,
|
|
||||||
attempts map[string][]recentAttemptRow,
|
|
||||||
targetID string,
|
|
||||||
) (string, string) {
|
|
||||||
newest := -1
|
|
||||||
|
|
||||||
for i := range deliveries {
|
|
||||||
if deliveries[i].TargetID == targetID {
|
|
||||||
newest = i
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if newest < 0 {
|
|
||||||
return "not sent", "text-gray-400"
|
|
||||||
}
|
|
||||||
|
|
||||||
tries := attempts[deliveries[newest].ID]
|
|
||||||
if len(tries) == 0 {
|
|
||||||
return string(deliveries[newest].Status), "text-gray-400"
|
|
||||||
}
|
|
||||||
|
|
||||||
code := tries[len(tries)-1].StatusCode
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case code == 0:
|
|
||||||
return "no response", "text-red-600"
|
|
||||||
case code >= http.StatusInternalServerError:
|
|
||||||
return strconv.Itoa(code), "text-red-600"
|
|
||||||
case code >= http.StatusBadRequest:
|
|
||||||
return strconv.Itoa(code), "text-yellow-600"
|
|
||||||
case code >= http.StatusMultipleChoices:
|
|
||||||
return strconv.Itoa(code), "text-gray-500"
|
|
||||||
case code >= http.StatusOK:
|
|
||||||
return strconv.Itoa(code), "text-green-600"
|
|
||||||
default:
|
|
||||||
return strconv.Itoa(code), "text-gray-500"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,362 +0,0 @@
|
|||||||
package handlers_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"gorm.io/gorm/clause"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
|
||||||
)
|
|
||||||
|
|
||||||
// statusTitle marks the status column's cell in a recent events
|
|
||||||
// row; it is absent from the page when the column is not shown.
|
|
||||||
const statusTitle = `title="HTTP status from the HTTP target"`
|
|
||||||
|
|
||||||
// recentEventsFixture is one started app and a webhook whose
|
|
||||||
// recent events list a test fills.
|
|
||||||
type recentEventsFixture struct {
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
webhook *database.Webhook
|
|
||||||
webhookDB *gorm.DB
|
|
||||||
}
|
|
||||||
|
|
||||||
func newRecentEventsFixture(t *testing.T) *recentEventsFixture {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
f := &recentEventsFixture{}
|
|
||||||
|
|
||||||
var dbMgr *database.WebhookDBManager
|
|
||||||
|
|
||||||
app := newTestApp(t, &f.h, &f.sess, &f.db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
f.webhook = seedWebhook(t, f.db)
|
|
||||||
|
|
||||||
webhookDB, err := dbMgr.GetDB(f.webhook.ID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
f.webhookDB = webhookDB
|
|
||||||
|
|
||||||
return f
|
|
||||||
}
|
|
||||||
|
|
||||||
func (f *recentEventsFixture) render(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return renderSourceDetailPage(t, f.h, f.sess, f.webhook.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
// event records an event received at receivedAt, with its body's
|
|
||||||
// size as the receiver records it.
|
|
||||||
func (f *recentEventsFixture) event(
|
|
||||||
t *testing.T, contentType, body string, receivedAt time.Time,
|
|
||||||
) *database.Event {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
event := &database.Event{
|
|
||||||
WebhookID: f.webhook.ID,
|
|
||||||
Method: http.MethodPost,
|
|
||||||
Body: body,
|
|
||||||
BodyBytes: int64(len(body)),
|
|
||||||
ContentType: contentType,
|
|
||||||
}
|
|
||||||
event.CreatedAt = receivedAt
|
|
||||||
|
|
||||||
require.NoError(t, f.webhookDB.Omit(
|
|
||||||
clause.Associations,
|
|
||||||
).Create(event).Error)
|
|
||||||
|
|
||||||
return event
|
|
||||||
}
|
|
||||||
|
|
||||||
// delivery records a delivery of the event to the target, queued
|
|
||||||
// when the event was received.
|
|
||||||
func (f *recentEventsFixture) delivery(
|
|
||||||
t *testing.T,
|
|
||||||
event *database.Event,
|
|
||||||
targetID string,
|
|
||||||
status database.DeliveryStatus,
|
|
||||||
) *database.Delivery {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
return f.deliveryQueuedAt(
|
|
||||||
t, event, targetID, status, event.CreatedAt,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// deliveryQueuedAt records a delivery of the event to the target,
|
|
||||||
// queued at queuedAt, as a replay is.
|
|
||||||
func (f *recentEventsFixture) deliveryQueuedAt(
|
|
||||||
t *testing.T,
|
|
||||||
event *database.Event,
|
|
||||||
targetID string,
|
|
||||||
status database.DeliveryStatus,
|
|
||||||
queuedAt time.Time,
|
|
||||||
) *database.Delivery {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
dlv := &database.Delivery{
|
|
||||||
EventID: event.ID,
|
|
||||||
TargetID: targetID,
|
|
||||||
Status: status,
|
|
||||||
}
|
|
||||||
dlv.CreatedAt = queuedAt
|
|
||||||
|
|
||||||
require.NoError(t, f.webhookDB.Omit(
|
|
||||||
clause.Associations,
|
|
||||||
).Create(dlv).Error)
|
|
||||||
|
|
||||||
return dlv
|
|
||||||
}
|
|
||||||
|
|
||||||
// attempt records one attempt of the delivery that finished took
|
|
||||||
// after the delivery was queued, with HTTP status code (0 for no
|
|
||||||
// response).
|
|
||||||
func (f *recentEventsFixture) attempt(
|
|
||||||
t *testing.T, dlv *database.Delivery, code int, took time.Duration,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
result := &database.DeliveryResult{
|
|
||||||
DeliveryID: dlv.ID,
|
|
||||||
AttemptNum: 1,
|
|
||||||
StatusCode: code,
|
|
||||||
}
|
|
||||||
result.CreatedAt = dlv.CreatedAt.Add(took)
|
|
||||||
|
|
||||||
require.NoError(t, f.webhookDB.Omit(
|
|
||||||
clause.Associations,
|
|
||||||
).Create(result).Error)
|
|
||||||
}
|
|
||||||
|
|
||||||
// statusCell is the status column's cell as the page renders it.
|
|
||||||
func statusCell(class, text string) string {
|
|
||||||
return `<span class="font-medium ` + class + `" ` + statusTitle +
|
|
||||||
`>` + text + `</span>`
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceDetail_ShowsFiftyNewestEvents proves the list is
|
|
||||||
// headed "50 Most Recent Events" and holds the 50 newest events,
|
|
||||||
// newest first, and not one more.
|
|
||||||
func TestHandleSourceDetail_ShowsFiftyNewestEvents(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
base := time.Now().Add(-time.Hour)
|
|
||||||
|
|
||||||
for i := range 51 {
|
|
||||||
f.event(
|
|
||||||
t, fmt.Sprintf("application/x-recent-%02d", i), "{}",
|
|
||||||
base.Add(time.Duration(i)*time.Second),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
body := f.render(t)
|
|
||||||
|
|
||||||
assert.Contains(t, body, ">50 Most Recent Events</h2>")
|
|
||||||
assert.Equal(t, 50, strings.Count(body, `title="Body size"`))
|
|
||||||
assert.NotContains(t, body, "application/x-recent-00")
|
|
||||||
assert.Contains(t, body, "application/x-recent-01")
|
|
||||||
assert.Less(
|
|
||||||
t,
|
|
||||||
strings.Index(body, "application/x-recent-50"),
|
|
||||||
strings.Index(body, "application/x-recent-49"),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceDetail_RecentEventColumns proves a row shows its
|
|
||||||
// time relative with the UTC timestamp on hover, its body size,
|
|
||||||
// and its processing time once every delivery has finished.
|
|
||||||
func TestHandleSourceDetail_RecentEventColumns(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
logTarget := seedTarget(t, f.db, f.webhook.ID, database.TargetTypeLog)
|
|
||||||
|
|
||||||
receivedAt := time.Now().Add(-210 * time.Second).
|
|
||||||
UTC().Truncate(time.Second)
|
|
||||||
|
|
||||||
done := f.event(
|
|
||||||
t, contentTypeJSON, strings.Repeat("x", 2048), receivedAt,
|
|
||||||
)
|
|
||||||
f.attempt(
|
|
||||||
t,
|
|
||||||
f.delivery(t, done, logTarget.ID, database.DeliveryStatusDelivered),
|
|
||||||
0, 1500*time.Millisecond,
|
|
||||||
)
|
|
||||||
|
|
||||||
waiting := f.event(t, "text/plain", "{}", receivedAt)
|
|
||||||
f.delivery(t, waiting, logTarget.ID, database.DeliveryStatusPending)
|
|
||||||
|
|
||||||
body := f.render(t)
|
|
||||||
|
|
||||||
assert.Contains(
|
|
||||||
t, body,
|
|
||||||
`<span title="`+receivedAt.Format(time.DateTime)+
|
|
||||||
` UTC">3 minutes ago</span>`,
|
|
||||||
)
|
|
||||||
assert.Contains(t, body, `<span title="Body size">2.0 kB</span>`)
|
|
||||||
assert.Contains(t, body, ">1.5s</span>")
|
|
||||||
assert.Contains(t, body, ">in progress</span>")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceDetail_StatusWithSingleHTTPTarget proves that a
|
|
||||||
// webhook with exactly one HTTP target shows, colour-coded, what
|
|
||||||
// that target answered for each event. The log target beside it
|
|
||||||
// does not count against "exactly one".
|
|
||||||
func TestHandleSourceDetail_StatusWithSingleHTTPTarget(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
target := seedTarget(t, f.db, f.webhook.ID, database.TargetTypeHTTP)
|
|
||||||
seedTarget(t, f.db, f.webhook.ID, database.TargetTypeLog)
|
|
||||||
|
|
||||||
now := time.Now()
|
|
||||||
|
|
||||||
for _, code := range []int{204, 302, 404, 503, 0} {
|
|
||||||
dlv := f.delivery(
|
|
||||||
t, f.event(t, contentTypeJSON, "{}", now), target.ID,
|
|
||||||
database.DeliveryStatusDelivered,
|
|
||||||
)
|
|
||||||
f.attempt(t, dlv, code, time.Second)
|
|
||||||
}
|
|
||||||
|
|
||||||
f.delivery(
|
|
||||||
t, f.event(t, contentTypeJSON, "{}", now), target.ID,
|
|
||||||
database.DeliveryStatusPending,
|
|
||||||
)
|
|
||||||
f.event(t, contentTypeJSON, "{}", now)
|
|
||||||
|
|
||||||
// A replay is a newer delivery, and its answer is the one shown.
|
|
||||||
replayed := f.event(t, contentTypeJSON, "{}", now)
|
|
||||||
f.attempt(t, f.delivery(
|
|
||||||
t, replayed, target.ID, database.DeliveryStatusFailed,
|
|
||||||
), 502, time.Second)
|
|
||||||
f.attempt(t, f.deliveryQueuedAt(
|
|
||||||
t, replayed, target.ID, database.DeliveryStatusDelivered,
|
|
||||||
now.Add(time.Minute),
|
|
||||||
), 200, time.Second)
|
|
||||||
|
|
||||||
body := f.render(t)
|
|
||||||
|
|
||||||
assert.Contains(t, body, statusCell("text-green-600", "204"))
|
|
||||||
assert.Contains(t, body, statusCell("text-gray-500", "302"))
|
|
||||||
assert.Contains(t, body, statusCell("text-yellow-600", "404"))
|
|
||||||
assert.Contains(t, body, statusCell("text-red-600", "503"))
|
|
||||||
assert.Contains(t, body, statusCell("text-red-600", "no response"))
|
|
||||||
assert.Contains(t, body, statusCell("text-gray-400", "pending"))
|
|
||||||
assert.Contains(t, body, statusCell("text-gray-400", "not sent"))
|
|
||||||
assert.Contains(t, body, statusCell("text-green-600", "200"))
|
|
||||||
assert.NotContains(t, body, ">502<")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceDetail_NoStatusWithoutSingleHTTPTarget proves the
|
|
||||||
// status column is absent when the webhook has no HTTP target or
|
|
||||||
// more than one.
|
|
||||||
func TestHandleSourceDetail_NoStatusWithoutSingleHTTPTarget(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
cases := map[string][]database.TargetType{
|
|
||||||
"none": {database.TargetTypeLog},
|
|
||||||
"several": {database.TargetTypeHTTP, database.TargetTypeHTTP},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, types := range cases {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
event := f.event(t, contentTypeJSON, "{}", time.Now())
|
|
||||||
|
|
||||||
for _, tt := range types {
|
|
||||||
target := seedTarget(t, f.db, f.webhook.ID, tt)
|
|
||||||
f.attempt(t, f.delivery(
|
|
||||||
t, event, target.ID,
|
|
||||||
database.DeliveryStatusDelivered,
|
|
||||||
), 200, time.Second)
|
|
||||||
}
|
|
||||||
|
|
||||||
body := f.render(t)
|
|
||||||
|
|
||||||
assert.Contains(t, body, `title="Body size"`)
|
|
||||||
assert.NotContains(t, body, statusTitle)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleWebhook_RecordsBodySize proves the receiver records the
|
|
||||||
// body's size in bytes, not characters, with the event it stores.
|
|
||||||
func TestHandleWebhook_RecordsBodySize(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
seedEntrypoint(t, f.db, f.webhook.ID)
|
|
||||||
|
|
||||||
// Two bytes per character.
|
|
||||||
body := strings.Repeat("é", 1024)
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(
|
|
||||||
context.Background(), http.MethodPost, "/h/x",
|
|
||||||
strings.NewReader(body),
|
|
||||||
)
|
|
||||||
|
|
||||||
rctx := chi.NewRouteContext()
|
|
||||||
rctx.URLParams.Add("uuid", "ep-"+f.webhook.ID)
|
|
||||||
|
|
||||||
req = req.WithContext(context.WithValue(
|
|
||||||
req.Context(), chi.RouteCtxKey, rctx,
|
|
||||||
))
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
f.h.HandleWebhook().ServeHTTP(w, req)
|
|
||||||
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
|
|
||||||
var stored database.Event
|
|
||||||
|
|
||||||
require.NoError(t, f.webhookDB.First(&stored).Error)
|
|
||||||
assert.Equal(t, int64(2048), stored.BodyBytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceDetail_FailedLoadIsAnError proves that when the
|
|
||||||
// list cannot be loaded the page answers with an error, rather than
|
|
||||||
// an empty list claiming the webhook has no events.
|
|
||||||
func TestHandleSourceDetail_FailedLoadIsAnError(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
f := newRecentEventsFixture(t)
|
|
||||||
target := seedTarget(t, f.db, f.webhook.ID, database.TargetTypeHTTP)
|
|
||||||
|
|
||||||
f.attempt(t, f.delivery(
|
|
||||||
t, f.event(t, contentTypeJSON, "{}", time.Now()), target.ID,
|
|
||||||
database.DeliveryStatusDelivered,
|
|
||||||
), 200, time.Second)
|
|
||||||
|
|
||||||
// The attempts are the list's last query, so its events and
|
|
||||||
// deliveries have already loaded when it fails.
|
|
||||||
require.NoError(t, f.webhookDB.Exec(
|
|
||||||
"DROP TABLE delivery_results",
|
|
||||||
).Error)
|
|
||||||
|
|
||||||
w := serveSourceDetailPage(t, f.h, f.sess, f.webhook.ID)
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusInternalServerError, w.Code)
|
|
||||||
assert.NotContains(t, w.Body.String(), "No events received yet.")
|
|
||||||
}
|
|
||||||
@@ -1,129 +0,0 @@
|
|||||||
package handlers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/netip"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
// notSet is what the Settings page shows for a value that is empty.
|
|
||||||
const notSet = "not set"
|
|
||||||
|
|
||||||
// settingRow is one line of the Settings page: an environment
|
|
||||||
// variable, what it controls, and the value the server loaded for it.
|
|
||||||
type settingRow struct {
|
|
||||||
Name string
|
|
||||||
Description string
|
|
||||||
Value string
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleSettings returns a handler for the read-only Settings page,
|
|
||||||
// which lists the configuration the server started with.
|
|
||||||
func (h *Handlers) HandleSettings() http.HandlerFunc {
|
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
h.renderTemplate(w, r, "settings.html", map[string]any{
|
|
||||||
"Settings": settingRows(h.params.Config),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// settingRows lists every field of cfg under the environment variable
|
|
||||||
// it is read from, with the description the README's configuration
|
|
||||||
// table gives it (less its pointers to other README sections), in the
|
|
||||||
// table's order. METRICS_PASSWORD and SENTRY_DSN are credentials, so
|
|
||||||
// their values never reach the page: only whether they are set.
|
|
||||||
func settingRows(cfg *config.Config) []settingRow {
|
|
||||||
metricsUsername := cfg.MetricsUsername
|
|
||||||
if metricsUsername == "" {
|
|
||||||
metricsUsername = notSet
|
|
||||||
}
|
|
||||||
|
|
||||||
return []settingRow{
|
|
||||||
{"WEBHOOKER_ENVIRONMENT", "dev or prod", cfg.Environment},
|
|
||||||
{"PORT", "HTTP listen port", strconv.Itoa(cfg.Port)},
|
|
||||||
{
|
|
||||||
"BIND_ADDRESS",
|
|
||||||
"IP address the HTTP listener binds. Loopback by default, " +
|
|
||||||
"so the cleartext listener is not published on every " +
|
|
||||||
"interface. The Docker image ships 0.0.0.0 instead",
|
|
||||||
cfg.BindAddress,
|
|
||||||
},
|
|
||||||
{"DATA_DIR", "Directory for all SQLite databases", cfg.DataDir},
|
|
||||||
{"DEBUG", "Enable debug logging", strconv.FormatBool(cfg.Debug)},
|
|
||||||
{
|
|
||||||
"METRICS_USERNAME",
|
|
||||||
"Basic auth username for /metrics. Must be set together " +
|
|
||||||
"with METRICS_PASSWORD; one without the other fails " +
|
|
||||||
"startup",
|
|
||||||
metricsUsername,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"METRICS_PASSWORD",
|
|
||||||
"Basic auth password for /metrics. Must be set together " +
|
|
||||||
"with METRICS_USERNAME; one without the other fails " +
|
|
||||||
"startup",
|
|
||||||
setOrNotSet(cfg.MetricsPassword),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"SENTRY_DSN",
|
|
||||||
"Sentry error reporting DSN. Unset leaves error reporting " +
|
|
||||||
"off; a value the Sentry SDK cannot parse fails startup " +
|
|
||||||
"rather than serving with reporting silently off",
|
|
||||||
setOrNotSet(cfg.SentryDSN),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"RETENTION_SWEEP_INTERVAL",
|
|
||||||
"How often the retention reaper and archive sweeper run " +
|
|
||||||
"(Go duration, must be positive). A value that does " +
|
|
||||||
"not parse, or is zero or negative, fails startup",
|
|
||||||
cfg.RetentionSweepInterval.String(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"SESSION_IDLE_TIMEOUT",
|
|
||||||
"Idle session timeout (Go duration)",
|
|
||||||
cfg.SessionIdleTimeout.String(),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"RECEIVER_RATE_LIMIT",
|
|
||||||
"Receiver requests/minute per IP per entrypoint " +
|
|
||||||
"(10x that per IP across the route)",
|
|
||||||
strconv.Itoa(cfg.ReceiverRateLimit),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"TRUSTED_PROXIES",
|
|
||||||
"CIDRs whose forwarded headers are trusted. A set value " +
|
|
||||||
"replaces the default. If any client can reach webhooker, " +
|
|
||||||
"or the proxy in front of it, from an RFC 1918 source " +
|
|
||||||
"address, set it to the proxy's address alone",
|
|
||||||
cidrList(cfg.TrustedProxies),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"ALLOWED_EGRESS_CIDRS",
|
|
||||||
"CIDRs that delivery targets may reach despite the " +
|
|
||||||
"SSRF blocklist",
|
|
||||||
cidrList(cfg.AllowedEgressCIDRs),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// setOrNotSet is how the Settings page shows a credential: whether it
|
|
||||||
// has a value, never the value itself.
|
|
||||||
func setOrNotSet(value string) string {
|
|
||||||
if value == "" {
|
|
||||||
return notSet
|
|
||||||
}
|
|
||||||
|
|
||||||
return "set"
|
|
||||||
}
|
|
||||||
|
|
||||||
// cidrList renders a CIDR list setting for the Settings page.
|
|
||||||
func cidrList(prefixes []netip.Prefix) string {
|
|
||||||
if len(prefixes) == 0 {
|
|
||||||
return "none"
|
|
||||||
}
|
|
||||||
|
|
||||||
return strings.Join(config.PrefixStrings(prefixes), ", ")
|
|
||||||
}
|
|
||||||
@@ -1,148 +0,0 @@
|
|||||||
package handlers_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"html"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"net/netip"
|
|
||||||
"regexp"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"sneak.berlin/go/webhooker/internal/config"
|
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
|
||||||
)
|
|
||||||
|
|
||||||
// settingsShown renders the Settings page over cfg as a logged-in user
|
|
||||||
// and returns the value it shows for each variable name, plus the
|
|
||||||
// whole page.
|
|
||||||
func settingsShown(
|
|
||||||
t *testing.T, cfg *config.Config,
|
|
||||||
) (map[string]string, string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
var h *handlers.Handlers
|
|
||||||
|
|
||||||
var sess *session.Session
|
|
||||||
|
|
||||||
app := newTestAppWithConfig(t, cfg, &h, &sess)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(
|
|
||||||
context.Background(), http.MethodGet, "/settings", nil,
|
|
||||||
)
|
|
||||||
for _, c := range authenticatedCookies(t, sess, "id", "admin") {
|
|
||||||
req.AddCookie(c)
|
|
||||||
}
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
h.HandleSettings().ServeHTTP(w, req)
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
|
|
||||||
body := w.Body.String()
|
|
||||||
|
|
||||||
row := regexp.MustCompile(
|
|
||||||
`<code[^>]*>([A-Z_]+)</code>\s*<code[^>]*>([^<]*)</code>`,
|
|
||||||
)
|
|
||||||
|
|
||||||
shown := map[string]string{}
|
|
||||||
for _, match := range row.FindAllStringSubmatch(body, -1) {
|
|
||||||
shown[match[1]] = html.UnescapeString(match[2])
|
|
||||||
}
|
|
||||||
|
|
||||||
return shown, body
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSettingsPageShowsLoadedConfiguration(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
// Each of METRICS_USERNAME, METRICS_PASSWORD and SENTRY_DSN is the
|
|
||||||
// only one of the three set in one of the content tests, so each
|
|
||||||
// row is checked against its own field.
|
|
||||||
cfg := &config.Config{
|
|
||||||
DataDir: t.TempDir(),
|
|
||||||
Debug: true,
|
|
||||||
Environment: config.EnvironmentDev,
|
|
||||||
MetricsUsername: "scraper",
|
|
||||||
MetricsPassword: "",
|
|
||||||
Port: 9123,
|
|
||||||
SentryDSN: "",
|
|
||||||
BindAddress: "192.0.2.10",
|
|
||||||
RetentionSweepInterval: 17 * time.Minute,
|
|
||||||
SessionIdleTimeout: 3 * time.Hour,
|
|
||||||
ReceiverRateLimit: 77,
|
|
||||||
TrustedProxies: []netip.Prefix{
|
|
||||||
netip.MustParsePrefix("10.1.0.0/16"),
|
|
||||||
},
|
|
||||||
AllowedEgressCIDRs: []netip.Prefix{
|
|
||||||
netip.MustParsePrefix("192.168.5.0/24"),
|
|
||||||
netip.MustParsePrefix("fd00::/8"),
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
shown, body := settingsShown(t, cfg)
|
|
||||||
|
|
||||||
assert.Equal(t, map[string]string{
|
|
||||||
"WEBHOOKER_ENVIRONMENT": "dev",
|
|
||||||
"PORT": "9123",
|
|
||||||
"BIND_ADDRESS": "192.0.2.10",
|
|
||||||
"DATA_DIR": cfg.DataDir,
|
|
||||||
"DEBUG": "true",
|
|
||||||
"METRICS_USERNAME": "scraper",
|
|
||||||
"METRICS_PASSWORD": "not set",
|
|
||||||
"SENTRY_DSN": "not set",
|
|
||||||
"RETENTION_SWEEP_INTERVAL": "17m0s",
|
|
||||||
"SESSION_IDLE_TIMEOUT": "3h0m0s",
|
|
||||||
"RECEIVER_RATE_LIMIT": "77",
|
|
||||||
"TRUSTED_PROXIES": "10.1.0.0/16",
|
|
||||||
"ALLOWED_EGRESS_CIDRS": "192.168.5.0/24, fd00::/8",
|
|
||||||
}, shown)
|
|
||||||
|
|
||||||
assert.Contains(
|
|
||||||
t, body, `href="/settings"`,
|
|
||||||
"the navigation bar links to the page",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSettingsPageShowsUnsetValues(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const metricsPassword = "metrics-password-1f9a"
|
|
||||||
|
|
||||||
shown, body := settingsShown(t, &config.Config{
|
|
||||||
DataDir: t.TempDir(),
|
|
||||||
MetricsPassword: metricsPassword,
|
|
||||||
})
|
|
||||||
|
|
||||||
assert.Equal(t, "not set", shown["METRICS_USERNAME"])
|
|
||||||
assert.Equal(t, "set", shown["METRICS_PASSWORD"])
|
|
||||||
assert.Equal(t, "not set", shown["SENTRY_DSN"])
|
|
||||||
assert.NotContains(t, body, metricsPassword)
|
|
||||||
assert.Equal(t, "none", shown["TRUSTED_PROXIES"])
|
|
||||||
assert.Equal(t, "none", shown["ALLOWED_EGRESS_CIDRS"])
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSettingsPageShowsSentryDSNOnlyAsSet(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
const (
|
|
||||||
sentryKey = "dsnkey7c2e"
|
|
||||||
sentryDSN = "https://" + sentryKey + "@errors.example.com/42"
|
|
||||||
)
|
|
||||||
|
|
||||||
shown, body := settingsShown(t, &config.Config{
|
|
||||||
DataDir: t.TempDir(),
|
|
||||||
SentryDSN: sentryDSN,
|
|
||||||
})
|
|
||||||
|
|
||||||
assert.Equal(t, "not set", shown["METRICS_USERNAME"])
|
|
||||||
assert.Equal(t, "not set", shown["METRICS_PASSWORD"])
|
|
||||||
assert.Equal(t, "set", shown["SENTRY_DSN"])
|
|
||||||
assert.NotContains(t, body, sentryKey)
|
|
||||||
}
|
|
||||||
@@ -1,10 +1,8 @@
|
|||||||
package handlers_test
|
package handlers_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"log/slog"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
@@ -17,7 +15,6 @@ import (
|
|||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
"gorm.io/gorm/clause"
|
"gorm.io/gorm/clause"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/delivery"
|
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
"sneak.berlin/go/webhooker/internal/handlers"
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
"sneak.berlin/go/webhooker/internal/session"
|
||||||
)
|
)
|
||||||
@@ -82,10 +79,6 @@ func seedTarget(
|
|||||||
// from a delete statement.
|
// from a delete statement.
|
||||||
var errInjectedDelete = errors.New("injected delete failure")
|
var errInjectedDelete = errors.New("injected delete failure")
|
||||||
|
|
||||||
// errInjectedSave is the failure failSaveOnTable reports from a
|
|
||||||
// save of an existing row.
|
|
||||||
var errInjectedSave = errors.New("injected save failure")
|
|
||||||
|
|
||||||
// seedEntrypoint inserts an entrypoint for a webhook.
|
// seedEntrypoint inserts an entrypoint for a webhook.
|
||||||
func seedEntrypoint(
|
func seedEntrypoint(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
@@ -153,42 +146,19 @@ func failDeleteOnTable(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// failSaveOnTable is failDeleteOnTable for saves: every update of
|
|
||||||
// an existing row in the named table fails.
|
|
||||||
func failSaveOnTable(
|
|
||||||
t *testing.T,
|
|
||||||
db *database.Database,
|
|
||||||
table string,
|
|
||||||
) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
require.NoError(t, db.DB().Callback().Update().
|
|
||||||
Before("gorm:update").
|
|
||||||
Register(
|
|
||||||
"test:fail_save_"+table,
|
|
||||||
func(tx *gorm.DB) {
|
|
||||||
if tx.Statement.Table == table {
|
|
||||||
_ = tx.AddError(errInjectedSave)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// archivePathFor returns the archive database path the
|
// archivePathFor returns the archive database path the
|
||||||
// delivery engine would use for a database target: beside the
|
// delivery engine would use for a webhook: beside the webhook's
|
||||||
// webhook's event database in the data directory.
|
// event database in the data directory.
|
||||||
func archivePathFor(
|
func archivePathFor(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
mgr *database.WebhookDBManager,
|
mgr *database.WebhookDBManager,
|
||||||
wh *database.Webhook,
|
webhookID string,
|
||||||
tgt *database.Target,
|
|
||||||
) string {
|
) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
return filepath.Join(
|
return filepath.Join(
|
||||||
filepath.Dir(mgr.DBPath(wh.ID)),
|
filepath.Dir(mgr.DBPath(webhookID)),
|
||||||
delivery.ArchiveFileName(wh.Name, tgt.Name, tgt.ID),
|
"archive-"+webhookID+".db",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -225,8 +195,8 @@ func postRequest(
|
|||||||
|
|
||||||
// TestHandleSourceDelete_EvictsArchiveWriter proves that
|
// TestHandleSourceDelete_EvictsArchiveWriter proves that
|
||||||
// deleting a webhook reaches the delivery engine and releases
|
// deleting a webhook reaches the delivery engine and releases
|
||||||
// the webhook's archive writers, exercised through the real
|
// the webhook's archive writer, exercised through the real
|
||||||
// deletion handler rather than by calling the engine directly.
|
// deletion handler rather than by calling the evictor directly.
|
||||||
func TestHandleSourceDelete_EvictsArchiveWriter(t *testing.T) {
|
func TestHandleSourceDelete_EvictsArchiveWriter(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
@@ -234,7 +204,7 @@ func TestHandleSourceDelete_EvictsArchiveWriter(t *testing.T) {
|
|||||||
h *handlers.Handlers
|
h *handlers.Handlers
|
||||||
sess *session.Session
|
sess *session.Session
|
||||||
db *database.Database
|
db *database.Database
|
||||||
ev *recordingArchives
|
ev *recordingEvictor
|
||||||
)
|
)
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &ev)
|
app := newTestApp(t, &h, &sess, &db, &ev)
|
||||||
@@ -250,7 +220,7 @@ func TestHandleSourceDelete_EvictsArchiveWriter(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
req := postRequest(
|
req := postRequest(
|
||||||
"/hook/"+wh.ID+"/delete",
|
"/source/"+wh.ID+"/delete",
|
||||||
cookies,
|
cookies,
|
||||||
map[string]string{paramSourceID: wh.ID},
|
map[string]string{paramSourceID: wh.ID},
|
||||||
)
|
)
|
||||||
@@ -284,10 +254,9 @@ func TestHandleSourceDelete_KeepsArchiveFile(t *testing.T) {
|
|||||||
t.Cleanup(app.RequireStop)
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
wh := seedWebhook(t, db)
|
||||||
tgt := seedTarget(t, db, wh.ID, database.TargetTypeDatabase)
|
|
||||||
|
|
||||||
// Place an archive file where the delivery engine would.
|
// Place an archive file where the delivery engine would.
|
||||||
archivePath := archivePathFor(t, mgr, wh, tgt)
|
archivePath := archivePathFor(t, mgr, wh.ID)
|
||||||
require.NoError(
|
require.NoError(
|
||||||
t,
|
t,
|
||||||
writeArchivePlaceholder(archivePath),
|
writeArchivePlaceholder(archivePath),
|
||||||
@@ -298,7 +267,7 @@ func TestHandleSourceDelete_KeepsArchiveFile(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
req := postRequest(
|
req := postRequest(
|
||||||
"/hook/"+wh.ID+"/delete",
|
"/source/"+wh.ID+"/delete",
|
||||||
cookies,
|
cookies,
|
||||||
map[string]string{paramSourceID: wh.ID},
|
map[string]string{paramSourceID: wh.ID},
|
||||||
)
|
)
|
||||||
@@ -354,7 +323,7 @@ func TestHandleSourceDelete_FailedDeleteKeepsEverything(
|
|||||||
)
|
)
|
||||||
|
|
||||||
req := postRequest(
|
req := postRequest(
|
||||||
"/hook/"+wh.ID+"/delete",
|
"/source/"+wh.ID+"/delete",
|
||||||
cookies,
|
cookies,
|
||||||
map[string]string{paramSourceID: wh.ID},
|
map[string]string{paramSourceID: wh.ID},
|
||||||
)
|
)
|
||||||
@@ -368,7 +337,7 @@ func TestHandleSourceDelete_FailedDeleteKeepsEverything(
|
|||||||
)
|
)
|
||||||
assert.Empty(
|
assert.Empty(
|
||||||
t, w.Header().Get("Location"),
|
t, w.Header().Get("Location"),
|
||||||
"a failed deletion must not redirect to /hooks",
|
"a failed deletion must not redirect to /sources",
|
||||||
)
|
)
|
||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
@@ -433,7 +402,7 @@ func TestHandleSourceDelete_RemovesConfigAndEventDatabase(
|
|||||||
)
|
)
|
||||||
|
|
||||||
req := postRequest(
|
req := postRequest(
|
||||||
"/hook/"+wh.ID+"/delete",
|
"/source/"+wh.ID+"/delete",
|
||||||
cookies,
|
cookies,
|
||||||
map[string]string{paramSourceID: wh.ID},
|
map[string]string{paramSourceID: wh.ID},
|
||||||
)
|
)
|
||||||
@@ -442,9 +411,7 @@ func TestHandleSourceDelete_RemovesConfigAndEventDatabase(
|
|||||||
h.HandleSourceDelete().ServeHTTP(w, req)
|
h.HandleSourceDelete().ServeHTTP(w, req)
|
||||||
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Equal(
|
assert.Equal(t, "/sources", w.Header().Get("Location"))
|
||||||
t, "/hooks?notice=webhook-deleted", w.Header().Get("Location"),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t, int64(0),
|
t, int64(0),
|
||||||
@@ -468,132 +435,68 @@ func TestHandleSourceDelete_RemovesConfigAndEventDatabase(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleSourceDelete_LeftoverSidecar proves that when the event
|
// TestHandleTargetDelete_EvictsWhenLastDatabaseTargetGone
|
||||||
// database file is removed but a sidecar beside it is not, the
|
// proves that removing the last database target releases the
|
||||||
// operator is told the events are gone, never that the event
|
// archive writer.
|
||||||
// database file is still there.
|
func TestHandleTargetDelete_EvictsWhenLastDatabaseTargetGone(
|
||||||
func TestHandleSourceDelete_LeftoverSidecar(t *testing.T) {
|
t *testing.T,
|
||||||
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
var (
|
var (
|
||||||
h *handlers.Handlers
|
h *handlers.Handlers
|
||||||
sess *session.Session
|
sess *session.Session
|
||||||
db *database.Database
|
db *database.Database
|
||||||
mgr *database.WebhookDBManager
|
ev *recordingEvictor
|
||||||
)
|
)
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &mgr)
|
app := newTestApp(t, &h, &sess, &db, &ev)
|
||||||
app.RequireStart()
|
app.RequireStart()
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
logs := new(bytes.Buffer)
|
|
||||||
h.SetLogForTest(slog.New(slog.NewTextHandler(logs, nil)))
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
wh := seedWebhook(t, db)
|
||||||
|
tgt := seedTarget(
|
||||||
require.NoError(t, mgr.CreateDB(wh.ID))
|
t, db, wh.ID, database.TargetTypeDatabase,
|
||||||
// Closing removes the sidecars, so the -wal below is the only
|
)
|
||||||
// one there.
|
|
||||||
require.NoError(t, mgr.CloseAll())
|
|
||||||
|
|
||||||
// A non-empty directory in the -wal file's place, which
|
|
||||||
// os.Remove cannot remove whoever runs the test.
|
|
||||||
eventDBPath := mgr.DBPath(wh.ID)
|
|
||||||
require.NoError(t, os.MkdirAll(
|
|
||||||
filepath.Join(eventDBPath+"-wal", "keep"), 0o700,
|
|
||||||
))
|
|
||||||
|
|
||||||
cookies := authenticatedCookies(
|
cookies := authenticatedCookies(
|
||||||
t, sess, deleteTestUserID, deleteTestUsername,
|
t, sess, deleteTestUserID, deleteTestUsername,
|
||||||
)
|
)
|
||||||
|
|
||||||
req := postRequest(
|
req := postRequest(
|
||||||
"/hook/"+wh.ID+"/delete",
|
"/source/"+wh.ID+"/targets/"+tgt.ID+"/delete",
|
||||||
cookies,
|
cookies,
|
||||||
map[string]string{paramSourceID: wh.ID},
|
map[string]string{
|
||||||
|
paramSourceID: wh.ID,
|
||||||
|
paramTargetID: tgt.ID,
|
||||||
|
},
|
||||||
)
|
)
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
h.HandleSourceDelete().ServeHTTP(w, req)
|
h.HandleTargetDelete().ServeHTTP(w, req)
|
||||||
|
|
||||||
assert.Equal(t, http.StatusInternalServerError, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.NoFileExists(t, eventDBPath)
|
assert.Equal(
|
||||||
assert.Contains(t, logs.String(), "its events are gone")
|
t, []string{wh.ID}, ev.Evicted(),
|
||||||
assert.Contains(t, logs.String(), eventDBPath+"-wal")
|
"removing the last database target should evict",
|
||||||
assert.NotContains(
|
|
||||||
t, logs.String(), handlers.EventDBLeftMsgForTest,
|
|
||||||
"the events are gone, so the operator must not be told "+
|
|
||||||
"the event database file survived",
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleSourceDelete_LeftoverDatabaseFile proves that when the
|
// TestHandleTargetDelete_KeepsWriterWhenDatabaseTargetRemains
|
||||||
// event database file itself cannot be removed, the operator is told
|
// proves that deleting one of several database targets leaves
|
||||||
// it is still on disk, never that its events are gone.
|
// the still-needed archive writer alone: the surviving target
|
||||||
func TestHandleSourceDelete_LeftoverDatabaseFile(t *testing.T) {
|
// keeps archiving to the same file, so the writer must stay.
|
||||||
|
func TestHandleTargetDelete_KeepsWriterWhenDatabaseTargetRemains(
|
||||||
|
t *testing.T,
|
||||||
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
var (
|
var (
|
||||||
h *handlers.Handlers
|
h *handlers.Handlers
|
||||||
sess *session.Session
|
sess *session.Session
|
||||||
db *database.Database
|
db *database.Database
|
||||||
mgr *database.WebhookDBManager
|
ev *recordingEvictor
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &mgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
logs := new(bytes.Buffer)
|
|
||||||
h.SetLogForTest(slog.New(slog.NewTextHandler(logs, nil)))
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
|
|
||||||
// A non-empty directory in the database file's place, which
|
|
||||||
// os.Remove cannot remove whoever runs the test.
|
|
||||||
eventDBPath := mgr.DBPath(wh.ID)
|
|
||||||
require.NoError(t, os.MkdirAll(
|
|
||||||
filepath.Join(eventDBPath, "keep"), 0o700,
|
|
||||||
))
|
|
||||||
|
|
||||||
cookies := authenticatedCookies(
|
|
||||||
t, sess, deleteTestUserID, deleteTestUsername,
|
|
||||||
)
|
|
||||||
|
|
||||||
req := postRequest(
|
|
||||||
"/hook/"+wh.ID+"/delete",
|
|
||||||
cookies,
|
|
||||||
map[string]string{paramSourceID: wh.ID},
|
|
||||||
)
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
|
|
||||||
h.HandleSourceDelete().ServeHTTP(w, req)
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusInternalServerError, w.Code)
|
|
||||||
assert.Contains(
|
|
||||||
t, logs.String(), "event database file is still on disk",
|
|
||||||
)
|
|
||||||
assert.Contains(t, logs.String(), eventDBPath)
|
|
||||||
assert.NotContains(
|
|
||||||
t, logs.String(), handlers.SidecarLeftMsgForTest,
|
|
||||||
"the database file is still on disk, so the operator must "+
|
|
||||||
"not be told its events are gone",
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleTargetDelete_EvictsThatTarget proves that deleting a
|
|
||||||
// database target releases that target's archive writer and no
|
|
||||||
// other: the webhook's other database target keeps its own.
|
|
||||||
func TestHandleTargetDelete_EvictsThatTarget(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
ev *recordingArchives
|
|
||||||
)
|
)
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &ev)
|
app := newTestApp(t, &h, &sess, &db, &ev)
|
||||||
@@ -612,7 +515,7 @@ func TestHandleTargetDelete_EvictsThatTarget(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
req := postRequest(
|
req := postRequest(
|
||||||
"/hook/"+wh.ID+"/targets/"+doomed.ID+"/delete",
|
"/source/"+wh.ID+"/targets/"+doomed.ID+"/delete",
|
||||||
cookies,
|
cookies,
|
||||||
map[string]string{
|
map[string]string{
|
||||||
paramSourceID: wh.ID,
|
paramSourceID: wh.ID,
|
||||||
@@ -624,17 +527,17 @@ func TestHandleTargetDelete_EvictsThatTarget(t *testing.T) {
|
|||||||
h.HandleTargetDelete().ServeHTTP(w, req)
|
h.HandleTargetDelete().ServeHTTP(w, req)
|
||||||
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Equal(
|
assert.Empty(
|
||||||
t, []string{doomed.ID}, ev.EvictedTargets(),
|
t, ev.Evicted(),
|
||||||
"deleting a database target should evict its writer",
|
"a second database target still needs the writer",
|
||||||
)
|
)
|
||||||
assert.Empty(t, ev.Evicted(), "the webhook is not deleted")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleTargetDelete_IgnoresAnotherWebhooksTarget proves that
|
// TestHandleTargetDelete_KeepsWriterWhenOtherTypeDeleted proves
|
||||||
// a target id from the URL that is not a target of the webhook
|
// that deleting a target of an unrelated type leaves a
|
||||||
// deletes nothing and so evicts nothing.
|
// still-needed archive writer alone: the webhook's database
|
||||||
func TestHandleTargetDelete_IgnoresAnotherWebhooksTarget(
|
// target is untouched, so its writer must stay.
|
||||||
|
func TestHandleTargetDelete_KeepsWriterWhenOtherTypeDeleted(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
@@ -643,7 +546,7 @@ func TestHandleTargetDelete_IgnoresAnotherWebhooksTarget(
|
|||||||
h *handlers.Handlers
|
h *handlers.Handlers
|
||||||
sess *session.Session
|
sess *session.Session
|
||||||
db *database.Database
|
db *database.Database
|
||||||
ev *recordingArchives
|
ev *recordingEvictor
|
||||||
)
|
)
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &ev)
|
app := newTestApp(t, &h, &sess, &db, &ev)
|
||||||
@@ -652,20 +555,19 @@ func TestHandleTargetDelete_IgnoresAnotherWebhooksTarget(
|
|||||||
t.Cleanup(app.RequireStop)
|
t.Cleanup(app.RequireStop)
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
wh := seedWebhook(t, db)
|
||||||
elsewhere := seedTarget(
|
seedTarget(t, db, wh.ID, database.TargetTypeDatabase)
|
||||||
t, db, seedWebhook(t, db).ID, database.TargetTypeDatabase,
|
other := seedTarget(t, db, wh.ID, database.TargetTypeLog)
|
||||||
)
|
|
||||||
|
|
||||||
cookies := authenticatedCookies(
|
cookies := authenticatedCookies(
|
||||||
t, sess, deleteTestUserID, deleteTestUsername,
|
t, sess, deleteTestUserID, deleteTestUsername,
|
||||||
)
|
)
|
||||||
|
|
||||||
req := postRequest(
|
req := postRequest(
|
||||||
"/hook/"+wh.ID+"/targets/"+elsewhere.ID+"/delete",
|
"/source/"+wh.ID+"/targets/"+other.ID+"/delete",
|
||||||
cookies,
|
cookies,
|
||||||
map[string]string{
|
map[string]string{
|
||||||
paramSourceID: wh.ID,
|
paramSourceID: wh.ID,
|
||||||
paramTargetID: elsewhere.ID,
|
paramTargetID: other.ID,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
@@ -674,7 +576,7 @@ func TestHandleTargetDelete_IgnoresAnotherWebhooksTarget(
|
|||||||
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
assert.Empty(
|
assert.Empty(
|
||||||
t, ev.EvictedTargets(),
|
t, ev.Evicted(),
|
||||||
"another webhook's target must not be evicted",
|
"a surviving database target must keep its writer",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -81,7 +81,7 @@ func (f *baseURLFixture) entrypointURL(
|
|||||||
req := httptest.NewRequestWithContext(
|
req := httptest.NewRequestWithContext(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
http.MethodGet,
|
http.MethodGet,
|
||||||
"/hook/"+f.webhook,
|
"/source/"+f.webhook,
|
||||||
nil,
|
nil,
|
||||||
)
|
)
|
||||||
req.Host = host
|
req.Host = host
|
||||||
@@ -213,7 +213,7 @@ func TestSourceDetailBaseURL_ForwardedProtoSpellings(t *testing.T) {
|
|||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
tc.scheme+"://"+host+"/h/"+fixture.path,
|
tc.scheme+"://"+host+"/webhook/"+fixture.path,
|
||||||
fixture.entrypointURL(
|
fixture.entrypointURL(
|
||||||
t, host, forwardedProto(tc.header),
|
t, host, forwardedProto(tc.header),
|
||||||
),
|
),
|
||||||
@@ -244,7 +244,7 @@ func TestSourceDetailBaseURL_DirectTLSBeatsPlaintextHeader(
|
|||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"https://"+host+"/h/"+fixture.path,
|
"https://"+host+"/webhook/"+fixture.path,
|
||||||
got,
|
got,
|
||||||
"a connection this process terminated with TLS "+
|
"a connection this process terminated with TLS "+
|
||||||
"outranks a header claiming plaintext",
|
"outranks a header claiming plaintext",
|
||||||
@@ -272,7 +272,7 @@ func TestSourceDetailBaseURL_KeepsHostAuthority(t *testing.T) {
|
|||||||
|
|
||||||
assert.Equal(
|
assert.Equal(
|
||||||
t,
|
t,
|
||||||
"https://"+host+"/h/"+fixture.path,
|
"https://"+host+"/webhook/"+fixture.path,
|
||||||
fixture.entrypointURL(
|
fixture.entrypointURL(
|
||||||
t, host, forwardedProto("HTTPS"),
|
t, host, forwardedProto("HTTPS"),
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -62,27 +62,10 @@ func renderSourceDetailPage(
|
|||||||
) string {
|
) string {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
w := serveSourceDetailPage(t, h, sess, webhookID)
|
|
||||||
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
|
|
||||||
return w.Body.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// serveSourceDetailPage runs the real source detail handler for a
|
|
||||||
// webhook and returns its response, whatever its status.
|
|
||||||
func serveSourceDetailPage(
|
|
||||||
t *testing.T,
|
|
||||||
h *handlers.Handlers,
|
|
||||||
sess *session.Session,
|
|
||||||
webhookID string,
|
|
||||||
) *httptest.ResponseRecorder {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
req := httptest.NewRequestWithContext(
|
req := httptest.NewRequestWithContext(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
http.MethodGet,
|
http.MethodGet,
|
||||||
"/hook/"+webhookID,
|
"/source/"+webhookID,
|
||||||
nil,
|
nil,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -104,7 +87,9 @@ func serveSourceDetailPage(
|
|||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
h.HandleSourceDetail().ServeHTTP(w, req)
|
h.HandleSourceDetail().ServeHTTP(w, req)
|
||||||
|
|
||||||
return w
|
require.Equal(t, http.StatusOK, w.Code)
|
||||||
|
|
||||||
|
return w.Body.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleSourceDetail_MasksSlackWebhookURL is the
|
// TestHandleSourceDetail_MasksSlackWebhookURL is the
|
||||||
@@ -241,37 +226,3 @@ func TestHandleSourceDetail_RendersNamedTargetFields(
|
|||||||
assert.Contains(t, body, "(unavailable)")
|
assert.Contains(t, body, "(unavailable)")
|
||||||
assert.NotContains(t, body, "beak")
|
assert.NotContains(t, body, "beak")
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleSourceDetail_FitsWideAndNarrowWindows pins the webhook
|
|
||||||
// page's maximum width at 108rem (1728 px), half again the 72rem of
|
|
||||||
// max-w-6xl that the webhook list and the event log use, so an
|
|
||||||
// entrypoint URL fits on one line in a 1920-pixel window; and the
|
|
||||||
// wrapping of its title row, so the buttons beside the title do not
|
|
||||||
// push a phone-width window into scrolling sideways.
|
|
||||||
func TestHandleSourceDetail_FitsWideAndNarrowWindows(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := seedWebhook(t, db)
|
|
||||||
|
|
||||||
body := renderSourceDetailPage(t, h, sess, wh.ID)
|
|
||||||
|
|
||||||
assert.Contains(
|
|
||||||
t, body,
|
|
||||||
`<div class="mx-auto px-6 py-8" style="max-width: 108rem"`,
|
|
||||||
)
|
|
||||||
assert.Contains(
|
|
||||||
t, body,
|
|
||||||
`<div class="flex flex-wrap justify-between items-center gap-2 mt-2">`,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,467 +0,0 @@
|
|||||||
package handlers_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"regexp"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
"gorm.io/gorm/clause"
|
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
|
||||||
"sneak.berlin/go/webhooker/internal/logger"
|
|
||||||
"sneak.berlin/go/webhooker/internal/session"
|
|
||||||
)
|
|
||||||
|
|
||||||
// failedHighlight is how the list marks a number of failed deliveries
|
|
||||||
// that is not zero.
|
|
||||||
const failedHighlight = `class="font-medium text-red-600"`
|
|
||||||
|
|
||||||
// listWebhook adds a webhook with the given name, owned by the test
|
|
||||||
// user.
|
|
||||||
func listWebhook(
|
|
||||||
t *testing.T, db *database.Database, name string,
|
|
||||||
) *database.Webhook {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
wh := &database.Webhook{UserID: deleteTestUserID, Name: name}
|
|
||||||
require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error)
|
|
||||||
|
|
||||||
return wh
|
|
||||||
}
|
|
||||||
|
|
||||||
// addEntrypoints adds the given number of entrypoints, all active or
|
|
||||||
// all inactive, to a webhook and returns their paths.
|
|
||||||
func addEntrypoints(
|
|
||||||
t *testing.T, db *database.Database, webhookID string,
|
|
||||||
count int, active bool,
|
|
||||||
) []string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
paths := make([]string, count)
|
|
||||||
for i := range paths {
|
|
||||||
paths[i] = statsEntrypoint(t, db, webhookID, active)
|
|
||||||
}
|
|
||||||
|
|
||||||
return paths
|
|
||||||
}
|
|
||||||
|
|
||||||
// addTargets adds the given number of targets, all active or all
|
|
||||||
// inactive, to a webhook and returns them.
|
|
||||||
func addTargets(
|
|
||||||
t *testing.T, db *database.Database, webhookID string,
|
|
||||||
count int, active bool,
|
|
||||||
) []*database.Target {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
targets := make([]*database.Target, count)
|
|
||||||
for i := range targets {
|
|
||||||
targets[i] = seedTarget(t, db, webhookID, database.TargetTypeLog)
|
|
||||||
require.NoError(t, db.DB().Model(targets[i]).
|
|
||||||
Update("active", active).Error)
|
|
||||||
}
|
|
||||||
|
|
||||||
return targets
|
|
||||||
}
|
|
||||||
|
|
||||||
// renderWebhookList runs the real webhook list handler as the test user
|
|
||||||
// and returns the rendered page.
|
|
||||||
func renderWebhookList(
|
|
||||||
t *testing.T, h *handlers.Handlers, sess *session.Session,
|
|
||||||
) string {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
cookies := authenticatedCookies(
|
|
||||||
t, sess, deleteTestUserID, deleteTestUsername,
|
|
||||||
)
|
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
|
||||||
h.HandleSourceList().ServeHTTP(
|
|
||||||
w, getRequest(t, "/hooks", cookies, nil),
|
|
||||||
)
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
|
||||||
|
|
||||||
return w.Body.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// listCard returns one webhook's entry in a rendered webhook list, its
|
|
||||||
// markup as rendered and its text with the markup taken out and each
|
|
||||||
// run of space made one space.
|
|
||||||
func listCard(t *testing.T, page, webhookID string) (string, string) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
_, card, found := strings.Cut(page, `href="/hook/`+webhookID+`"`)
|
|
||||||
require.True(t, found, "the list has no entry for %s", webhookID)
|
|
||||||
|
|
||||||
card, _, _ = strings.Cut(card, "</a>")
|
|
||||||
text := regexp.MustCompile(`<[^>]*>`).ReplaceAllString(card, " ")
|
|
||||||
|
|
||||||
return card, strings.Join(strings.Fields(text), " ")
|
|
||||||
}
|
|
||||||
|
|
||||||
// receiveEvents posts the given number of events to an entrypoint
|
|
||||||
// through the real receiver, and returns the webhook's event database
|
|
||||||
// and its events, oldest first.
|
|
||||||
func receiveEvents(
|
|
||||||
t *testing.T,
|
|
||||||
h *handlers.Handlers,
|
|
||||||
dbMgr *database.WebhookDBManager,
|
|
||||||
webhookID, path string,
|
|
||||||
count int,
|
|
||||||
) (*gorm.DB, []database.Event) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
router := receiverRouter(h)
|
|
||||||
|
|
||||||
for range count {
|
|
||||||
require.Equal(t, http.StatusOK, postReceiver(t, router, path))
|
|
||||||
}
|
|
||||||
|
|
||||||
webhookDB, err := dbMgr.GetDB(webhookID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
events := listEvents(t, webhookDB)
|
|
||||||
require.Len(t, events, count)
|
|
||||||
|
|
||||||
return webhookDB, events
|
|
||||||
}
|
|
||||||
|
|
||||||
// seedFailingWebhook adds a webhook with six entrypoints, two of them
|
|
||||||
// inactive, and seven targets, five of them inactive. Four events reach
|
|
||||||
// its two active targets, arriving 31, 5, 4 and 3 hours ago, and its
|
|
||||||
// event totals row records the last one. Three deliveries failed in the
|
|
||||||
// last 24 hours, two to the first target and one to the second, one
|
|
||||||
// failed 30 hours ago, two were delivered, and two are still pending.
|
|
||||||
// It returns the webhook and when its last event arrived.
|
|
||||||
func seedFailingWebhook(
|
|
||||||
t *testing.T,
|
|
||||||
h *handlers.Handlers,
|
|
||||||
db *database.Database,
|
|
||||||
dbMgr *database.WebhookDBManager,
|
|
||||||
) (*database.Webhook, time.Time) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
wh := listWebhook(t, db, "failing")
|
|
||||||
paths := addEntrypoints(t, db, wh.ID, 4, true)
|
|
||||||
addEntrypoints(t, db, wh.ID, 2, false)
|
|
||||||
|
|
||||||
active := addTargets(t, db, wh.ID, 2, true)
|
|
||||||
first, second := active[0], active[1]
|
|
||||||
|
|
||||||
addTargets(t, db, wh.ID, 5, false)
|
|
||||||
|
|
||||||
webhookDB, events := receiveEvents(t, h, dbMgr, wh.ID, paths[0], 4)
|
|
||||||
now := time.Now()
|
|
||||||
lastEventAt := now.Add(-3 * time.Hour)
|
|
||||||
|
|
||||||
statsAge(t, webhookDB, events[0].ID, now.Add(-31*time.Hour))
|
|
||||||
statsAge(t, webhookDB, events[1].ID, now.Add(-5*time.Hour))
|
|
||||||
statsAge(t, webhookDB, events[2].ID, now.Add(-4*time.Hour))
|
|
||||||
statsAge(t, webhookDB, events[3].ID, lastEventAt)
|
|
||||||
require.NoError(t, database.AddEventTotals(webhookDB,
|
|
||||||
database.EventTotals{LastEventAt: &lastEventAt}))
|
|
||||||
|
|
||||||
statsFinish(t, webhookDB,
|
|
||||||
statsDelivery(t, webhookDB, events[0].ID, first.ID),
|
|
||||||
database.DeliveryStatusFailed, now.Add(-30*time.Hour))
|
|
||||||
statsFinish(t, webhookDB,
|
|
||||||
statsDelivery(t, webhookDB, events[0].ID, second.ID),
|
|
||||||
database.DeliveryStatusDelivered, now.Add(-30*time.Hour))
|
|
||||||
statsFinish(t, webhookDB,
|
|
||||||
statsDelivery(t, webhookDB, events[1].ID, first.ID),
|
|
||||||
database.DeliveryStatusFailed, now.Add(-time.Hour))
|
|
||||||
statsFinish(t, webhookDB,
|
|
||||||
statsDelivery(t, webhookDB, events[2].ID, first.ID),
|
|
||||||
database.DeliveryStatusFailed, now.Add(-time.Minute))
|
|
||||||
statsFinish(t, webhookDB,
|
|
||||||
statsDelivery(t, webhookDB, events[2].ID, second.ID),
|
|
||||||
database.DeliveryStatusFailed, now.Add(-time.Minute))
|
|
||||||
statsFinish(t, webhookDB,
|
|
||||||
statsDelivery(t, webhookDB, events[3].ID, second.ID),
|
|
||||||
database.DeliveryStatusDelivered, now.Add(-time.Minute))
|
|
||||||
|
|
||||||
return wh, lastEventAt
|
|
||||||
}
|
|
||||||
|
|
||||||
// seedHealthyWebhook adds a webhook with four entrypoints and two
|
|
||||||
// targets, all active, and three events, arriving 8, 7 and 6 hours ago
|
|
||||||
// and each delivered to both targets. Its event totals row records the
|
|
||||||
// last event. It returns the webhook and when its last event arrived.
|
|
||||||
func seedHealthyWebhook(
|
|
||||||
t *testing.T,
|
|
||||||
h *handlers.Handlers,
|
|
||||||
db *database.Database,
|
|
||||||
dbMgr *database.WebhookDBManager,
|
|
||||||
) (*database.Webhook, time.Time) {
|
|
||||||
t.Helper()
|
|
||||||
|
|
||||||
wh := listWebhook(t, db, "healthy")
|
|
||||||
paths := addEntrypoints(t, db, wh.ID, 4, true)
|
|
||||||
targets := addTargets(t, db, wh.ID, 2, true)
|
|
||||||
|
|
||||||
webhookDB, events := receiveEvents(t, h, dbMgr, wh.ID, paths[0], 3)
|
|
||||||
now := time.Now()
|
|
||||||
lastEventAt := now.Add(-6 * time.Hour)
|
|
||||||
|
|
||||||
statsAge(t, webhookDB, events[0].ID, now.Add(-8*time.Hour))
|
|
||||||
statsAge(t, webhookDB, events[1].ID, now.Add(-7*time.Hour))
|
|
||||||
statsAge(t, webhookDB, events[2].ID, lastEventAt)
|
|
||||||
require.NoError(t, database.AddEventTotals(webhookDB,
|
|
||||||
database.EventTotals{LastEventAt: &lastEventAt}))
|
|
||||||
|
|
||||||
for _, ev := range events {
|
|
||||||
for _, target := range targets {
|
|
||||||
statsFinish(t, webhookDB,
|
|
||||||
statsDelivery(t, webhookDB, ev.ID, target.ID),
|
|
||||||
database.DeliveryStatusDelivered, now)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return wh, lastEventAt
|
|
||||||
}
|
|
||||||
|
|
||||||
// lastEventText is how the list shows when the last event arrived.
|
|
||||||
func lastEventText(at time.Time) string {
|
|
||||||
return at.UTC().Format("2006-01-02 15:04:05 UTC")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSourceList_ShowsActivityOfEachWebhook checks the figures the list
|
|
||||||
// shows for a webhook with recent failures, a healthy one, a new one
|
|
||||||
// that has received no event, and one without an event database.
|
|
||||||
func TestSourceList_ShowsActivityOfEachWebhook(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
failing, failingLastEvent := seedFailingWebhook(t, h, db, dbMgr)
|
|
||||||
healthy, healthyLastEvent := seedHealthyWebhook(t, h, db, dbMgr)
|
|
||||||
|
|
||||||
// Creating a webhook creates its event database.
|
|
||||||
fresh := listWebhook(t, db, "fresh")
|
|
||||||
require.NoError(t, dbMgr.CreateDB(fresh.ID))
|
|
||||||
addEntrypoints(t, db, fresh.ID, 2, true)
|
|
||||||
addTargets(t, db, fresh.ID, 3, true)
|
|
||||||
|
|
||||||
quiet := listWebhook(t, db, "quiet")
|
|
||||||
addEntrypoints(t, db, quiet.ID, 2, true)
|
|
||||||
addTargets(t, db, quiet.ID, 3, true)
|
|
||||||
|
|
||||||
page := renderWebhookList(t, h, sess)
|
|
||||||
|
|
||||||
card, text := listCard(t, page, failing.ID)
|
|
||||||
assert.Contains(t, text, "6 entrypoints, 2 inactive")
|
|
||||||
assert.Contains(t, text, "7 targets, 5 inactive")
|
|
||||||
assert.Contains(t, text, "4 events within retention")
|
|
||||||
assert.Contains(t, text, "Last event "+lastEventText(failingLastEvent))
|
|
||||||
assert.Contains(t, card,
|
|
||||||
failedHighlight+">3 failed deliveries in the last 24 hours<")
|
|
||||||
|
|
||||||
card, text = listCard(t, page, healthy.ID)
|
|
||||||
assert.Contains(t, text, "4 entrypoints")
|
|
||||||
assert.Contains(t, text, "2 targets")
|
|
||||||
assert.Contains(t, text, "3 events within retention")
|
|
||||||
assert.Contains(t, text, "Last event "+lastEventText(healthyLastEvent))
|
|
||||||
assert.Contains(t, text, "0 failed deliveries in the last 24 hours")
|
|
||||||
assert.NotContains(t, text, "inactive")
|
|
||||||
assert.NotContains(t, card, failedHighlight)
|
|
||||||
|
|
||||||
card, text = listCard(t, page, fresh.ID)
|
|
||||||
assert.Contains(t, text, "2 entrypoints")
|
|
||||||
assert.Contains(t, text, "3 targets")
|
|
||||||
assert.Contains(t, text, "0 events within retention")
|
|
||||||
assert.Contains(t, text, "No events yet")
|
|
||||||
assert.Contains(t, text, "0 failed deliveries in the last 24 hours")
|
|
||||||
assert.NotContains(t, card, failedHighlight)
|
|
||||||
|
|
||||||
card, text = listCard(t, page, quiet.ID)
|
|
||||||
assert.Contains(t, text, "2 entrypoints")
|
|
||||||
assert.Contains(t, text, "3 targets")
|
|
||||||
assert.Contains(t, text, "0 events within retention")
|
|
||||||
assert.Contains(t, text, "No events yet")
|
|
||||||
assert.Contains(t, text, "0 failed deliveries in the last 24 hours")
|
|
||||||
assert.NotContains(t, card, failedHighlight)
|
|
||||||
assert.False(t, dbMgr.DBExists(quiet.ID),
|
|
||||||
"showing the list must not create an event database")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSourceList_CountsOnlyEventsWithinRetention checks that once
|
|
||||||
// retention has removed one of a webhook's three events, the list
|
|
||||||
// counts the two still stored.
|
|
||||||
func TestSourceList_CountsOnlyEventsWithinRetention(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
log *logger.Logger
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr, &log)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := &database.Webhook{
|
|
||||||
UserID: deleteTestUserID, Name: "pruned", RetentionDays: 14,
|
|
||||||
}
|
|
||||||
require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error)
|
|
||||||
|
|
||||||
paths := addEntrypoints(t, db, wh.ID, 3, true)
|
|
||||||
addTargets(t, db, wh.ID, 4, true)
|
|
||||||
|
|
||||||
webhookDB, events := receiveEvents(t, h, dbMgr, wh.ID, paths[0], 3)
|
|
||||||
|
|
||||||
statsAge(t, webhookDB, events[0].ID, time.Now().Add(-15*24*time.Hour))
|
|
||||||
statsPrune(t, db, dbMgr, log, webhookDB)
|
|
||||||
require.Len(t, listEvents(t, webhookDB), 2)
|
|
||||||
|
|
||||||
_, text := listCard(t, renderWebhookList(t, h, sess), wh.ID)
|
|
||||||
assert.Contains(t, text, "3 entrypoints")
|
|
||||||
assert.Contains(t, text, "4 targets")
|
|
||||||
assert.Contains(t, text, "2 events within retention")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSourceList_LastEventSurvivesPruningEveryEvent checks that once
|
|
||||||
// retention has removed every event of a webhook, the list still shows
|
|
||||||
// when the last one arrived rather than "No events yet".
|
|
||||||
func TestSourceList_LastEventSurvivesPruningEveryEvent(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
log *logger.Logger
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr, &log)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := &database.Webhook{
|
|
||||||
UserID: deleteTestUserID, Name: "emptied", RetentionDays: 1,
|
|
||||||
}
|
|
||||||
require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error)
|
|
||||||
|
|
||||||
paths := addEntrypoints(t, db, wh.ID, 2, true)
|
|
||||||
addTargets(t, db, wh.ID, 3, true)
|
|
||||||
|
|
||||||
webhookDB, events := receiveEvents(t, h, dbMgr, wh.ID, paths[0], 1)
|
|
||||||
lastEventAt := time.Now().Add(-50 * time.Hour)
|
|
||||||
|
|
||||||
statsAge(t, webhookDB, events[0].ID, lastEventAt)
|
|
||||||
require.NoError(t, database.AddEventTotals(webhookDB,
|
|
||||||
database.EventTotals{LastEventAt: &lastEventAt}))
|
|
||||||
statsPrune(t, db, dbMgr, log, webhookDB)
|
|
||||||
require.Empty(t, listEvents(t, webhookDB))
|
|
||||||
|
|
||||||
_, text := listCard(t, renderWebhookList(t, h, sess), wh.ID)
|
|
||||||
assert.Contains(t, text, "0 events within retention")
|
|
||||||
assert.Contains(t, text, "Last event "+lastEventText(lastEventAt))
|
|
||||||
assert.NotContains(t, text, "No events yet")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSourceList_CountsOfOneInSingular checks that a webhook with one
|
|
||||||
// entrypoint, one target, one event within retention and one failed
|
|
||||||
// delivery in the last 24 hours has each written in the singular.
|
|
||||||
func TestSourceList_CountsOfOneInSingular(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
wh := listWebhook(t, db, "single")
|
|
||||||
paths := addEntrypoints(t, db, wh.ID, 1, true)
|
|
||||||
targets := addTargets(t, db, wh.ID, 1, true)
|
|
||||||
|
|
||||||
webhookDB, events := receiveEvents(t, h, dbMgr, wh.ID, paths[0], 1)
|
|
||||||
now := time.Now()
|
|
||||||
lastEventAt := now.Add(-9 * time.Hour)
|
|
||||||
|
|
||||||
statsAge(t, webhookDB, events[0].ID, lastEventAt)
|
|
||||||
require.NoError(t, database.AddEventTotals(webhookDB,
|
|
||||||
database.EventTotals{LastEventAt: &lastEventAt}))
|
|
||||||
statsFinish(t, webhookDB,
|
|
||||||
statsDelivery(t, webhookDB, events[0].ID, targets[0].ID),
|
|
||||||
database.DeliveryStatusFailed, now.Add(-time.Hour))
|
|
||||||
|
|
||||||
card, text := listCard(t, renderWebhookList(t, h, sess), wh.ID)
|
|
||||||
assert.Contains(t, card, ">1 entrypoint<")
|
|
||||||
assert.Contains(t, card, ">1 target<")
|
|
||||||
assert.Contains(t, card, ">1 event within retention<")
|
|
||||||
assert.Contains(t, text, "Last event "+lastEventText(lastEventAt))
|
|
||||||
assert.Contains(t, card,
|
|
||||||
failedHighlight+">1 failed delivery in the last 24 hours<")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSourceList_UnreadableEventDatabase checks that a webhook whose
|
|
||||||
// event database cannot be read says so in its entry instead of
|
|
||||||
// showing zeros, and that the rest of the list is still shown.
|
|
||||||
func TestSourceList_UnreadableEventDatabase(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
var (
|
|
||||||
h *handlers.Handlers
|
|
||||||
sess *session.Session
|
|
||||||
db *database.Database
|
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
)
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
|
||||||
app.RequireStart()
|
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
|
||||||
|
|
||||||
broken := listWebhook(t, db, "broken")
|
|
||||||
addEntrypoints(t, db, broken.ID, 2, true)
|
|
||||||
addTargets(t, db, broken.ID, 3, true)
|
|
||||||
|
|
||||||
brokenDB, err := dbMgr.GetDB(broken.ID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t,
|
|
||||||
brokenDB.Migrator().DropTable(&database.EventTotals{}))
|
|
||||||
|
|
||||||
quiet := listWebhook(t, db, "quiet")
|
|
||||||
addEntrypoints(t, db, quiet.ID, 2, true)
|
|
||||||
addTargets(t, db, quiet.ID, 3, true)
|
|
||||||
|
|
||||||
page := renderWebhookList(t, h, sess)
|
|
||||||
|
|
||||||
_, text := listCard(t, page, broken.ID)
|
|
||||||
assert.Contains(t, text, "2 entrypoints")
|
|
||||||
assert.Contains(t, text, "3 targets")
|
|
||||||
assert.Contains(t, text, "The event figures could not be read.")
|
|
||||||
assert.NotContains(t, text, "events")
|
|
||||||
assert.NotContains(t, text, "failed")
|
|
||||||
|
|
||||||
_, text = listCard(t, page, quiet.ID)
|
|
||||||
assert.Contains(t, text, "No events yet")
|
|
||||||
}
|
|
||||||
@@ -28,7 +28,7 @@ func deleteTargetThroughHandler(
|
|||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
req := postRequest(
|
req := postRequest(
|
||||||
"/hook/"+webhookID+"/targets/"+targetID+"/delete",
|
"/source/"+webhookID+"/targets/"+targetID+"/delete",
|
||||||
authenticatedCookies(
|
authenticatedCookies(
|
||||||
t, sess, deleteTestUserID, deleteTestUsername,
|
t, sess, deleteTestUserID, deleteTestUsername,
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -84,7 +84,7 @@ func renderSourceLogsPageWithQuery(
|
|||||||
req := httptest.NewRequestWithContext(
|
req := httptest.NewRequestWithContext(
|
||||||
context.Background(),
|
context.Background(),
|
||||||
http.MethodGet,
|
http.MethodGet,
|
||||||
"/hook/"+webhookID+"/events"+query,
|
"/source/"+webhookID+"/logs"+query,
|
||||||
nil,
|
nil,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -2,20 +2,16 @@ package handlers_test
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/go-chi/chi"
|
"github.com/go-chi/chi"
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"gorm.io/gorm"
|
|
||||||
"gorm.io/gorm/clause"
|
"gorm.io/gorm/clause"
|
||||||
"sneak.berlin/go/webhooker/internal/database"
|
"sneak.berlin/go/webhooker/internal/database"
|
||||||
"sneak.berlin/go/webhooker/internal/handlers"
|
"sneak.berlin/go/webhooker/internal/handlers"
|
||||||
@@ -109,7 +105,7 @@ func submitCreate(
|
|||||||
form.Set("retention_days", *retention)
|
form.Set("retention_days", *retention)
|
||||||
}
|
}
|
||||||
|
|
||||||
req := formRequest("/hooks/new", cookies, form, nil)
|
req := formRequest("/sources/new", cookies, form, nil)
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
h.HandleSourceCreateSubmit().ServeHTTP(w, req)
|
h.HandleSourceCreateSubmit().ServeHTTP(w, req)
|
||||||
@@ -191,8 +187,6 @@ func storedRetentionDays(
|
|||||||
type sourceTestEnv struct {
|
type sourceTestEnv struct {
|
||||||
handlers *handlers.Handlers
|
handlers *handlers.Handlers
|
||||||
db *database.Database
|
db *database.Database
|
||||||
dbMgr *database.WebhookDBManager
|
|
||||||
archives *recordingArchives
|
|
||||||
cookies []*http.Cookie
|
cookies []*http.Cookie
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -205,11 +199,7 @@ func setupSourceTest(t *testing.T) *sourceTestEnv {
|
|||||||
|
|
||||||
var db *database.Database
|
var db *database.Database
|
||||||
|
|
||||||
var dbMgr *database.WebhookDBManager
|
app := newTestApp(t, &h, &sess, &db)
|
||||||
|
|
||||||
var archives *recordingArchives
|
|
||||||
|
|
||||||
app := newTestApp(t, &h, &sess, &db, &dbMgr, &archives)
|
|
||||||
app.RequireStart()
|
app.RequireStart()
|
||||||
|
|
||||||
t.Cleanup(app.RequireStop)
|
t.Cleanup(app.RequireStop)
|
||||||
@@ -217,8 +207,6 @@ func setupSourceTest(t *testing.T) *sourceTestEnv {
|
|||||||
return &sourceTestEnv{
|
return &sourceTestEnv{
|
||||||
handlers: h,
|
handlers: h,
|
||||||
db: db,
|
db: db,
|
||||||
dbMgr: dbMgr,
|
|
||||||
archives: archives,
|
|
||||||
cookies: authenticatedCookies(
|
cookies: authenticatedCookies(
|
||||||
t, sess, sourceTestUserID, "sourceuser",
|
t, sess, sourceTestUserID, "sourceuser",
|
||||||
),
|
),
|
||||||
@@ -277,7 +265,7 @@ func TestHandleSourceCreate_PrefillsDefaultFromConstant(t *testing.T) {
|
|||||||
|
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
env.handlers.HandleSourceCreate().ServeHTTP(
|
env.handlers.HandleSourceCreate().ServeHTTP(
|
||||||
w, getRequest(t, "/hooks/new", env.cookies, nil),
|
w, getRequest(t, "/sources/new", env.cookies, nil),
|
||||||
)
|
)
|
||||||
|
|
||||||
require.Equal(t, http.StatusOK, w.Code)
|
require.Equal(t, http.StatusOK, w.Code)
|
||||||
@@ -372,42 +360,31 @@ func TestHandleSourceCreateSubmit_OverflowingRetentionIsRejected(
|
|||||||
// boundary between "too large to represent" and "retain forever": the
|
// boundary between "too large to represent" and "retain forever": the
|
||||||
// sentinel is above MaxFiniteRetentionDays, but it is the value the
|
// sentinel is above MaxFiniteRetentionDays, but it is the value the
|
||||||
// edit form pre-fills, so it must be accepted rather than rejected as
|
// edit form pre-fills, so it must be accepted rather than rejected as
|
||||||
// out of range. A value above the sentinel is stored as the sentinel.
|
// out of range.
|
||||||
func TestHandleSourceCreateSubmit_SentinelIsAcceptedAsForever(
|
func TestHandleSourceCreateSubmit_SentinelIsAcceptedAsForever(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
|
||||||
for _, days := range []int{
|
env := setupSourceTest(t)
|
||||||
|
sentinel := strconv.Itoa(database.RetentionForeverDays)
|
||||||
|
|
||||||
|
w := submitCreate(t, env.handlers, env.cookies, "forever", &sentinel)
|
||||||
|
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||||
|
|
||||||
|
wh := onlyWebhook(t, env.db)
|
||||||
|
assert.Equal(
|
||||||
|
t,
|
||||||
database.RetentionForeverDays,
|
database.RetentionForeverDays,
|
||||||
database.RetentionForeverDays + 1,
|
storedRetentionDays(t, env.db, wh.ID),
|
||||||
} {
|
)
|
||||||
raw := strconv.Itoa(days)
|
|
||||||
|
|
||||||
t.Run(raw, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
|
|
||||||
w := submitCreate(t, env.handlers, env.cookies, "forever", &raw)
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
|
||||||
|
|
||||||
wh := onlyWebhook(t, env.db)
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
database.RetentionForeverDays,
|
|
||||||
storedRetentionDays(t, env.db, wh.ID),
|
|
||||||
)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestHandleSourceCreateSubmit_RejectedFormKeepsUserInput checks that a
|
// TestHandleSourceCreateSubmit_RejectedFormKeepsUserInput checks that a
|
||||||
// validation failure hands the user's typing back, matching what the
|
// validation failure hands the user's typing back, matching what the
|
||||||
// edit form already does. Losing a long description to a mistyped
|
// edit form already does. Losing a long description to a mistyped
|
||||||
// retention value is the kind of thing that makes people give up on a
|
// retention value is the kind of thing that makes people give up on a
|
||||||
// form. Both values carry HTML-special characters, which must come
|
// form.
|
||||||
// back escaped rather than as markup.
|
|
||||||
func TestHandleSourceCreateSubmit_RejectedFormKeepsUserInput(
|
func TestHandleSourceCreateSubmit_RejectedFormKeepsUserInput(
|
||||||
t *testing.T,
|
t *testing.T,
|
||||||
) {
|
) {
|
||||||
@@ -416,8 +393,8 @@ func TestHandleSourceCreateSubmit_RejectedFormKeepsUserInput(
|
|||||||
env := setupSourceTest(t)
|
env := setupSourceTest(t)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
name = `kept"><b>name`
|
name = "kept-name"
|
||||||
description = `a </textarea> worth not losing`
|
description = "a description worth not losing"
|
||||||
)
|
)
|
||||||
|
|
||||||
form := url.Values{}
|
form := url.Values{}
|
||||||
@@ -425,7 +402,7 @@ func TestHandleSourceCreateSubmit_RejectedFormKeepsUserInput(
|
|||||||
form.Set("description", description)
|
form.Set("description", description)
|
||||||
form.Set("retention_days", "nonsense")
|
form.Set("retention_days", "nonsense")
|
||||||
|
|
||||||
req := formRequest("/hooks/new", env.cookies, form, nil)
|
req := formRequest("/sources/new", env.cookies, form, nil)
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
env.handlers.HandleSourceCreateSubmit().ServeHTTP(w, req)
|
env.handlers.HandleSourceCreateSubmit().ServeHTTP(w, req)
|
||||||
@@ -434,10 +411,8 @@ func TestHandleSourceCreateSubmit_RejectedFormKeepsUserInput(
|
|||||||
|
|
||||||
body := w.Body.String()
|
body := w.Body.String()
|
||||||
|
|
||||||
assert.Contains(t, body, `value="kept"><b>name"`)
|
assert.Contains(t, body, `value="`+name+`"`)
|
||||||
assert.Contains(t, body, `a </textarea> worth not losing`)
|
assert.Contains(t, body, description)
|
||||||
assert.NotContains(t, body, name)
|
|
||||||
assert.NotContains(t, body, description)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// submitEdit posts the webhook edit form for the given webhook.
|
// submitEdit posts the webhook edit form for the given webhook.
|
||||||
@@ -455,7 +430,7 @@ func submitEdit(
|
|||||||
form.Set("retention_days", retention)
|
form.Set("retention_days", retention)
|
||||||
|
|
||||||
req := formRequest(
|
req := formRequest(
|
||||||
"/hook/"+wh.ID+"/edit",
|
"/source/"+wh.ID+"/edit",
|
||||||
env.cookies,
|
env.cookies,
|
||||||
form,
|
form,
|
||||||
map[string]string{sourceIDParam: wh.ID},
|
map[string]string{sourceIDParam: wh.ID},
|
||||||
@@ -523,301 +498,6 @@ func TestHandleSourceEditSubmit_EmptyRetentionLeavesValueUnchanged(
|
|||||||
assert.Equal(t, 7, storedRetentionDays(t, env.db, wh.ID))
|
assert.Equal(t, 7, storedRetentionDays(t, env.db, wh.ID))
|
||||||
}
|
}
|
||||||
|
|
||||||
// renamedWebhookName is the name the rename tests give a webhook.
|
|
||||||
const renamedWebhookName = "Renamed"
|
|
||||||
|
|
||||||
// TestHandleSourceEditSubmit_RenamesArchives proves that a save
|
|
||||||
// that keeps the webhook's name renames nothing, and that renaming a
|
|
||||||
// webhook renames the archive of each of its database targets and
|
|
||||||
// asks nothing of its other targets.
|
|
||||||
func TestHandleSourceEditSubmit_RenamesArchives(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
wh := seedWebhookWithRetention(t, env.db, 7)
|
|
||||||
first := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
|
||||||
second := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
|
||||||
seedTarget(t, env.db, wh.ID, database.TargetTypeLog)
|
|
||||||
|
|
||||||
w := submitEdit(t, env, wh, "")
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
|
||||||
assert.Empty(t, env.archives.Renames())
|
|
||||||
|
|
||||||
wh.Name = renamedWebhookName
|
|
||||||
|
|
||||||
w = submitEdit(t, env, wh, "")
|
|
||||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
|
||||||
|
|
||||||
assert.ElementsMatch(
|
|
||||||
t,
|
|
||||||
[]archiveRename{
|
|
||||||
{first.ID, renamedWebhookName, first.Name},
|
|
||||||
{second.ID, renamedWebhookName, second.Name},
|
|
||||||
},
|
|
||||||
env.archives.Renames(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceEditSubmit_FailedRenameKeepsTheName proves that a
|
|
||||||
// webhook whose archive cannot be renamed keeps its stored name, so
|
|
||||||
// the name on disk and the name in the UI do not part, and that the
|
|
||||||
// handler puts back what it may already have moved.
|
|
||||||
func TestHandleSourceEditSubmit_FailedRenameKeepsTheName(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
wh := seedWebhookWithRetention(t, env.db, 7)
|
|
||||||
tgt := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
|
||||||
|
|
||||||
env.archives.FailRenames(tgt.ID, errInjectedRename)
|
|
||||||
|
|
||||||
oldName := wh.Name
|
|
||||||
wh.Name = renamedWebhookName
|
|
||||||
|
|
||||||
w := submitEdit(t, env, wh, "")
|
|
||||||
require.Equal(t, http.StatusInternalServerError, w.Code)
|
|
||||||
|
|
||||||
var stored database.Webhook
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t, env.db.DB().First(&stored, "id = ?", wh.ID).Error,
|
|
||||||
)
|
|
||||||
assert.Equal(t, oldName, stored.Name)
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
[]archiveRename{
|
|
||||||
{tgt.ID, renamedWebhookName, tgt.Name},
|
|
||||||
{tgt.ID, oldName, tgt.Name},
|
|
||||||
},
|
|
||||||
env.archives.Renames(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceEditSubmit_FailedSaveRenamesBack proves that when
|
|
||||||
// the archive is renamed but the new name cannot be saved, the
|
|
||||||
// archive is renamed back to the stored name and the stored name
|
|
||||||
// stays.
|
|
||||||
func TestHandleSourceEditSubmit_FailedSaveRenamesBack(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
wh := seedWebhookWithRetention(t, env.db, 7)
|
|
||||||
tgt := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
|
||||||
|
|
||||||
failSaveOnTable(t, env.db, "webhooks")
|
|
||||||
|
|
||||||
oldName := wh.Name
|
|
||||||
wh.Name = renamedWebhookName
|
|
||||||
|
|
||||||
w := submitEdit(t, env, wh, "")
|
|
||||||
require.Equal(t, http.StatusInternalServerError, w.Code)
|
|
||||||
|
|
||||||
var stored database.Webhook
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t, env.db.DB().First(&stored, "id = ?", wh.ID).Error,
|
|
||||||
)
|
|
||||||
assert.Equal(t, oldName, stored.Name)
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
[]archiveRename{
|
|
||||||
{tgt.ID, renamedWebhookName, tgt.Name},
|
|
||||||
{tgt.ID, oldName, tgt.Name},
|
|
||||||
},
|
|
||||||
env.archives.Renames(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// errInjectedRead is the failure a test makes reads of the main
|
|
||||||
// database report.
|
|
||||||
var errInjectedRead = errors.New("injected read failure")
|
|
||||||
|
|
||||||
// TestHandleSourceEditSubmit_FailedSaveRenamesBackWithoutReading
|
|
||||||
// proves that when the save fails and every later read of the main
|
|
||||||
// database fails too, each archive the rename moved is still renamed
|
|
||||||
// back: the move back needs no second read of the webhook's targets.
|
|
||||||
func TestHandleSourceEditSubmit_FailedSaveRenamesBackWithoutReading(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
wh := seedWebhookWithRetention(t, env.db, 7)
|
|
||||||
first := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
|
||||||
second := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
|
||||||
|
|
||||||
var saveFailed atomic.Bool
|
|
||||||
|
|
||||||
require.NoError(t, env.db.DB().Callback().Update().
|
|
||||||
Before("gorm:update").
|
|
||||||
Register("test:fail_save", func(tx *gorm.DB) {
|
|
||||||
saveFailed.Store(true)
|
|
||||||
|
|
||||||
_ = tx.AddError(errInjectedSave)
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
require.NoError(t, env.db.DB().Callback().Query().
|
|
||||||
Before("gorm:query").
|
|
||||||
Register("test:fail_reads_after_save", func(tx *gorm.DB) {
|
|
||||||
if saveFailed.Load() {
|
|
||||||
_ = tx.AddError(errInjectedRead)
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
|
|
||||||
oldName := wh.Name
|
|
||||||
wh.Name = renamedWebhookName
|
|
||||||
|
|
||||||
w := submitEdit(t, env, wh, "")
|
|
||||||
require.Equal(t, http.StatusInternalServerError, w.Code)
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
[]archiveRename{
|
|
||||||
{first.ID, renamedWebhookName, first.Name},
|
|
||||||
{second.ID, renamedWebhookName, second.Name},
|
|
||||||
{first.ID, oldName, first.Name},
|
|
||||||
{second.ID, oldName, second.Name},
|
|
||||||
},
|
|
||||||
env.archives.Renames(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceEditSubmit_EditsDoNotInterleave proves that a second
|
|
||||||
// webhook edit submitted while the first is inside its archive rename
|
|
||||||
// does not run until the first is saved, so afterwards the stored
|
|
||||||
// names are the ones the archive was last renamed to. The stand-in's
|
|
||||||
// last rename is the name the file has on disk.
|
|
||||||
func TestHandleSourceEditSubmit_EditsDoNotInterleave(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
wh := seedWebhookWithRetention(t, env.db, 7)
|
|
||||||
tgt := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
|
||||||
|
|
||||||
entered, release := env.archives.BlockNextRename()
|
|
||||||
|
|
||||||
firstEdit, secondEdit := wh, wh
|
|
||||||
firstEdit.Name = "First"
|
|
||||||
secondEdit.Name = "Second"
|
|
||||||
|
|
||||||
firstCode := make(chan int, 1)
|
|
||||||
|
|
||||||
go func() { firstCode <- submitEdit(t, env, firstEdit, "").Code }()
|
|
||||||
|
|
||||||
<-entered
|
|
||||||
|
|
||||||
secondCode := make(chan int, 1)
|
|
||||||
|
|
||||||
go func() { secondCode <- submitEdit(t, env, secondEdit, "").Code }()
|
|
||||||
|
|
||||||
// Were the edits not ordered, the second would run to its end in
|
|
||||||
// this time, while the first is still inside its rename.
|
|
||||||
time.Sleep(200 * time.Millisecond)
|
|
||||||
release()
|
|
||||||
|
|
||||||
assert.Equal(t, http.StatusSeeOther, <-firstCode)
|
|
||||||
assert.Equal(t, http.StatusSeeOther, <-secondCode)
|
|
||||||
|
|
||||||
var (
|
|
||||||
storedWebhook database.Webhook
|
|
||||||
storedTarget database.Target
|
|
||||||
)
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t, env.db.DB().First(&storedWebhook, "id = ?", wh.ID).Error,
|
|
||||||
)
|
|
||||||
require.NoError(
|
|
||||||
t, env.db.DB().First(&storedTarget, "id = ?", tgt.ID).Error,
|
|
||||||
)
|
|
||||||
|
|
||||||
renames := env.archives.Renames()
|
|
||||||
require.NotEmpty(t, renames)
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
archiveRename{tgt.ID, storedWebhook.Name, storedTarget.Name},
|
|
||||||
renames[len(renames)-1],
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceEditSubmit_FailedRenameRenamesTheOthersBack proves
|
|
||||||
// that when a webhook has three database targets and only the middle
|
|
||||||
// one's archive cannot be renamed, the stored name stays and both
|
|
||||||
// others are renamed back, the last one included: the move back does
|
|
||||||
// not stop at the target it cannot rename. The handler reaches the
|
|
||||||
// targets in the order they were created, which the exact sequence
|
|
||||||
// below pins, so the refused target always comes before the last.
|
|
||||||
func TestHandleSourceEditSubmit_FailedRenameRenamesTheOthersBack(
|
|
||||||
t *testing.T,
|
|
||||||
) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
wh := seedWebhookWithRetention(t, env.db, 7)
|
|
||||||
first := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
|
||||||
middle := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
|
||||||
last := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
|
||||||
|
|
||||||
env.archives.FailRenames(middle.ID, errNameTaken)
|
|
||||||
|
|
||||||
oldName := wh.Name
|
|
||||||
wh.Name = renamedWebhookName
|
|
||||||
|
|
||||||
w := submitEdit(t, env, wh, "")
|
|
||||||
require.Equal(t, http.StatusConflict, w.Code)
|
|
||||||
|
|
||||||
var stored database.Webhook
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t, env.db.DB().First(&stored, "id = ?", wh.ID).Error,
|
|
||||||
)
|
|
||||||
assert.Equal(t, oldName, stored.Name)
|
|
||||||
|
|
||||||
assert.Equal(
|
|
||||||
t,
|
|
||||||
[]archiveRename{
|
|
||||||
{first.ID, renamedWebhookName, first.Name},
|
|
||||||
{middle.ID, renamedWebhookName, middle.Name},
|
|
||||||
{last.ID, renamedWebhookName, last.Name},
|
|
||||||
{first.ID, oldName, first.Name},
|
|
||||||
{middle.ID, oldName, middle.Name},
|
|
||||||
{last.ID, oldName, last.Name},
|
|
||||||
},
|
|
||||||
env.archives.Renames(),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestHandleSourceEditSubmit_ArchiveNameTaken proves that when a file
|
|
||||||
// already has an archive's new name, the edit is refused with an
|
|
||||||
// error naming that file, and the webhook keeps its stored name.
|
|
||||||
func TestHandleSourceEditSubmit_ArchiveNameTaken(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
|
|
||||||
env := setupSourceTest(t)
|
|
||||||
wh := seedWebhookWithRetention(t, env.db, 7)
|
|
||||||
tgt := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
|
||||||
|
|
||||||
env.archives.FailRenames(tgt.ID, errNameTaken)
|
|
||||||
|
|
||||||
oldName := wh.Name
|
|
||||||
wh.Name = renamedWebhookName
|
|
||||||
|
|
||||||
w := submitEdit(t, env, wh, "")
|
|
||||||
require.Equal(t, http.StatusConflict, w.Code)
|
|
||||||
assert.Contains(t, w.Body.String(), "archive-taken.db")
|
|
||||||
|
|
||||||
var stored database.Webhook
|
|
||||||
|
|
||||||
require.NoError(
|
|
||||||
t, env.db.DB().First(&stored, "id = ?", wh.ID).Error,
|
|
||||||
)
|
|
||||||
assert.Equal(t, oldName, stored.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestSourceEditForm_ForeverWebhookRoundTrips walks the exact path that
|
// TestSourceEditForm_ForeverWebhookRoundTrips walks the exact path that
|
||||||
// the removed max="365" cap used to break: render the edit form for a
|
// the removed max="365" cap used to break: render the edit form for a
|
||||||
// retain-forever webhook, confirm the pre-filled sentinel is not capped
|
// retain-forever webhook, confirm the pre-filled sentinel is not capped
|
||||||
@@ -832,7 +512,7 @@ func TestSourceEditForm_ForeverWebhookRoundTrips(t *testing.T) {
|
|||||||
)
|
)
|
||||||
|
|
||||||
req := getRequest(
|
req := getRequest(
|
||||||
t, "/hook/"+wh.ID+"/edit", env.cookies,
|
t, "/source/"+wh.ID+"/edit", env.cookies,
|
||||||
map[string]string{sourceIDParam: wh.ID},
|
map[string]string{sourceIDParam: wh.ID},
|
||||||
)
|
)
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
@@ -887,7 +567,7 @@ func TestSourceListAndDetail_ShowForeverNotTheSentinelNumber(
|
|||||||
|
|
||||||
listW := httptest.NewRecorder()
|
listW := httptest.NewRecorder()
|
||||||
env.handlers.HandleSourceList().ServeHTTP(
|
env.handlers.HandleSourceList().ServeHTTP(
|
||||||
listW, getRequest(t, "/hooks", env.cookies, nil),
|
listW, getRequest(t, "/sources", env.cookies, nil),
|
||||||
)
|
)
|
||||||
|
|
||||||
require.Equal(t, http.StatusOK, listW.Code)
|
require.Equal(t, http.StatusOK, listW.Code)
|
||||||
@@ -898,7 +578,7 @@ func TestSourceListAndDetail_ShowForeverNotTheSentinelNumber(
|
|||||||
env.handlers.HandleSourceDetail().ServeHTTP(
|
env.handlers.HandleSourceDetail().ServeHTTP(
|
||||||
detailW,
|
detailW,
|
||||||
getRequest(
|
getRequest(
|
||||||
t, "/hook/"+wh.ID, env.cookies,
|
t, "/source/"+wh.ID, env.cookies,
|
||||||
map[string]string{sourceIDParam: wh.ID},
|
map[string]string{sourceIDParam: wh.ID},
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -76,11 +76,11 @@ func postTargetCreate(
|
|||||||
router := chi.NewRouter()
|
router := chi.NewRouter()
|
||||||
router.Use(mw.Logging())
|
router.Use(mw.Logging())
|
||||||
router.Post(
|
router.Post(
|
||||||
"/hook/{sourceID}/targets",
|
"/source/{sourceID}/targets",
|
||||||
env.handlers.HandleTargetCreate(),
|
env.handlers.HandleTargetCreate(),
|
||||||
)
|
)
|
||||||
|
|
||||||
target := "/hook/" + webhookID + "/targets"
|
target := "/source/" + webhookID + "/targets"
|
||||||
if query != "" {
|
if query != "" {
|
||||||
target += "?" + query
|
target += "?" + query
|
||||||
}
|
}
|
||||||
@@ -114,7 +114,7 @@ func postTargetCreate(
|
|||||||
// regression test for the ingress leak. r.FormValue falls back to the
|
// regression test for the ingress leak. r.FormValue falls back to the
|
||||||
// query string when a field is absent from the POST body, so
|
// query string when a field is absent from the POST body, so
|
||||||
//
|
//
|
||||||
// POST /hook/{id}/targets?url=https://hooks.slack.com/services/...
|
// POST /source/{id}/targets?url=https://hooks.slack.com/services/...
|
||||||
//
|
//
|
||||||
// with an empty url field used to create a working target from a value
|
// with an empty url field used to create a working target from a value
|
||||||
// carried on the request line — where logs, proxies, Referer headers
|
// carried on the request line — where logs, proxies, Referer headers
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user