Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e5b68b2df7 |
+4
-10
@@ -1,20 +1,14 @@
|
||||
# .git is sent so the build can derive the version it stamps into the binary
|
||||
# (script/version). Its config, which can hold a remote URL carrying a
|
||||
# credential and which `git describe` does not need, is left out of a
|
||||
# directory context. A context sent as a tar is not filtered by this file, so
|
||||
# it carries .git/config unless its sender leaves it out.
|
||||
.git/config
|
||||
|
||||
# No tracked file may be listed here: git in the build would see it as
|
||||
# deleted and mark the version -dirty.
|
||||
#
|
||||
# .ci-fingerprint is deliberately NOT excluded: it is the CI cache barrier
|
||||
# that keeps the check stages from replaying a cached pass. See the lint
|
||||
# stage of the Dockerfile.
|
||||
.git/
|
||||
bin/
|
||||
# Extracted from 3p/ by `make assets` inside the build; a host copy is not
|
||||
# needed. The tarball in 3p/ must stay in the context.
|
||||
static/js/alpine.min.js
|
||||
*.md
|
||||
LICENSE
|
||||
.editorconfig
|
||||
.env
|
||||
.env.*
|
||||
*.db
|
||||
|
||||
@@ -12,8 +12,9 @@ jobs:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 2024-10-23
|
||||
with:
|
||||
# The superseded-status step needs history to walk ancestors (it
|
||||
# aborts on a shallow clone).
|
||||
# The fingerprint step below needs history to find the last commit
|
||||
# that touched the Docker build context, and the superseded-status
|
||||
# step needs it to walk ancestors (it aborts on a shallow clone).
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Mark superseded run statuses
|
||||
@@ -27,11 +28,16 @@ jobs:
|
||||
run: script/ci-mark-superseded
|
||||
|
||||
- name: Fingerprint the build context
|
||||
# Writes the hash of the commit being checked into the context, which
|
||||
# invalidates the `COPY . .` layer of both check stages: a commit
|
||||
# that was never linted, format-checked, tested and built cannot
|
||||
# report success from cache.
|
||||
run: git rev-parse HEAD > .ci-fingerprint
|
||||
# `.dockerignore` keeps docs out of the build context, so a docs-only
|
||||
# commit legitimately replays the whole image from cache and stays
|
||||
# cheap. Every other commit writes a new fingerprint into the context,
|
||||
# which invalidates the `COPY . .` layer of both check stages: a
|
||||
# commit that was never linted, formatted-checked, tested and built
|
||||
# cannot report success from cache.
|
||||
run: |
|
||||
set -eu
|
||||
fp="$(git log -1 --format=%H -- . ':!*.md' ':!LICENSE' ':!.editorconfig')"
|
||||
printf '%s\n' "${fp:-$GITHUB_SHA}" > .ci-fingerprint
|
||||
|
||||
- name: Build Docker image (runs make fmt-check, golangci-lint, make test, make build)
|
||||
- name: Build Docker image (runs make check)
|
||||
run: script/cibuild
|
||||
|
||||
+9
-22
@@ -12,8 +12,8 @@ WORKDIR /src
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
# Copy source code. In CI the context also carries .ci-fingerprint, which
|
||||
# holds the hash of the commit being checked (see
|
||||
# Copy source code. In CI the context also carries .ci-fingerprint, whose
|
||||
# value changes with every commit that touches the build context (see
|
||||
# .gitea/workflows/check.yml). That invalidates this layer, so the checks
|
||||
# below cannot report success by replaying a cached pass. Do not add it to
|
||||
# .dockerignore.
|
||||
@@ -38,13 +38,8 @@ FROM golang:1.26.1-bookworm@sha256:4465644228bc2857a954b092167e12aa59c006a349228
|
||||
COPY --from=lint /src/go.sum /dev/null
|
||||
|
||||
# jq is a runtime dependency of script/ci-mark-superseded, which the test
|
||||
# suite executes. git is what script/version derives the version with.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends make curl ca-certificates jq git && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# A build context sent as a tar archive keeps its files' owners, and git
|
||||
# refuses to read a checkout owned by another user. Trust this one
|
||||
# whoever owns it.
|
||||
RUN git config --system --add safe.directory /build
|
||||
# suite executes.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends make curl ca-certificates jq && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
@@ -60,22 +55,14 @@ COPY . .
|
||||
# from its tarball in 3p/.
|
||||
RUN make test
|
||||
|
||||
# Version stamped into the binary: the VERSION build arg when one is
|
||||
# given, otherwise what script/version derives from the .git the build
|
||||
# context carries, so any `docker build .` of a clone stamps its commit.
|
||||
# With neither, as from a source tarball, it is "unknown".
|
||||
# Version stamped into the binary. .dockerignore excludes .git/, so
|
||||
# nothing in this stage can derive it: script/docker resolves it on the
|
||||
# host and passes it in. The default is what a bare `docker build .`
|
||||
# with no --build-arg gets, and it names no tag the tree may not be at.
|
||||
#
|
||||
# Declared here, below the test step, so a changed version does not
|
||||
# invalidate its cached layer.
|
||||
ARG VERSION
|
||||
|
||||
# A context that carries .git must not stamp "unknown": that means git is
|
||||
# missing here or could not read the checkout, and the image could not be
|
||||
# traced back to its commit.
|
||||
RUN if [ -d .git ] && [ "$(make version VERSION="$VERSION")" = unknown ]; then \
|
||||
echo "version is unknown although the build context carries .git" >&2; \
|
||||
exit 1; \
|
||||
fi
|
||||
ARG VERSION=unknown
|
||||
|
||||
RUN make build VERSION="$VERSION"
|
||||
|
||||
|
||||
@@ -4,12 +4,12 @@
|
||||
.DEFAULT_GOAL := check
|
||||
|
||||
# Version stamped into the binary. Derived from git by script/version;
|
||||
# override it (`make build VERSION=v1.2.3`) to stamp a given value, which is
|
||||
# how the Dockerfile passes its build arg in.
|
||||
# override it (`make build VERSION=v1.2.3`) where git metadata is
|
||||
# unavailable, which is how the Dockerfile passes its build arg in.
|
||||
VERSION ?= $(shell script/version)
|
||||
|
||||
# An empty override (`make build VERSION=`, or the Dockerfile's `make build
|
||||
# VERSION="$VERSION"` when no VERSION build arg was given) means unset,
|
||||
# An empty override (`make build VERSION=`, or a `--build-arg VERSION=`
|
||||
# landing on the Dockerfile's `make build VERSION="$VERSION"`) means unset,
|
||||
# exactly as it does in script/version -- stamping "" would leave the binary
|
||||
# reporting no version and the footer back on its "dev" fallback. `override`
|
||||
# is required: a plain assignment loses to the command-line definition it
|
||||
|
||||
@@ -40,6 +40,12 @@ duplicate. That is deliberate — the alternative is a silent lost
|
||||
delivery — and the README says so under Rationale. It is not a defect
|
||||
to re-file.
|
||||
|
||||
One caveat on reading a green check: a docs-only commit deliberately
|
||||
replays from the layer cache
|
||||
(https://git.eeqj.de/sneak/webhooker/issues/119), so a green status on
|
||||
such a commit evidences a replay rather than an executed run. A code
|
||||
commit invalidates the `COPY` layer and genuinely executes.
|
||||
|
||||
# Next Step
|
||||
|
||||
Clear the rest of the open 1.0.0 milestone
|
||||
@@ -381,7 +387,7 @@ point of the branch.
|
||||
- 2026-03-05 security headers middleware, session regeneration on
|
||||
login, request body size limits (#41)
|
||||
- 2026-03-04 tests for delivery, middleware, and session packages
|
||||
(#32); removed the build-architecture global (#31)
|
||||
(#32); removed globals.Buildarch (#31)
|
||||
- 2026-03-04 1.0 MVP merge: Webhook/Entrypoint/Target rename, core
|
||||
delivery engine with bounded worker pool and circuit breaker,
|
||||
parallel fan-out, per-webhook event databases, management UI (#16)
|
||||
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
"sneak.berlin/go/webhooker/internal/handlers"
|
||||
"sneak.berlin/go/webhooker/internal/healthcheck"
|
||||
"sneak.berlin/go/webhooker/internal/logger"
|
||||
"sneak.berlin/go/webhooker/internal/metrics"
|
||||
"sneak.berlin/go/webhooker/internal/middleware"
|
||||
"sneak.berlin/go/webhooker/internal/resetpw"
|
||||
"sneak.berlin/go/webhooker/internal/server"
|
||||
@@ -178,10 +177,6 @@ func newApp() *fx.App {
|
||||
healthcheck.New,
|
||||
session.New,
|
||||
handlers.New,
|
||||
// The registry /metrics serves, and the delivery
|
||||
// collectors registered on it.
|
||||
metrics.NewRegistry,
|
||||
metrics.New,
|
||||
middleware.New,
|
||||
// The one SSRF guard both target-creation validation
|
||||
// and the delivery dialer consult, so they cannot
|
||||
@@ -192,10 +187,11 @@ func newApp() *fx.App {
|
||||
// Wire *delivery.Engine as delivery.Notifier so the
|
||||
// webhook handler can notify the engine of new deliveries.
|
||||
func(e *delivery.Engine) delivery.Notifier { return e },
|
||||
// Wire *delivery.Engine as delivery.Archives so deleting
|
||||
// or renaming a webhook or target reaches its archive
|
||||
// files.
|
||||
func(e *delivery.Engine) delivery.Archives { return e },
|
||||
// Wire *delivery.Engine as delivery.WebhookEvictor so
|
||||
// deleting a webhook releases its archive writer.
|
||||
func(e *delivery.Engine) delivery.WebhookEvictor {
|
||||
return e
|
||||
},
|
||||
server.New,
|
||||
),
|
||||
fx.Invoke(
|
||||
|
||||
+74
-29
@@ -75,11 +75,6 @@ const (
|
||||
// internet-exposed endpoint.
|
||||
defaultReceiverRateLimit = 120
|
||||
|
||||
// defaultTrustedProxies is TRUSTED_PROXIES when it is unset: the
|
||||
// RFC 1918 private ranges, which a reverse proxy reaching the
|
||||
// process over a Docker network or a private LAN connects from.
|
||||
defaultTrustedProxies = "10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
|
||||
|
||||
// maxPort is the highest valid TCP port number. The lower
|
||||
// bound (at least 1) is enforced by envPositiveInt.
|
||||
maxPort = 65535
|
||||
@@ -149,6 +144,7 @@ type ConfigParams struct {
|
||||
type Config struct {
|
||||
DataDir string
|
||||
Debug bool
|
||||
MaintenanceMode bool
|
||||
Environment string
|
||||
MetricsPassword string
|
||||
MetricsUsername string
|
||||
@@ -176,14 +172,13 @@ type Config struct {
|
||||
|
||||
// TrustedProxies is the set of networks whose members are
|
||||
// allowed to speak for the client with X-Forwarded-For, the
|
||||
// only forwarded header read. Unless TRUSTED_PROXIES is set it
|
||||
// is the RFC 1918 private ranges (defaultTrustedProxies); a set
|
||||
// value replaces them. If any client can reach the process, or
|
||||
// the proxy in front of it, from an RFC 1918 source address
|
||||
// (directly, or through anything that can rewrite source
|
||||
// addresses, such as NAT or a published container port), it
|
||||
// must be set to the proxy's address alone, or every rate limit
|
||||
// can be bypassed by those clients.
|
||||
// only forwarded header read. It is empty unless
|
||||
// TRUSTED_PROXIES is set, and empty means no peer is
|
||||
// trusted: forwarded headers are then ignored entirely and
|
||||
// clients are identified by the connection's own address.
|
||||
// Members can choose their own rate-limit key, so this must
|
||||
// name proxy hosts only, never a block that also covers
|
||||
// clients.
|
||||
TrustedProxies []netip.Prefix
|
||||
|
||||
// AllowedEgressCIDRs is the set of networks a delivery target
|
||||
@@ -195,13 +190,12 @@ type Config struct {
|
||||
// otherwise refuse. The guard itself is always on: there is no
|
||||
// setting that disables SSRF protection, and delivery's
|
||||
// alwaysBlockedNetworks stays blocked no matter what is listed
|
||||
// here. That set is link-local, the unspecified addresses
|
||||
// 0.0.0.0 and ::, and the cloud metadata endpoints outside
|
||||
// link-local that disclose credentials or user data at a
|
||||
// provider-fixed, non-public address; it is not exhaustive of
|
||||
// every cloud's metadata address. See
|
||||
// alwaysBlockedNetworks for the authoritative list and why
|
||||
// each entry is on it.
|
||||
// here. That set is link-local plus the cloud metadata
|
||||
// endpoints outside it that disclose credentials or user data
|
||||
// at a provider-fixed, non-public address; it is not
|
||||
// exhaustive of every cloud's metadata address. See
|
||||
// alwaysBlockedNetworks for the authoritative list and the
|
||||
// criterion it is built from.
|
||||
AllowedEgressCIDRs []netip.Prefix
|
||||
|
||||
params *ConfigParams
|
||||
@@ -466,15 +460,14 @@ func parseCIDR(entry string) (netip.Prefix, error) {
|
||||
|
||||
// envPrefixList returns the value of the named environment variable
|
||||
// parsed as a comma-separated list of CIDR blocks (bare addresses
|
||||
// allowed). An unset, empty, or blank value is read as defaultValue
|
||||
// instead. A set value containing an unparseable entry is a hard
|
||||
// error naming the key and the bad entry, so startup fails loudly
|
||||
// rather than silently running with a list the operator did not
|
||||
// intend.
|
||||
func envPrefixList(key, defaultValue string) ([]netip.Prefix, error) {
|
||||
// allowed). An unset, empty, or blank value yields an empty list. A
|
||||
// set value containing an unparseable entry is a hard error naming
|
||||
// the key and the bad entry, so startup fails loudly rather than
|
||||
// silently running with a list the operator did not intend.
|
||||
func envPrefixList(key string) ([]netip.Prefix, error) {
|
||||
v := strings.TrimSpace(os.Getenv(key))
|
||||
if v == "" {
|
||||
v = defaultValue
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var prefixes []netip.Prefix
|
||||
@@ -657,6 +650,11 @@ func loadFromEnv() (*Config, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
maintenanceMode, err := envBool("MAINTENANCE_MODE", false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
retentionSweepInterval, err := envPositiveDuration(
|
||||
"RETENTION_SWEEP_INTERVAL",
|
||||
defaultRetentionSweepInterval,
|
||||
@@ -683,12 +681,12 @@ func loadFromEnv() (*Config, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
trustedProxies, err := envPrefixList("TRUSTED_PROXIES", defaultTrustedProxies)
|
||||
trustedProxies, err := envPrefixList("TRUSTED_PROXIES")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
allowedEgressCIDRs, err := envPrefixList("ALLOWED_EGRESS_CIDRS", "")
|
||||
allowedEgressCIDRs, err := envPrefixList("ALLOWED_EGRESS_CIDRS")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -706,6 +704,7 @@ func loadFromEnv() (*Config, error) {
|
||||
return &Config{
|
||||
DataDir: DataDir(),
|
||||
Debug: debug,
|
||||
MaintenanceMode: maintenanceMode,
|
||||
Environment: environment,
|
||||
MetricsUsername: metricsUsername,
|
||||
MetricsPassword: metricsPassword,
|
||||
@@ -761,6 +760,50 @@ func (c *Config) warnEgressAllowlist(log *slog.Logger) {
|
||||
)
|
||||
}
|
||||
|
||||
// warnSharedRateLimitBucket logs a startup warning whenever
|
||||
// TRUSTED_PROXIES is empty, in any environment.
|
||||
//
|
||||
// With no trusted proxies every rate limiter keys on the connecting
|
||||
// peer's address. Whether that is harmless or dangerous depends on
|
||||
// what is in front of the process, which this code cannot observe:
|
||||
// with nothing in front, the peer is the client and the limits are
|
||||
// per-client as intended; behind a reverse proxy the peer is the proxy
|
||||
// for every request, so all clients share one bucket per limiter.
|
||||
//
|
||||
// The login endpoint no longer spends budget on arrival — it verifies
|
||||
// credentials first and charges only failures — so a shared bucket
|
||||
// cannot deny the operator a correct password. What it does collapse
|
||||
// is the failure counting: one client's wrong passwords throttle
|
||||
// everyone else's wrong passwords, and the receiver's limits become
|
||||
// service-wide ceilings.
|
||||
//
|
||||
// The warning is deliberately not gated on WEBHOOKER_ENVIRONMENT:
|
||||
// behind a proxy every client shares one bucket in dev and prod alike.
|
||||
//
|
||||
// The default of trusting nobody is deliberate — trusting forwarded
|
||||
// headers from arbitrary peers lets any client choose its own bucket —
|
||||
// so this warns rather than failing startup or changing the key.
|
||||
func (c *Config) warnSharedRateLimitBucket(log *slog.Logger) {
|
||||
if len(c.TrustedProxies) > 0 {
|
||||
return
|
||||
}
|
||||
|
||||
log.Warn(
|
||||
"TRUSTED_PROXIES is empty: every rate limit keys on the "+
|
||||
"connecting peer's address. With nothing proxying to "+
|
||||
"this process that is the client itself and the limits "+
|
||||
"are per-client as intended. Behind a reverse proxy the "+
|
||||
"peer is the proxy on every request, so all clients "+
|
||||
"share one bucket per limit: the receiver limits become "+
|
||||
"service-wide ceilings, and one client's failed logins "+
|
||||
"throttle every other client's failed logins — a "+
|
||||
"correct password still gets in. If anything proxies to "+
|
||||
"this process, set TRUSTED_PROXIES to its address.",
|
||||
"environment", c.Environment,
|
||||
"trustedProxies", len(c.TrustedProxies),
|
||||
)
|
||||
}
|
||||
|
||||
// New creates a Config by reading environment variables.
|
||||
//
|
||||
//nolint:revive // lc parameter is required by fx even if unused.
|
||||
@@ -792,6 +835,7 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) {
|
||||
// host can reach the admin UI.
|
||||
"bindAddress", s.BindAddress,
|
||||
"debug", s.Debug,
|
||||
"maintenanceMode", s.MaintenanceMode,
|
||||
"dataDir", s.DataDir,
|
||||
"retentionSweepInterval", s.RetentionSweepInterval.String(),
|
||||
// Logged because a perfectly valid non-positive value here
|
||||
@@ -805,6 +849,7 @@ func New(lc fx.Lifecycle, params ConfigParams) (*Config, error) {
|
||||
"hasMetricsAuth", s.MetricsAuthEnabled(),
|
||||
)
|
||||
|
||||
s.warnSharedRateLimitBucket(log)
|
||||
s.warnEgressAllowlist(log)
|
||||
|
||||
return s, nil
|
||||
|
||||
+101
-26
@@ -124,11 +124,6 @@ func testEnvironmentConfigSuccess(
|
||||
|
||||
app := fxtest.New(
|
||||
t,
|
||||
// fx's own log is discarded, not sent to t.Logf: a hook still
|
||||
// running after a start or stop timeout would write there after
|
||||
// the test has returned. The same holds for every fxtest.New
|
||||
// below.
|
||||
fx.NopLogger,
|
||||
fx.Provide(
|
||||
globals.New,
|
||||
logger.New,
|
||||
@@ -277,7 +272,6 @@ func testRetentionSweepIntervalSuccess(
|
||||
|
||||
app := fxtest.New(
|
||||
t,
|
||||
fx.NopLogger,
|
||||
fx.Provide(
|
||||
globals.New,
|
||||
logger.New,
|
||||
@@ -370,7 +364,6 @@ func testSessionIdleTimeoutSuccess(
|
||||
|
||||
app := fxtest.New(
|
||||
t,
|
||||
fx.NopLogger,
|
||||
fx.Provide(
|
||||
globals.New,
|
||||
logger.New,
|
||||
@@ -411,7 +404,6 @@ func TestDefaultDataDir(t *testing.T) {
|
||||
|
||||
app := fxtest.New(
|
||||
t,
|
||||
fx.NopLogger,
|
||||
fx.Provide(
|
||||
globals.New,
|
||||
logger.New,
|
||||
@@ -542,7 +534,6 @@ func testReceiverRateLimitSuccess(
|
||||
|
||||
app := fxtest.New(
|
||||
t,
|
||||
fx.NopLogger,
|
||||
fx.Provide(
|
||||
globals.New,
|
||||
logger.New,
|
||||
@@ -560,11 +551,6 @@ func testReceiverRateLimitSuccess(
|
||||
}
|
||||
|
||||
func TestTrustedProxies(t *testing.T) {
|
||||
// Unset, the RFC 1918 private ranges are trusted, so a reverse
|
||||
// proxy on a Docker network or a private LAN is covered without
|
||||
// configuration.
|
||||
defaultProxies := []string{cidrPrivateV4, "172.16.0.0/12", "192.168.0.0/16"}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
set bool
|
||||
@@ -573,21 +559,18 @@ func TestTrustedProxies(t *testing.T) {
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
// The default must be "trust nobody": an empty list
|
||||
// means forwarded headers are ignored, never that
|
||||
// every peer may speak for the client.
|
||||
name: caseUnsetUsesDefault,
|
||||
set: false,
|
||||
expected: defaultProxies,
|
||||
expected: []string{},
|
||||
},
|
||||
{
|
||||
name: "blank value uses default",
|
||||
name: "blank value trusts nothing",
|
||||
set: true,
|
||||
value: " ",
|
||||
expected: defaultProxies,
|
||||
},
|
||||
{
|
||||
name: "set value replaces the default entirely",
|
||||
set: true,
|
||||
value: "203.0.113.7",
|
||||
expected: []string{"203.0.113.7/32"},
|
||||
expected: []string{},
|
||||
},
|
||||
{
|
||||
name: caseValidValueParsed,
|
||||
@@ -659,7 +642,6 @@ func testTrustedProxiesSuccess(
|
||||
|
||||
app := fxtest.New(
|
||||
t,
|
||||
fx.NopLogger,
|
||||
fx.Provide(
|
||||
globals.New,
|
||||
logger.New,
|
||||
@@ -773,7 +755,6 @@ func testAllowedEgressCIDRsSuccess(
|
||||
|
||||
app := fxtest.New(
|
||||
t,
|
||||
fx.NopLogger,
|
||||
fx.Provide(
|
||||
globals.New,
|
||||
logger.New,
|
||||
@@ -864,6 +845,101 @@ func TestEgressAllowlistWarning(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestSharedRateLimitBucketWarning covers the startup warning that
|
||||
// tells an operator a deployment behind a reverse proxy shares one
|
||||
// rate-limit bucket between every client, which turns the receiver
|
||||
// limits into service-wide ceilings and collapses login failure
|
||||
// counting. It must fire whenever TRUSTED_PROXIES is empty, in any
|
||||
// environment, because behind a proxy every client shares one bucket
|
||||
// in dev and prod alike. It stays quiet once proxies are named.
|
||||
func TestSharedRateLimitBucketWarning(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
environment string
|
||||
trustedProxies string
|
||||
expectWarning bool
|
||||
}{
|
||||
{
|
||||
name: "prod without trusted proxies warns",
|
||||
environment: config.EnvironmentProd,
|
||||
expectWarning: true,
|
||||
},
|
||||
{
|
||||
name: "prod with trusted proxies is quiet",
|
||||
environment: config.EnvironmentProd,
|
||||
trustedProxies: cidrPrivateV4,
|
||||
expectWarning: false,
|
||||
},
|
||||
{
|
||||
name: "dev without trusted proxies warns",
|
||||
environment: config.EnvironmentDev,
|
||||
expectWarning: true,
|
||||
},
|
||||
{
|
||||
name: "dev with trusted proxies is quiet",
|
||||
environment: config.EnvironmentDev,
|
||||
trustedProxies: cidrPrivateV4,
|
||||
expectWarning: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Cannot use t.Parallel() here because t.Setenv
|
||||
// is incompatible with parallel subtests.
|
||||
t.Setenv("WEBHOOKER_ENVIRONMENT", tt.environment)
|
||||
|
||||
if tt.trustedProxies == "" {
|
||||
require.NoError(
|
||||
t, os.Unsetenv("TRUSTED_PROXIES"),
|
||||
)
|
||||
} else {
|
||||
t.Setenv("TRUSTED_PROXIES", tt.trustedProxies)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
log := slog.New(slog.NewJSONHandler(
|
||||
&buf, &slog.HandlerOptions{
|
||||
Level: slog.LevelDebug,
|
||||
},
|
||||
))
|
||||
|
||||
require.NoError(
|
||||
t,
|
||||
config.WarnSharedRateLimitBucketForTest(log),
|
||||
)
|
||||
|
||||
if !tt.expectWarning {
|
||||
assert.Empty(t, buf.String())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
logged := buf.String()
|
||||
|
||||
assert.Contains(t, logged, `"level":"WARN"`)
|
||||
assert.Contains(t, logged, "TRUSTED_PROXIES")
|
||||
assert.Contains(t, logged, "share one bucket")
|
||||
assert.Contains(
|
||||
t, logged, "throttle every other client's failed logins",
|
||||
)
|
||||
// The warning must not claim a lockout the login
|
||||
// endpoint no longer permits: credentials are verified
|
||||
// before any budget is spent.
|
||||
assert.Contains(
|
||||
t, logged, "a correct password still gets in",
|
||||
)
|
||||
// The text must stay accurate for a developer with
|
||||
// nothing in front of the process, where an empty
|
||||
// list costs nothing.
|
||||
assert.Contains(
|
||||
t, logged, "nothing proxying to this process",
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// metricsEnv describes what one subtest below puts in the
|
||||
// environment for a single METRICS_ variable. A variable that is
|
||||
// set to the empty string and one that is not set at all are
|
||||
@@ -1017,7 +1093,6 @@ func assertMetricsAuthAccepted(t *testing.T, expectAuth bool) {
|
||||
|
||||
app := fxtest.New(
|
||||
t,
|
||||
fx.NopLogger,
|
||||
fx.Provide(globals.New, logger.New, config.New),
|
||||
fx.Populate(&cfg),
|
||||
)
|
||||
|
||||
@@ -18,9 +18,10 @@ const testEnvKey = "WEBHOOKER_TEST_VALUE"
|
||||
|
||||
// Real configuration variables exercised by the config.New tests.
|
||||
const (
|
||||
envKeyPort = "PORT"
|
||||
envKeyDebug = "DEBUG"
|
||||
envKeyBindAddress = "BIND_ADDRESS"
|
||||
envKeyPort = "PORT"
|
||||
envKeyDebug = "DEBUG"
|
||||
envKeyMaintenanceMode = "MAINTENANCE_MODE"
|
||||
envKeyBindAddress = "BIND_ADDRESS"
|
||||
)
|
||||
|
||||
// Sample BIND_ADDRESS values used by the tables below.
|
||||
@@ -603,6 +604,12 @@ func flagEnvValueCases() []badEnvValueCase {
|
||||
value: "ture",
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
name: "unparseable MAINTENANCE_MODE aborts startup",
|
||||
key: envKeyMaintenanceMode,
|
||||
value: "sometimes",
|
||||
expectError: true,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -649,7 +656,8 @@ func TestNewUsesDefaultsWhenUnset(t *testing.T) {
|
||||
t.Setenv("WEBHOOKER_ENVIRONMENT", "dev")
|
||||
|
||||
for _, key := range []string{
|
||||
envKeyPort, envKeyDebug, envKeyBindAddress, envKeySentryDSN,
|
||||
envKeyPort, envKeyDebug, envKeyMaintenanceMode,
|
||||
envKeyBindAddress, envKeySentryDSN,
|
||||
} {
|
||||
require.NoError(t, os.Unsetenv(key))
|
||||
}
|
||||
@@ -660,6 +668,7 @@ func TestNewUsesDefaultsWhenUnset(t *testing.T) {
|
||||
|
||||
assert.Equal(t, 8080, cfg.Port)
|
||||
assert.False(t, cfg.Debug)
|
||||
assert.False(t, cfg.MaintenanceMode)
|
||||
|
||||
// Loopback, not the wildcard: the default must not publish the
|
||||
// cleartext admin UI and the unauthenticated receiver on every
|
||||
|
||||
@@ -6,6 +6,21 @@ import "log/slog"
|
||||
// the external config_test package so each helper can be covered by
|
||||
// its own table-driven test without weakening the package API.
|
||||
|
||||
// WarnSharedRateLimitBucketForTest loads a Config from the current
|
||||
// environment and emits its startup warnings to log. The real logger
|
||||
// writes to stdout, so this lets the warning's firing condition be
|
||||
// asserted against a handler the test controls.
|
||||
func WarnSharedRateLimitBucketForTest(log *slog.Logger) error {
|
||||
c, err := loadFromEnv()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
c.warnSharedRateLimitBucket(log)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// WarnEgressAllowlistForTest loads a Config from the current
|
||||
// environment and emits its egress-allowlist startup warning to
|
||||
// log, so a test can assert both that the warning fires only when
|
||||
|
||||
@@ -93,11 +93,11 @@ func TestEventTierQueriesUseTheirIndexes(t *testing.T) {
|
||||
deliveries []database.Delivery
|
||||
results []database.DeliveryResult
|
||||
depths []struct{ Depth int }
|
||||
removed []database.TargetTotals
|
||||
)
|
||||
|
||||
byStatus := "idx_deliveries_status (status=? AND deleted_at=?)"
|
||||
byEvent := "idx_deliveries_event_id (event_id=? AND deleted_at=?)"
|
||||
byAge := "idx_events_deleted_at_created_at (deleted_at=? AND created_at<?)"
|
||||
|
||||
// The delivery engine: recovery and the retry sweep, the sweep for
|
||||
// stranded pending deliveries, and the queue depth count.
|
||||
@@ -123,80 +123,25 @@ func TestEventTierQueriesUseTheirIndexes(t *testing.T) {
|
||||
Order("attempt_num ASC").Find(&results),
|
||||
"idx_delivery_results_delivery_id (delivery_id=? AND deleted_at=?)")
|
||||
|
||||
// Retention (reapExpired, deleteEvents): one batch of expired
|
||||
// events, then their attempts, deliveries and the events.
|
||||
var expired []string
|
||||
// Retention's three deletes (reapExpired), whose subqueries are built
|
||||
// afresh for each statement as it builds them.
|
||||
expiredEventIDs := func() *gorm.DB {
|
||||
return dry.Model(&database.Event{}).Select("id").
|
||||
Where("created_at < ?", cutoff)
|
||||
}
|
||||
|
||||
assertPlanUses(t, db, dry.Unscoped().Model(&database.Event{}).
|
||||
Where("created_at < ?", cutoff).
|
||||
Limit(database.ExportReapBatchSize).Pluck("id", &expired),
|
||||
"idx_events_created_at (created_at<?)")
|
||||
assertPlanUses(t, db, dry.Unscoped().Where(
|
||||
"delivery_id IN (?)", dry.Unscoped().Model(&database.Delivery{}).
|
||||
Select("id").Where("event_id IN ?", ids),
|
||||
"delivery_id IN (?)", dry.Model(&database.Delivery{}).
|
||||
Select("id").Where("event_id IN (?)", expiredEventIDs()),
|
||||
).Delete(&database.DeliveryResult{}),
|
||||
"idx_delivery_results_delivery_id (delivery_id=?)",
|
||||
"idx_deliveries_event_id (event_id=?)")
|
||||
assertPlanUses(t, db, dry.Unscoped().Model(&database.Delivery{}).
|
||||
Select("target_id, count(*) AS deliveries_removed, "+
|
||||
"count(CASE WHEN status = ? THEN 1 END) AS failed_removed",
|
||||
database.DeliveryStatusFailed).
|
||||
Where("event_id IN ?", ids).Group("target_id").Find(&removed),
|
||||
"idx_deliveries_event_id (event_id=?)")
|
||||
assertPlanUses(t, db, dry.Unscoped().Where("event_id IN ?", ids).
|
||||
Delete(&database.Delivery{}), "idx_deliveries_event_id (event_id=?)")
|
||||
assertPlanUses(t, db, dry.Unscoped().Where("id IN ?", ids).
|
||||
Delete(&database.Event{}), "sqlite_autoindex_events_1 (id=?)")
|
||||
}
|
||||
|
||||
// TestStatisticsQueriesUseTheirIndexes does the same for the webhook
|
||||
// page's statistics (readEventStats in the handlers): deliveries in
|
||||
// progress, each target's deliveries finished since a time, which must
|
||||
// come from the index alone, and events received since a time.
|
||||
func TestStatisticsQueriesUseTheirIndexes(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
mgr, lc := setupTestWebhookDBManager(t)
|
||||
ctx := context.Background()
|
||||
require.NoError(t, lc.Start(ctx))
|
||||
|
||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
||||
|
||||
db, err := mgr.GetDB(uuid.New().String())
|
||||
require.NoError(t, err)
|
||||
|
||||
dry := db.Session(&gorm.Session{DryRun: true})
|
||||
since := time.Now()
|
||||
|
||||
var (
|
||||
count int64
|
||||
byTarget []struct{ TargetID string }
|
||||
)
|
||||
|
||||
assertPlanUses(t, db, dry.Model(&database.Delivery{}).
|
||||
Where("status IN ?", []database.DeliveryStatus{
|
||||
database.DeliveryStatusPending,
|
||||
database.DeliveryStatusRetrying,
|
||||
}).Count(&count),
|
||||
"idx_deliveries_status (status=? AND deleted_at=?)")
|
||||
assertPlanUses(t, db, dry.Model(&database.Delivery{}).
|
||||
Select("target_id, "+
|
||||
"count(CASE WHEN status = ? THEN 1 END) AS delivered, "+
|
||||
"count(CASE WHEN status = ? THEN 1 END) AS failed",
|
||||
database.DeliveryStatusDelivered,
|
||||
database.DeliveryStatusFailed).
|
||||
Where("status IN ? AND finished_at >= ?",
|
||||
[]database.DeliveryStatus{
|
||||
database.DeliveryStatusDelivered,
|
||||
database.DeliveryStatusFailed,
|
||||
}, since).
|
||||
Group("target_id").Find(&byTarget),
|
||||
"COVERING INDEX idx_deliveries_status "+
|
||||
"(status=? AND deleted_at=? AND finished_at>?)")
|
||||
assertPlanUses(t, db, dry.Model(&database.Event{}).
|
||||
Where("created_at >= ?", since).Count(&count),
|
||||
"idx_events_deleted_at_created_at "+
|
||||
"(deleted_at=? AND created_at>?)")
|
||||
"idx_delivery_results_delivery_id (delivery_id=?)", byEvent, byAge)
|
||||
assertPlanUses(t, db, dry.Unscoped().Where(
|
||||
"event_id IN (?)", expiredEventIDs(),
|
||||
).Delete(&database.Delivery{}),
|
||||
"idx_deliveries_event_id (event_id=?)", byAge)
|
||||
assertPlanUses(t, db, dry.Unscoped().Where(
|
||||
"created_at < ?", cutoff,
|
||||
).Delete(&database.Event{}), "idx_events_created_at (created_at<?)")
|
||||
}
|
||||
|
||||
// assertPlanUses asserts that SQLite's plan for a statement GORM built
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.uber.org/fx"
|
||||
@@ -29,10 +28,6 @@ func NewTestRetentionReaper(
|
||||
}
|
||||
}
|
||||
|
||||
// ExportReapBatchSize exposes how many expired events one retention
|
||||
// transaction deletes.
|
||||
const ExportReapBatchSize = reapBatchSize
|
||||
|
||||
// ExportSweep runs a single retention sweep synchronously for tests.
|
||||
func (r *RetentionReaper) ExportSweep(ctx context.Context) {
|
||||
r.sweep(ctx)
|
||||
@@ -84,14 +79,3 @@ func (d *Database) ExportSetBannerOut(w io.Writer) {
|
||||
func DummyPasswordHashForTest() string {
|
||||
return dummyPasswordHash()
|
||||
}
|
||||
|
||||
// HashAtShippedCostForTest makes HashPassword hash at the shipped
|
||||
// memory cost until t ends. t must not run in parallel with other
|
||||
// tests, which would hash at that cost alongside it.
|
||||
func HashAtShippedCostForTest(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
hashAtShippedCostInTest = true
|
||||
|
||||
t.Cleanup(func() { hashAtShippedCostInTest = false })
|
||||
}
|
||||
|
||||
@@ -15,7 +15,6 @@ type APIKey struct {
|
||||
Description string `json:"description"`
|
||||
LastUsedAt *time.Time `json:"lastUsedAt,omitempty"`
|
||||
|
||||
// Relations. No model marshals the record it belongs to:
|
||||
// User.APIKeys leads back here, and the JSON could loop.
|
||||
User User `json:"-"`
|
||||
// Relations
|
||||
User User `json:"user,omitzero"`
|
||||
}
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
import "gorm.io/gorm"
|
||||
|
||||
// DeliveryStatus represents the status of a delivery
|
||||
type DeliveryStatus string
|
||||
@@ -41,7 +37,7 @@ type Delivery struct {
|
||||
BaseModel
|
||||
|
||||
EventID string `gorm:"type:uuid;not null;index:idx_deliveries_event_id,priority:1" json:"eventId"`
|
||||
TargetID string `gorm:"type:uuid;not null;index:idx_deliveries_status,priority:4" json:"targetId"`
|
||||
TargetID string `gorm:"type:uuid;not null" json:"targetId"`
|
||||
Status DeliveryStatus `gorm:"not null;default:'pending';index:idx_deliveries_status,priority:1" json:"status"`
|
||||
|
||||
// DeletedAt repeats the BaseModel field only to be the second column
|
||||
@@ -49,17 +45,8 @@ type Delivery struct {
|
||||
// gives.
|
||||
DeletedAt gorm.DeletedAt `gorm:"index:idx_deliveries_event_id,priority:2;index:idx_deliveries_status,priority:2" json:"deletedAt,omitzero"`
|
||||
|
||||
// FinishedAt is when the delivery became delivered or failed, and
|
||||
// nil while it is pending or retrying. It and then TargetID end the
|
||||
// status index, so the webhook page counts each target's deliveries
|
||||
// that finished in a recent window by reading just that window from
|
||||
// the index.
|
||||
FinishedAt *time.Time `gorm:"index:idx_deliveries_status,priority:3" json:"finishedAt,omitempty"`
|
||||
|
||||
// Relations. No model marshals the record it belongs to:
|
||||
// Event.Deliveries and Target.Deliveries lead back here, and the
|
||||
// JSON could loop.
|
||||
Event Event `json:"-"`
|
||||
Target Target `json:"-"`
|
||||
// Relations
|
||||
Event Event `json:"event,omitzero"`
|
||||
Target Target `json:"target,omitzero"`
|
||||
DeliveryResults []DeliveryResult `json:"deliveryResults,omitempty"`
|
||||
}
|
||||
|
||||
@@ -23,7 +23,6 @@ type DeliveryResult struct {
|
||||
Error string `json:"error,omitempty"`
|
||||
Duration int64 `json:"durationMs"` // Duration in milliseconds
|
||||
|
||||
// Relations. No model marshals the record it belongs to:
|
||||
// Delivery.DeliveryResults leads back here, and the JSON could loop.
|
||||
Delivery Delivery `json:"-"`
|
||||
// Relations
|
||||
Delivery Delivery `json:"delivery,omitzero"`
|
||||
}
|
||||
|
||||
@@ -15,7 +15,6 @@ type Entrypoint struct {
|
||||
Description string `json:"description"`
|
||||
Active bool `gorm:"default:true" json:"active"`
|
||||
|
||||
// Relations. No model marshals the record it belongs to:
|
||||
// Webhook.Entrypoints leads back here, and the JSON could loop.
|
||||
Webhook Webhook `json:"-"`
|
||||
// Relations
|
||||
Webhook Webhook `json:"webhook,omitzero"`
|
||||
}
|
||||
|
||||
@@ -44,9 +44,8 @@ type Event struct {
|
||||
// kept as the record of where the copy came from either way.
|
||||
ResubmittedFromID *string `gorm:"type:uuid;index" json:"resubmittedFromId,omitempty"`
|
||||
|
||||
// Relations. No model marshals the record it belongs to, so
|
||||
// Webhook and Entrypoint are left out of the JSON.
|
||||
Webhook Webhook `json:"-"`
|
||||
Entrypoint Entrypoint `json:"-"`
|
||||
// Relations
|
||||
Webhook Webhook `json:"webhook,omitzero"`
|
||||
Entrypoint Entrypoint `json:"entrypoint,omitzero"`
|
||||
Deliveries []Delivery `json:"deliveries,omitempty"`
|
||||
}
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
package database_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
)
|
||||
|
||||
// TestPreloadedModelsMarshalWithoutTheirParent pins that a child's
|
||||
// reference to the record it belongs to is left out of the JSON, so a
|
||||
// webhook and its targets cannot marshal each other in a loop, and that
|
||||
// GORM still preloads that reference, since it ignores json tags.
|
||||
func TestPreloadedModelsMarshalWithoutTheirParent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := startedTestDB(t)
|
||||
|
||||
stored := database.Webhook{
|
||||
UserID: uuid.New().String(),
|
||||
Name: testWebhookName,
|
||||
Entrypoints: []database.Entrypoint{{Path: uuid.New().String()}},
|
||||
Targets: []database.Target{{
|
||||
Name: "log",
|
||||
Type: database.TargetTypeLog,
|
||||
}},
|
||||
}
|
||||
require.NoError(t, db.Create(&stored).Error)
|
||||
|
||||
entrypointID := stored.Entrypoints[0].ID
|
||||
targetID := stored.Targets[0].ID
|
||||
|
||||
var webhook database.Webhook
|
||||
|
||||
require.NoError(t, db.
|
||||
Preload("Entrypoints.Webhook").
|
||||
Preload("Targets.Webhook").
|
||||
First(&webhook, "id = ?", stored.ID).Error)
|
||||
|
||||
require.Len(t, webhook.Entrypoints, 1)
|
||||
require.Len(t, webhook.Targets, 1)
|
||||
assert.Equal(t, stored.ID, webhook.Entrypoints[0].Webhook.ID)
|
||||
assert.Equal(t, stored.ID, webhook.Targets[0].Webhook.ID)
|
||||
|
||||
encoded := marshalModel(t, webhook)
|
||||
|
||||
assert.Contains(t, encoded, entrypointID)
|
||||
assert.Contains(t, encoded, targetID)
|
||||
|
||||
// Each child holds the parent's id as its webhookId, so the parent
|
||||
// is looked for by its own id field.
|
||||
parentIDField := `"id":"` + stored.ID + `"`
|
||||
|
||||
assert.NotContains(t, marshalModel(t, webhook.Entrypoints[0]), parentIDField)
|
||||
assert.NotContains(t, marshalModel(t, webhook.Targets[0]), parentIDField)
|
||||
|
||||
var target database.Target
|
||||
|
||||
require.NoError(t, db.
|
||||
Preload("Webhook").
|
||||
First(&target, "id = ?", targetID).Error)
|
||||
|
||||
assert.Equal(t, stored.ID, target.Webhook.ID)
|
||||
|
||||
encoded = marshalModel(t, target)
|
||||
|
||||
assert.Contains(t, encoded, stored.ID)
|
||||
assert.NotContains(t, encoded, parentIDField)
|
||||
}
|
||||
|
||||
// TestModelsMarshalWithoutTheirParent covers the other references to a
|
||||
// parent: each model is built with its parent set, and the parent's id
|
||||
// must not appear in the JSON.
|
||||
func TestModelsMarshalWithoutTheirParent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
parent := database.BaseModel{ID: uuid.New().String()}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
model any
|
||||
}{
|
||||
{
|
||||
name: "Webhook.User",
|
||||
model: database.Webhook{User: database.User{BaseModel: parent}},
|
||||
},
|
||||
{
|
||||
name: "APIKey.User",
|
||||
model: database.APIKey{User: database.User{BaseModel: parent}},
|
||||
},
|
||||
{
|
||||
name: "Delivery.Event",
|
||||
model: database.Delivery{Event: database.Event{BaseModel: parent}},
|
||||
},
|
||||
{
|
||||
name: "Delivery.Target",
|
||||
model: database.Delivery{Target: database.Target{BaseModel: parent}},
|
||||
},
|
||||
{
|
||||
name: "DeliveryResult.Delivery",
|
||||
model: database.DeliveryResult{
|
||||
Delivery: database.Delivery{BaseModel: parent},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Event.Webhook",
|
||||
model: database.Event{Webhook: database.Webhook{BaseModel: parent}},
|
||||
},
|
||||
{
|
||||
name: "Event.Entrypoint",
|
||||
model: database.Event{
|
||||
Entrypoint: database.Entrypoint{BaseModel: parent},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert.NotContains(t, marshalModel(t, tc.model), parent.ID)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -34,8 +34,7 @@ type Target struct {
|
||||
MaxRetries int `json:"maxRetries,omitempty"`
|
||||
MaxQueueSize int `json:"maxQueueSize,omitempty"`
|
||||
|
||||
// Relations. No model marshals the record it belongs to:
|
||||
// Webhook.Targets leads back here, and the JSON could loop.
|
||||
Webhook Webhook `json:"-"`
|
||||
// Relations
|
||||
Webhook Webhook `json:"webhook,omitzero"`
|
||||
Deliveries []Delivery `json:"deliveries,omitempty"`
|
||||
}
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// The running totals in a webhook's event database keep the webhook
|
||||
// page's lifetime figures right after retention has removed the rows
|
||||
// they count, and let the page show them without counting every row.
|
||||
// Each total changes in the transaction that writes or deletes the
|
||||
// rows it counts.
|
||||
|
||||
// EventTotals is the single row counting a webhook's events: every
|
||||
// event ever stored, how many of them retention has deleted, and when
|
||||
// the newest arrived, which retention leaves as it is.
|
||||
type EventTotals struct {
|
||||
ID int64 `gorm:"primaryKey"`
|
||||
|
||||
Events int64 `gorm:"not null"`
|
||||
EventsRemoved int64 `gorm:"not null"`
|
||||
|
||||
// LastEventAt is when the newest event arrived, or nil before the
|
||||
// first.
|
||||
LastEventAt *time.Time
|
||||
}
|
||||
|
||||
// TableName names the table AddEventTotals updates.
|
||||
func (EventTotals) TableName() string {
|
||||
return "event_totals"
|
||||
}
|
||||
|
||||
// TargetTotals is one row per target counting its deliveries: every
|
||||
// delivery ever created, how many became delivered and how many
|
||||
// failed, and how many deliveries and failed deliveries retention has
|
||||
// deleted. The webhook's delivery figures are these rows summed.
|
||||
type TargetTotals struct {
|
||||
TargetID string `gorm:"type:uuid;primaryKey"`
|
||||
|
||||
Deliveries int64 `gorm:"not null"`
|
||||
Delivered int64 `gorm:"not null"`
|
||||
Failed int64 `gorm:"not null"`
|
||||
|
||||
DeliveriesRemoved int64 `gorm:"not null"`
|
||||
FailedRemoved int64 `gorm:"not null"`
|
||||
}
|
||||
|
||||
// TableName names the table AddTargetTotals updates.
|
||||
func (TargetTotals) TableName() string {
|
||||
return "target_totals"
|
||||
}
|
||||
|
||||
// AddEventTotals adds each count in add to the webhook's event totals,
|
||||
// and records add.LastEventAt as when the newest event arrived if it is
|
||||
// set. Call it on the transaction that writes or deletes the events it
|
||||
// counts.
|
||||
func AddEventTotals(tx *gorm.DB, add EventTotals) error {
|
||||
err := tx.Exec(
|
||||
`UPDATE event_totals SET
|
||||
events = events + ?,
|
||||
events_removed = events_removed + ?,
|
||||
last_event_at = coalesce(?, last_event_at)`,
|
||||
add.Events, add.EventsRemoved, add.LastEventAt,
|
||||
).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf("adding to event totals: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddTargetTotals adds each count in add to the totals of the target
|
||||
// add.TargetID names, creating its row the first time. Call it on the
|
||||
// transaction that writes or deletes the deliveries it counts.
|
||||
func AddTargetTotals(tx *gorm.DB, add TargetTotals) error {
|
||||
err := tx.Exec(
|
||||
`INSERT INTO target_totals (target_id, deliveries, delivered,
|
||||
failed, deliveries_removed, failed_removed)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (target_id) DO UPDATE SET
|
||||
deliveries = deliveries + excluded.deliveries,
|
||||
delivered = delivered + excluded.delivered,
|
||||
failed = failed + excluded.failed,
|
||||
deliveries_removed =
|
||||
deliveries_removed + excluded.deliveries_removed,
|
||||
failed_removed = failed_removed + excluded.failed_removed`,
|
||||
add.TargetID, add.Deliveries, add.Delivered,
|
||||
add.Failed, add.DeliveriesRemoved, add.FailedRemoved,
|
||||
).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"adding to totals of target %s: %w", add.TargetID, err,
|
||||
)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -66,9 +66,8 @@ type Webhook struct {
|
||||
// must equal DefaultRetentionDays.
|
||||
RetentionDays int `gorm:"default:30" json:"retentionDays"`
|
||||
|
||||
// Relations. No model marshals the record it belongs to:
|
||||
// User.Webhooks leads back here, and the JSON could loop.
|
||||
User User `json:"-"`
|
||||
// Relations
|
||||
User User `json:"user,omitzero"`
|
||||
Entrypoints []Entrypoint `json:"entrypoints,omitempty"`
|
||||
Targets []Target `json:"targets,omitempty"`
|
||||
}
|
||||
|
||||
@@ -2,8 +2,7 @@ package database
|
||||
|
||||
// Migrate runs database migrations for the main application database.
|
||||
// Only configuration-tier models are stored in the main database.
|
||||
// Event-tier models (Event, Delivery, DeliveryResult, EventTotals,
|
||||
// TargetTotals) live in
|
||||
// Event-tier models (Event, Delivery, DeliveryResult) live in
|
||||
// per-webhook dedicated databases managed by WebhookDBManager.
|
||||
func (d *Database) Migrate() error {
|
||||
return d.db.AutoMigrate(
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"math/big"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/crypto/argon2"
|
||||
)
|
||||
@@ -64,30 +63,10 @@ func DefaultPasswordConfig() *PasswordConfig {
|
||||
}
|
||||
}
|
||||
|
||||
// testArgon2Memory is the Argon2id memory cost, in KiB, that a test
|
||||
// binary hashes with: 1 MB instead of the shipped 64 MB. Every test
|
||||
// that starts a database hashes the bootstrap admin password, dozens
|
||||
// of them run in parallel, and under the race detector each 64 MB hash
|
||||
// holds about 150 MB. VerifyPassword reads the cost from the hash it
|
||||
// checks, so verification follows.
|
||||
const testArgon2Memory = 1024
|
||||
|
||||
// hashAtShippedCostInTest makes a test binary hash at the shipped
|
||||
// memory cost. Only TestHashPassword_ShippedParameters sets it.
|
||||
//
|
||||
//nolint:gochecknoglobals // set by one test, see above
|
||||
var hashAtShippedCostInTest bool
|
||||
|
||||
// HashPassword generates an Argon2id hash of the password. A binary
|
||||
// built by go test hashes at testArgon2Memory; one built by go build
|
||||
// always hashes at the defaults.
|
||||
// HashPassword generates an Argon2id hash of the password
|
||||
func HashPassword(password string) (string, error) {
|
||||
config := DefaultPasswordConfig()
|
||||
|
||||
if testing.Testing() && !hashAtShippedCostInTest {
|
||||
config.Memory = testArgon2Memory
|
||||
}
|
||||
|
||||
// Generate a salt
|
||||
salt := make([]byte, config.SaltLen)
|
||||
|
||||
|
||||
@@ -192,39 +192,6 @@ func TestHashPasswordUniqueness(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestHashPassword_ShippedParameters hashes and verifies through
|
||||
// HashPassword at the shipped Argon2id parameters. Every other test
|
||||
// hashes at the lower memory cost a test binary uses, so this is the
|
||||
// one that keeps production hashing covered. One hash and one
|
||||
// verification: each costs 64 MB.
|
||||
//
|
||||
//nolint:paralleltest // changes the hashing cost for the whole binary
|
||||
func TestHashPassword_ShippedParameters(t *testing.T) {
|
||||
database.HashAtShippedCostForTest(t)
|
||||
|
||||
password := "correct horse battery staple"
|
||||
|
||||
hash, err := database.HashPassword(password)
|
||||
if err != nil {
|
||||
t.Fatalf("hashing with the shipped parameters: %v", err)
|
||||
}
|
||||
|
||||
const shipped = "$argon2id$v=19$m=65536,t=1,p=4$"
|
||||
|
||||
if !strings.HasPrefix(hash, shipped) {
|
||||
t.Errorf("hash = %q, want prefix %q", hash, shipped)
|
||||
}
|
||||
|
||||
valid, err := database.VerifyPassword(password, hash)
|
||||
if err != nil {
|
||||
t.Fatalf("VerifyPassword() error = %v", err)
|
||||
}
|
||||
|
||||
if !valid {
|
||||
t.Error("VerifyPassword() returned false for correct password")
|
||||
}
|
||||
}
|
||||
|
||||
// TestVerifyDummyPassword_DoesRealWork covers the anti-enumeration
|
||||
// path. Login charges an unknown username a verification against a
|
||||
// dummy hash so that a nonexistent account is not answered in
|
||||
|
||||
+46
-111
@@ -18,19 +18,6 @@ import (
|
||||
// computation.
|
||||
const hoursPerDay = 24
|
||||
|
||||
// reapBatchSize is how many expired events one retention transaction
|
||||
// deletes. A transaction holds the event database's write lock, which
|
||||
// the receiver and the delivery workers wait for, so a large prune is
|
||||
// split into transactions each short enough to finish well inside the
|
||||
// busy timeout.
|
||||
const reapBatchSize = 1000
|
||||
|
||||
// reapBatchPause is how long retention waits after one batch before
|
||||
// starting the next. A writer waiting for the write lock checks for it
|
||||
// again after at most 100 ms, so a longer pause lets it in between two
|
||||
// batches instead of only after the whole prune.
|
||||
const reapBatchPause = 200 * time.Millisecond
|
||||
|
||||
// RetentionReaperParams holds the fx dependencies for the
|
||||
// RetentionReaper.
|
||||
type RetentionReaperParams struct {
|
||||
@@ -200,15 +187,13 @@ func (r *RetentionReaper) sweep(ctx context.Context) {
|
||||
continue
|
||||
}
|
||||
|
||||
r.reapWebhook(ctx, wh.ID, wh.RetentionDays)
|
||||
r.reapWebhook(wh.ID, wh.RetentionDays)
|
||||
}
|
||||
}
|
||||
|
||||
// reapWebhook removes every expired event (and its dependents) from a
|
||||
// single webhook's database, or as many as it reaches before ctx is
|
||||
// cancelled.
|
||||
// single webhook's database.
|
||||
func (r *RetentionReaper) reapWebhook(
|
||||
ctx context.Context,
|
||||
webhookID string,
|
||||
retentionDays int,
|
||||
) {
|
||||
@@ -228,7 +213,7 @@ func (r *RetentionReaper) reapWebhook(
|
||||
return
|
||||
}
|
||||
|
||||
deleted, err := reapExpired(ctx, db, cutoff)
|
||||
deleted, err := reapExpired(db, cutoff)
|
||||
if err != nil {
|
||||
r.log.Error(
|
||||
"retention sweep: failed to reap expired events",
|
||||
@@ -280,107 +265,57 @@ func retentionCutoff(
|
||||
), true
|
||||
}
|
||||
|
||||
// reapExpired hard-deletes the events older than cutoff, with their
|
||||
// deliveries and delivery results, reapBatchSize events per
|
||||
// transaction with reapBatchPause between transactions, until none is
|
||||
// left. Once ctx is cancelled it returns after the batch in hand,
|
||||
// leaving the rest to the next sweep, so stopping the app does not
|
||||
// wait for a long prune. It returns the number of events deleted.
|
||||
func reapExpired(
|
||||
ctx context.Context, db *gorm.DB, cutoff time.Time,
|
||||
) (int64, error) {
|
||||
var total int64
|
||||
|
||||
for {
|
||||
var eventIDs []string
|
||||
|
||||
err := db.Transaction(func(tx *gorm.DB) error {
|
||||
err := tx.Unscoped().Model(&Event{}).
|
||||
Where("created_at < ?", cutoff).
|
||||
Limit(reapBatchSize).
|
||||
Pluck("id", &eventIDs).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf("selecting expired events: %w", err)
|
||||
}
|
||||
|
||||
if len(eventIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return deleteEvents(tx, eventIDs)
|
||||
})
|
||||
if err != nil {
|
||||
return total, err
|
||||
}
|
||||
|
||||
total += int64(len(eventIDs))
|
||||
|
||||
if len(eventIDs) < reapBatchSize {
|
||||
return total, nil
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return total, nil
|
||||
case <-time.After(reapBatchPause):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// deleteEvents hard-deletes the given events and, in foreign-key-safe
|
||||
// order before them, their delivery results and deliveries, then adds
|
||||
// what it deleted to the running totals. It runs on reapExpired's
|
||||
// transaction, so the totals change exactly when the rows do. Deletes
|
||||
// are unscoped so rows are physically removed rather than
|
||||
// soft-deleted, reclaiming disk.
|
||||
func deleteEvents(tx *gorm.DB, eventIDs []string) error {
|
||||
// 1. The delivery results of the events' deliveries.
|
||||
err := tx.Unscoped().
|
||||
Where("delivery_id IN (?)", tx.Unscoped().Model(&Delivery{}).
|
||||
// reapExpired hard-deletes, in foreign-key-safe order, the delivery
|
||||
// results, deliveries, and events associated with events older than
|
||||
// cutoff. Deletes are unscoped so rows are physically removed rather
|
||||
// than soft-deleted, reclaiming disk. It returns the number of events
|
||||
// deleted.
|
||||
func reapExpired(db *gorm.DB, cutoff time.Time) (int64, error) {
|
||||
// Fresh subqueries are built per statement to avoid reusing a
|
||||
// mutated builder across executions.
|
||||
expiredEventIDs := func() *gorm.DB {
|
||||
return db.Model(&Event{}).
|
||||
Select("id").
|
||||
Where("event_id IN ?", eventIDs)).
|
||||
Delete(&DeliveryResult{}).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting expired delivery results: %w", err)
|
||||
Where("created_at < ?", cutoff)
|
||||
}
|
||||
expiredDeliveryIDs := func() *gorm.DB {
|
||||
return db.Model(&Delivery{}).
|
||||
Select("id").
|
||||
Where("event_id IN (?)", expiredEventIDs())
|
||||
}
|
||||
|
||||
// 2. The events' deliveries, after counting them, and the failed
|
||||
// ones among them, per target. The status is tested in the select
|
||||
// list rather than the WHERE clause: there, SQLite would read every
|
||||
// failed delivery the webhook has through the status index,
|
||||
// instead of only these through the event_id index.
|
||||
var removed []TargetTotals
|
||||
|
||||
err = tx.Unscoped().Model(&Delivery{}).
|
||||
Select("target_id, count(*) AS deliveries_removed, "+
|
||||
"count(CASE WHEN status = ? THEN 1 END) AS failed_removed",
|
||||
DeliveryStatusFailed).
|
||||
Where("event_id IN ?", eventIDs).
|
||||
Group("target_id").
|
||||
Find(&removed).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf("counting expired deliveries: %w", err)
|
||||
// 1. Delivery results whose delivery belongs to an expired event.
|
||||
res := db.Unscoped().
|
||||
Where("delivery_id IN (?)", expiredDeliveryIDs()).
|
||||
Delete(&DeliveryResult{})
|
||||
if res.Error != nil {
|
||||
return 0, fmt.Errorf(
|
||||
"deleting expired delivery results: %w",
|
||||
res.Error,
|
||||
)
|
||||
}
|
||||
|
||||
err = tx.Unscoped().
|
||||
Where("event_id IN ?", eventIDs).
|
||||
Delete(&Delivery{}).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf("deleting expired deliveries: %w", err)
|
||||
// 2. Deliveries belonging to an expired event.
|
||||
del := db.Unscoped().
|
||||
Where("event_id IN (?)", expiredEventIDs()).
|
||||
Delete(&Delivery{})
|
||||
if del.Error != nil {
|
||||
return 0, fmt.Errorf(
|
||||
"deleting expired deliveries: %w",
|
||||
del.Error,
|
||||
)
|
||||
}
|
||||
|
||||
// 3. The events themselves.
|
||||
ev := tx.Unscoped().Where("id IN ?", eventIDs).Delete(&Event{})
|
||||
// 3. The expired events themselves.
|
||||
ev := db.Unscoped().
|
||||
Where("created_at < ?", cutoff).
|
||||
Delete(&Event{})
|
||||
if ev.Error != nil {
|
||||
return fmt.Errorf("deleting expired events: %w", ev.Error)
|
||||
return 0, fmt.Errorf(
|
||||
"deleting expired events: %w",
|
||||
ev.Error,
|
||||
)
|
||||
}
|
||||
|
||||
for i := range removed {
|
||||
err = AddTargetTotals(tx, removed[i])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return AddEventTotals(tx, EventTotals{EventsRemoved: ev.RowsAffected})
|
||||
return ev.RowsAffected, nil
|
||||
}
|
||||
|
||||
@@ -1,410 +0,0 @@
|
||||
package database_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
)
|
||||
|
||||
// readEventTotals reads a webhook database's row of event totals,
|
||||
// asserting that it has exactly one.
|
||||
func readEventTotals(t *testing.T, db *gorm.DB) database.EventTotals {
|
||||
t.Helper()
|
||||
|
||||
var rows []database.EventTotals
|
||||
|
||||
require.NoError(t, db.Find(&rows).Error)
|
||||
require.Len(t, rows, 1)
|
||||
|
||||
return rows[0]
|
||||
}
|
||||
|
||||
// readTargetTotals reads a webhook database's target totals, keyed by
|
||||
// target.
|
||||
func readTargetTotals(
|
||||
t *testing.T, db *gorm.DB,
|
||||
) map[string]database.TargetTotals {
|
||||
t.Helper()
|
||||
|
||||
var rows []database.TargetTotals
|
||||
|
||||
require.NoError(t, db.Find(&rows).Error)
|
||||
|
||||
byTarget := make(map[string]database.TargetTotals, len(rows))
|
||||
for _, row := range rows {
|
||||
byTarget[row.TargetID] = row
|
||||
}
|
||||
|
||||
return byTarget
|
||||
}
|
||||
|
||||
// TestWebhookDBManager_TotalsSurviveReopen verifies that a new event
|
||||
// database starts with one row of zero event totals and no target
|
||||
// totals, that adding to a target twice adds to the one row, and that
|
||||
// opening the database again keeps everything added.
|
||||
func TestWebhookDBManager_TotalsSurviveReopen(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
mgr, lc := setupTestWebhookDBManager(t)
|
||||
ctx := context.Background()
|
||||
require.NoError(t, lc.Start(ctx))
|
||||
|
||||
defer func() { require.NoError(t, lc.Stop(ctx)) }()
|
||||
|
||||
webhookID := uuid.New().String()
|
||||
|
||||
db, err := mgr.GetDB(webhookID)
|
||||
require.NoError(t, err)
|
||||
|
||||
fresh := readEventTotals(t, db)
|
||||
assert.Equal(t, database.EventTotals{ID: fresh.ID}, fresh)
|
||||
assert.Empty(t, readTargetTotals(t, db))
|
||||
|
||||
first, second := uuid.New().String(), uuid.New().String()
|
||||
|
||||
require.NoError(t, database.AddEventTotals(db, database.EventTotals{
|
||||
Events: 2,
|
||||
}))
|
||||
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
||||
TargetID: first, Deliveries: 2, Delivered: 1,
|
||||
}))
|
||||
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
||||
TargetID: first, Failed: 1,
|
||||
}))
|
||||
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
||||
TargetID: second, Deliveries: 1,
|
||||
}))
|
||||
|
||||
// Drop the cached connection so the next open reopens the file,
|
||||
// as a restart would.
|
||||
require.NoError(t, mgr.CloseAll())
|
||||
|
||||
db, err = mgr.GetDB(webhookID)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, database.EventTotals{ID: fresh.ID, Events: 2},
|
||||
readEventTotals(t, db))
|
||||
assert.Equal(t, map[string]database.TargetTotals{
|
||||
first: {
|
||||
TargetID: first, Deliveries: 2, Delivered: 1, Failed: 1,
|
||||
},
|
||||
second: {TargetID: second, Deliveries: 1},
|
||||
}, readTargetTotals(t, db))
|
||||
}
|
||||
|
||||
// seedExpiredEvents stores count events created at the given time,
|
||||
// each with a delivered delivery to one target and a failed delivery
|
||||
// to the other, and one attempt for each delivery.
|
||||
func seedExpiredEvents(
|
||||
t *testing.T,
|
||||
db *gorm.DB,
|
||||
webhookID string,
|
||||
count int,
|
||||
createdAt time.Time,
|
||||
delivered, failed string,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
events := make([]database.Event, count)
|
||||
deliveries := make([]database.Delivery, 0, 2*count)
|
||||
|
||||
for i := range events {
|
||||
events[i] = database.Event{
|
||||
WebhookID: webhookID,
|
||||
EntrypointID: uuid.New().String(),
|
||||
Method: http.MethodPost,
|
||||
}
|
||||
events[i].ID = uuid.New().String()
|
||||
events[i].CreatedAt = createdAt
|
||||
|
||||
deliveries = append(deliveries,
|
||||
database.Delivery{
|
||||
EventID: events[i].ID,
|
||||
TargetID: delivered,
|
||||
Status: database.DeliveryStatusDelivered,
|
||||
},
|
||||
database.Delivery{
|
||||
EventID: events[i].ID,
|
||||
TargetID: failed,
|
||||
Status: database.DeliveryStatusFailed,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
require.NoError(t, db.CreateInBatches(events, 500).Error)
|
||||
require.NoError(t, db.CreateInBatches(deliveries, 500).Error)
|
||||
|
||||
results := make([]database.DeliveryResult, len(deliveries))
|
||||
for i := range deliveries {
|
||||
results[i] = database.DeliveryResult{
|
||||
DeliveryID: deliveries[i].ID, AttemptNum: 1,
|
||||
}
|
||||
}
|
||||
|
||||
require.NoError(t, db.CreateInBatches(results, 500).Error)
|
||||
}
|
||||
|
||||
// seedBareEvents stores count events created at the given time, with
|
||||
// no deliveries.
|
||||
func seedBareEvents(
|
||||
t *testing.T,
|
||||
db *gorm.DB,
|
||||
webhookID string,
|
||||
count int,
|
||||
createdAt time.Time,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
events := make([]database.Event, count)
|
||||
for i := range events {
|
||||
events[i] = database.Event{
|
||||
WebhookID: webhookID,
|
||||
EntrypointID: uuid.New().String(),
|
||||
Method: http.MethodPost,
|
||||
}
|
||||
events[i].CreatedAt = createdAt
|
||||
}
|
||||
|
||||
require.NoError(t, db.CreateInBatches(events, 500).Error)
|
||||
}
|
||||
|
||||
// TestRetentionReaper_PrunesMoreThanOneBatch verifies that a prune
|
||||
// larger than one transaction's batch removes every expired event with
|
||||
// its deliveries and delivery results, keeps the recent event, and
|
||||
// adds what it removed to the event and target totals, so the totals
|
||||
// within retention match the rows still stored.
|
||||
func TestRetentionReaper_PrunesMoreThanOneBatch(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupRetentionTest(t)
|
||||
|
||||
webhookID := createWebhook(t, env.mainDB.DB(), 30)
|
||||
|
||||
db, err := env.mgr.GetDB(webhookID)
|
||||
require.NoError(t, err)
|
||||
|
||||
expired := database.ExportReapBatchSize + 1
|
||||
delivered, failed := uuid.New().String(), uuid.New().String()
|
||||
seedExpiredEvents(t, db, webhookID, expired,
|
||||
time.Now().Add(-40*24*time.Hour), delivered, failed)
|
||||
|
||||
// One recent event, delivered to the first target.
|
||||
recent := seedEventChain(t, db, webhookID, time.Now())
|
||||
require.NoError(t, db.Model(&database.Delivery{}).
|
||||
Where("id = ?", recent.deliveryID).
|
||||
Update("target_id", delivered).Error)
|
||||
|
||||
// The totals storing those rows would have left.
|
||||
n := int64(expired)
|
||||
require.NoError(t, database.AddEventTotals(db, database.EventTotals{
|
||||
Events: n + 1,
|
||||
}))
|
||||
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
||||
TargetID: delivered, Deliveries: n + 1, Delivered: n + 1,
|
||||
}))
|
||||
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
||||
TargetID: failed, Deliveries: n, Failed: n,
|
||||
}))
|
||||
|
||||
env.reaper.ExportSweep(context.Background())
|
||||
|
||||
// Only the recent event's rows are left.
|
||||
for _, model := range []any{
|
||||
&database.Event{}, &database.Delivery{}, &database.DeliveryResult{},
|
||||
} {
|
||||
var count int64
|
||||
|
||||
require.NoError(t, db.Model(model).Count(&count).Error)
|
||||
assert.Equal(t, int64(1), count, "%T rows left", model)
|
||||
}
|
||||
|
||||
assertChainPresent(t, db, recent)
|
||||
|
||||
eventTotals := readEventTotals(t, db)
|
||||
assert.Equal(t, database.EventTotals{
|
||||
ID: eventTotals.ID, Events: n + 1, EventsRemoved: n,
|
||||
}, eventTotals)
|
||||
|
||||
targetTotals := readTargetTotals(t, db)
|
||||
assert.Equal(t, map[string]database.TargetTotals{
|
||||
delivered: {
|
||||
TargetID: delivered, Deliveries: n + 1, Delivered: n + 1,
|
||||
DeliveriesRemoved: n,
|
||||
},
|
||||
failed: {
|
||||
TargetID: failed, Deliveries: n, Failed: n,
|
||||
DeliveriesRemoved: n, FailedRemoved: n,
|
||||
},
|
||||
}, targetTotals)
|
||||
|
||||
// A sweep with nothing left to remove changes nothing.
|
||||
env.reaper.ExportSweep(context.Background())
|
||||
|
||||
assert.Equal(t, eventTotals, readEventTotals(t, db))
|
||||
assert.Equal(t, targetTotals, readTargetTotals(t, db))
|
||||
}
|
||||
|
||||
// TestRetentionReaper_WriteDuringPruneSucceeds verifies that a prune
|
||||
// of several batches lets other writers in between its batches: an
|
||||
// event stored once the first batch is deleted is stored while expired
|
||||
// events are still left, not only after the prune has finished.
|
||||
func TestRetentionReaper_WriteDuringPruneSucceeds(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupRetentionTest(t)
|
||||
|
||||
webhookID := createWebhook(t, env.mainDB.DB(), 30)
|
||||
|
||||
db, err := env.mgr.GetDB(webhookID)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Three batches of expired events, with nothing else stored: only
|
||||
// the number of batches matters here.
|
||||
expired := 3 * database.ExportReapBatchSize
|
||||
seedBareEvents(t, db, webhookID, expired,
|
||||
time.Now().Add(-40*24*time.Hour))
|
||||
|
||||
cutoff := time.Now().Add(-30 * 24 * time.Hour)
|
||||
countExpired := func() int64 {
|
||||
var count int64
|
||||
|
||||
require.NoError(t, db.Model(&database.Event{}).
|
||||
Where("created_at < ?", cutoff).
|
||||
Count(&count).Error)
|
||||
|
||||
return count
|
||||
}
|
||||
|
||||
pruned := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
defer close(pruned)
|
||||
|
||||
env.reaper.ExportSweep(context.Background())
|
||||
}()
|
||||
|
||||
t.Cleanup(func() { <-pruned })
|
||||
|
||||
// Every stored event is expired until the write below.
|
||||
require.Eventually(t, func() bool {
|
||||
var count int64
|
||||
|
||||
err := db.Model(&database.Event{}).Count(&count).Error
|
||||
|
||||
return err == nil && count < int64(expired)
|
||||
}, 10*time.Second, 10*time.Millisecond)
|
||||
|
||||
event := &database.Event{
|
||||
WebhookID: webhookID,
|
||||
EntrypointID: uuid.New().String(),
|
||||
Method: http.MethodPost,
|
||||
}
|
||||
require.NoError(t, db.Create(event).Error)
|
||||
|
||||
assert.Positive(t, countExpired(),
|
||||
"the event was stored only after the whole prune")
|
||||
|
||||
<-pruned
|
||||
|
||||
assert.Zero(t, countExpired())
|
||||
|
||||
var stored database.Event
|
||||
|
||||
require.NoError(t, db.First(&stored, "id = ?", event.ID).Error)
|
||||
}
|
||||
|
||||
// TestRetentionReaper_StopDuringPruneLeavesTheRest verifies that
|
||||
// stopping the reaper during a prune of several batches returns
|
||||
// between two batches, well inside the stop timeout, leaving the
|
||||
// remaining expired events for the next sweep, and that the totals
|
||||
// match the rows left.
|
||||
func TestRetentionReaper_StopDuringPruneLeavesTheRest(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupRetentionTest(t)
|
||||
|
||||
webhookID := createWebhook(t, env.mainDB.DB(), 30)
|
||||
|
||||
db, err := env.mgr.GetDB(webhookID)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Two batches and one more of expired events, a few of them with a
|
||||
// delivered and a failed delivery for the target totals to count.
|
||||
// Most carry nothing else, to keep the test quick.
|
||||
const withDeliveries = 10
|
||||
|
||||
expiredAt := time.Now().Add(-40 * 24 * time.Hour)
|
||||
delivered, failed := uuid.New().String(), uuid.New().String()
|
||||
seedExpiredEvents(t, db, webhookID, withDeliveries, expiredAt,
|
||||
delivered, failed)
|
||||
seedBareEvents(t, db, webhookID,
|
||||
2*database.ExportReapBatchSize+1-withDeliveries, expiredAt)
|
||||
|
||||
n := int64(2*database.ExportReapBatchSize + 1)
|
||||
require.NoError(t, database.AddEventTotals(db, database.EventTotals{
|
||||
Events: n,
|
||||
}))
|
||||
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
||||
TargetID: delivered, Deliveries: withDeliveries,
|
||||
Delivered: withDeliveries,
|
||||
}))
|
||||
require.NoError(t, database.AddTargetTotals(db, database.TargetTotals{
|
||||
TargetID: failed, Deliveries: withDeliveries,
|
||||
Failed: withDeliveries,
|
||||
}))
|
||||
|
||||
env.reaper.ExportSetInterval(time.Millisecond)
|
||||
env.reaper.ExportStart()
|
||||
|
||||
// Stop once the first batch is deleted. The stop lands in the pause
|
||||
// after it, or at worst during the second batch, so at least the
|
||||
// last event is left.
|
||||
require.Eventually(t, func() bool {
|
||||
var count int64
|
||||
|
||||
err := db.Model(&database.Event{}).Count(&count).Error
|
||||
|
||||
return err == nil && count < n
|
||||
}, 10*time.Second, 10*time.Millisecond)
|
||||
|
||||
// The app's stop timeout.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
require.NoError(t, env.reaper.ExportStop(ctx))
|
||||
|
||||
var events int64
|
||||
|
||||
require.NoError(t, db.Model(&database.Event{}).Count(&events).Error)
|
||||
assert.Positive(t, events, "the stop waited for the whole prune")
|
||||
|
||||
eventTotals := readEventTotals(t, db)
|
||||
assert.Equal(t, events, eventTotals.Events-eventTotals.EventsRemoved)
|
||||
|
||||
targetTotals := readTargetTotals(t, db)
|
||||
require.Len(t, targetTotals, 2)
|
||||
|
||||
for target, totals := range targetTotals {
|
||||
var deliveries, failures int64
|
||||
|
||||
require.NoError(t, db.Model(&database.Delivery{}).
|
||||
Where("target_id = ?", target).
|
||||
Count(&deliveries).Error)
|
||||
require.NoError(t, db.Model(&database.Delivery{}).
|
||||
Where("target_id = ? AND status = ?",
|
||||
target, database.DeliveryStatusFailed).
|
||||
Count(&failures).Error)
|
||||
|
||||
assert.Equal(t, deliveries,
|
||||
totals.Deliveries-totals.DeliveriesRemoved, target)
|
||||
assert.Equal(t, failures, totals.Failed-totals.FailedRemoved,
|
||||
target)
|
||||
}
|
||||
}
|
||||
@@ -35,8 +35,7 @@ var errInvalidCachedDBType = errors.New(
|
||||
|
||||
// WebhookDBManager manages per-webhook SQLite database files
|
||||
// for event storage. Each webhook gets its own dedicated
|
||||
// database containing Events, Deliveries, DeliveryResults and the
|
||||
// running totals of them (EventTotals, TargetTotals).
|
||||
// database containing Events, Deliveries, and DeliveryResults.
|
||||
// Database connections are opened lazily and cached.
|
||||
type WebhookDBManager struct {
|
||||
dataDir string
|
||||
@@ -296,7 +295,6 @@ func (m *WebhookDBManager) openDB(
|
||||
// Run migrations for event-tier models only
|
||||
err = db.AutoMigrate(
|
||||
&Event{}, &Delivery{}, &DeliveryResult{},
|
||||
&EventTotals{}, &TargetTotals{},
|
||||
)
|
||||
if err != nil {
|
||||
_ = sqlDB.Close()
|
||||
@@ -307,18 +305,6 @@ func (m *WebhookDBManager) openDB(
|
||||
)
|
||||
}
|
||||
|
||||
// A new database gets its row of event totals, all zero. Target
|
||||
// totals rows are created by the first delivery to each target.
|
||||
err = db.FirstOrCreate(&EventTotals{}).Error
|
||||
if err != nil {
|
||||
_ = sqlDB.Close()
|
||||
|
||||
return nil, fmt.Errorf(
|
||||
"creating event totals for webhook database %s: %w",
|
||||
webhookID, err,
|
||||
)
|
||||
}
|
||||
|
||||
m.log.Info(
|
||||
"opened per-webhook database",
|
||||
"webhook_id", webhookID,
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"time"
|
||||
|
||||
"go.uber.org/fx"
|
||||
"gorm.io/gorm"
|
||||
"sneak.berlin/go/webhooker/internal/config"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/lifecycle"
|
||||
@@ -26,14 +25,14 @@ type ArchiveSweeperParams struct {
|
||||
Logger *logger.Logger
|
||||
}
|
||||
|
||||
// ArchiveSweeper periodically prunes expired rows from the
|
||||
// archive databases of database targets that carry a positive
|
||||
// expiry.
|
||||
// ArchiveSweeper periodically prunes expired rows from
|
||||
// per-webhook archive databases whose database target carries a
|
||||
// positive expiry.
|
||||
//
|
||||
// Without it, pruning happens only when an archive is
|
||||
// (re)opened, and archives are only ever reopened by writes: an
|
||||
// archive whose target has stopped receiving events would keep
|
||||
// its expired rows forever. The sweep closes
|
||||
// archive belonging to a webhook that has stopped receiving
|
||||
// events would keep its expired rows forever. The sweep closes
|
||||
// that gap without changing anything for archives whose expiry
|
||||
// is unset or "never".
|
||||
//
|
||||
@@ -156,7 +155,7 @@ func (s *ArchiveSweeper) run(ctx context.Context) {
|
||||
// soft-deleted along with it, so GORM's default scope already
|
||||
// excludes them.
|
||||
//
|
||||
// A failure for one target is logged and the sweep continues,
|
||||
// A failure for one webhook is logged and the sweep continues,
|
||||
// matching how the write path already treats a prune error as
|
||||
// non-fatal.
|
||||
func (s *ArchiveSweeper) sweep(ctx context.Context) {
|
||||
@@ -211,20 +210,19 @@ func (s *ArchiveSweeper) sweepTarget(target *database.Target) {
|
||||
return
|
||||
}
|
||||
|
||||
err = s.eng.dbTarget.sweepArchive(target.ID, expiry)
|
||||
err = s.eng.dbTarget.sweepWebhook(target.WebhookID, expiry)
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// A writer evicted, or a target row gone, underneath the sweep
|
||||
// means the operator deleted the target or its webhook while
|
||||
// the sweep was walking the target list. That is an ordinary
|
||||
// A writer evicted underneath the sweep means the operator
|
||||
// deleted the webhook (or its last database target) while the
|
||||
// sweep was walking the target list. That is an ordinary
|
||||
// interleaving, not a failure, so it must not produce an
|
||||
// error line.
|
||||
if errors.Is(err, errArchiveWriterEvicted) ||
|
||||
errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
if errors.Is(err, errArchiveWriterEvicted) {
|
||||
s.log.Debug(
|
||||
"archive sweep: target deleted mid-sweep",
|
||||
"archive sweep: writer evicted mid-sweep",
|
||||
"webhook_id", target.WebhookID,
|
||||
"target_id", target.ID,
|
||||
)
|
||||
|
||||
@@ -34,23 +34,18 @@ const (
|
||||
sweepConcurrentWrites = 20
|
||||
)
|
||||
|
||||
// archiveTestWebhookName is the name of every webhook
|
||||
// seedDatabaseTarget creates. It is not safe in a file name as it
|
||||
// stands, so every archive test goes through archiveNamePart.
|
||||
const archiveTestWebhookName = "Sweep Test!"
|
||||
|
||||
// archiveEnv bundles the pieces an archive test drives: a main
|
||||
// configuration database holding webhooks and targets, a delivery
|
||||
// engine owning the archive writer registry, the archive sweeper,
|
||||
// and the data directory the archive files live in.
|
||||
type archiveEnv struct {
|
||||
// sweeperEnv bundles the pieces an archive sweep test drives:
|
||||
// a main configuration database holding webhooks and targets, a
|
||||
// delivery engine owning the archive writer registry, and the
|
||||
// data directory the archive files live in.
|
||||
type sweeperEnv struct {
|
||||
sweeper *delivery.ArchiveSweeper
|
||||
eng *delivery.Engine
|
||||
mainDB *database.Database
|
||||
dataDir string
|
||||
}
|
||||
|
||||
func setupArchiveTest(t *testing.T) *archiveEnv {
|
||||
func setupSweeperTest(t *testing.T) *sweeperEnv {
|
||||
t.Helper()
|
||||
|
||||
dataDir := t.TempDir()
|
||||
@@ -83,7 +78,7 @@ func setupArchiveTest(t *testing.T) *archiveEnv {
|
||||
1,
|
||||
)
|
||||
|
||||
return &archiveEnv{
|
||||
return &sweeperEnv{
|
||||
sweeper: delivery.NewTestArchiveSweeper(
|
||||
mainDB, eng, log,
|
||||
),
|
||||
@@ -93,27 +88,25 @@ func setupArchiveTest(t *testing.T) *archiveEnv {
|
||||
}
|
||||
}
|
||||
|
||||
// archivePath returns where the engine keeps a database target's
|
||||
// archive file, for the names seedDatabaseTarget gave it.
|
||||
func (env *archiveEnv) archivePath(tgt *database.Target) string {
|
||||
// archivePath returns where the engine keeps a webhook's
|
||||
// archive file.
|
||||
func (env *sweeperEnv) archivePath(webhookID string) string {
|
||||
return filepath.Join(
|
||||
env.dataDir,
|
||||
delivery.ArchiveFileName(
|
||||
archiveTestWebhookName, tgt.Name, tgt.ID,
|
||||
),
|
||||
env.dataDir, fmt.Sprintf("archive-%s.db", webhookID),
|
||||
)
|
||||
}
|
||||
|
||||
// seedDatabaseTarget creates a webhook with one database target
|
||||
// carrying the given target config JSON, and returns the target.
|
||||
func (env *archiveEnv) seedDatabaseTarget(
|
||||
// carrying the given target config JSON, and returns the
|
||||
// webhook id.
|
||||
func (env *sweeperEnv) seedDatabaseTarget(
|
||||
t *testing.T, configJSON string,
|
||||
) *database.Target {
|
||||
) string {
|
||||
t.Helper()
|
||||
|
||||
wh := &database.Webhook{
|
||||
UserID: uuid.New().String(),
|
||||
Name: archiveTestWebhookName,
|
||||
Name: "sweep-test",
|
||||
}
|
||||
require.NoError(
|
||||
t,
|
||||
@@ -122,19 +115,9 @@ func (env *archiveEnv) seedDatabaseTarget(
|
||||
Create(wh).Error,
|
||||
)
|
||||
|
||||
return env.addDatabaseTarget(t, wh.ID, configJSON)
|
||||
}
|
||||
|
||||
// addDatabaseTarget creates one more database target on an
|
||||
// existing webhook and returns it.
|
||||
func (env *archiveEnv) addDatabaseTarget(
|
||||
t *testing.T, webhookID, configJSON string,
|
||||
) *database.Target {
|
||||
t.Helper()
|
||||
|
||||
tgt := &database.Target{
|
||||
WebhookID: webhookID,
|
||||
Name: "Archive",
|
||||
WebhookID: wh.ID,
|
||||
Name: "archive",
|
||||
Type: database.TargetTypeDatabase,
|
||||
Active: true,
|
||||
Config: configJSON,
|
||||
@@ -146,19 +129,19 @@ func (env *archiveEnv) addDatabaseTarget(
|
||||
Create(tgt).Error,
|
||||
)
|
||||
|
||||
return tgt
|
||||
return wh.ID
|
||||
}
|
||||
|
||||
// seedArchiveRows creates the archive file for a target and
|
||||
// seedArchiveRows creates the archive file for a webhook and
|
||||
// inserts one row per supplied archived-at timestamp, returning
|
||||
// the archive path. The handle is closed before returning, so
|
||||
// the archive is idle exactly as it would be with no traffic.
|
||||
func (env *archiveEnv) seedArchiveRows(
|
||||
t *testing.T, tgt *database.Target, archivedAt ...time.Time,
|
||||
func (env *sweeperEnv) seedArchiveRows(
|
||||
t *testing.T, webhookID string, archivedAt ...time.Time,
|
||||
) string {
|
||||
t.Helper()
|
||||
|
||||
path := env.archivePath(tgt)
|
||||
path := env.archivePath(webhookID)
|
||||
|
||||
sqlDB, err := sql.Open(
|
||||
"sqlite", fmt.Sprintf("file:%s?mode=rwc", path),
|
||||
@@ -177,7 +160,7 @@ func (env *archiveEnv) seedArchiveRows(
|
||||
for i, at := range archivedAt {
|
||||
row := delivery.ExportArchivedEvent{
|
||||
EventID: fmt.Sprintf("ev-%d", i),
|
||||
WebhookID: tgt.WebhookID,
|
||||
WebhookID: webhookID,
|
||||
Method: http.MethodPost,
|
||||
Body: `{"seeded":true}`,
|
||||
ArchivedAt: at,
|
||||
@@ -260,13 +243,13 @@ func TestArchiveSweeper_LoopOutlivesStartHookContext(
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
env := setupSweeperTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
|
||||
now := time.Now()
|
||||
path := env.seedArchiveRows(
|
||||
t, tgt,
|
||||
t, webhookID,
|
||||
now.Add(-48*time.Hour),
|
||||
now.Add(-time.Minute),
|
||||
)
|
||||
@@ -304,60 +287,60 @@ func TestArchiveSweeper_LoopOutlivesStartHookContext(
|
||||
}
|
||||
|
||||
// TestArchiveSweep_DoesNotResurrectEvictedWriter covers the
|
||||
// interleaving where a sweep tick has already listed a target
|
||||
// when the target is deleted and its writer evicted. The sweep
|
||||
// must not put a writer back into the registry: nothing would
|
||||
// ever evict it again, which is precisely the leak this change
|
||||
// exists to close.
|
||||
// interleaving where a sweep tick has already listed a webhook's
|
||||
// target when the webhook is deleted and its writer evicted. The
|
||||
// sweep must not put a writer back into the registry: nothing
|
||||
// would ever evict it again, which is precisely the leak this
|
||||
// change exists to close.
|
||||
func TestArchiveSweep_DoesNotResurrectEvictedWriter(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
env := setupSweeperTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
env.seedArchiveRows(
|
||||
t, tgt, time.Now().Add(-48*time.Hour),
|
||||
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||
)
|
||||
|
||||
// Prime the registry the way a delivery would, then evict as
|
||||
// the deletion path does. The target row is deliberately left
|
||||
// in place: this is the tick that listed the target before
|
||||
// in place: this is the tick that listed the webhook before
|
||||
// the deletion committed.
|
||||
_, err := env.eng.ExportEnsureArchiveWriter(tgt.ID)
|
||||
_, err := env.eng.ExportEnsureArchiveWriter(webhookID)
|
||||
require.NoError(t, err)
|
||||
|
||||
env.eng.EvictTarget(tgt.ID)
|
||||
require.False(t, env.eng.ExportHasArchiveWriter(tgt.ID))
|
||||
env.eng.EvictWebhook(webhookID)
|
||||
require.False(t, env.eng.ExportHasArchiveWriter(webhookID))
|
||||
|
||||
env.sweeper.ExportSweep(context.Background())
|
||||
|
||||
assert.False(
|
||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||
"a sweep must never re-register a writer for a target "+
|
||||
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||
"a sweep must never re-register a writer for a webhook "+
|
||||
"whose registry entry has already been released",
|
||||
)
|
||||
}
|
||||
|
||||
// TestArchiveSweep_LeavesNoRegistryEntry states the same
|
||||
// invariant in its general form: sweeping an archive whose
|
||||
// target has no cached writer must not leave one behind, so the
|
||||
// webhook has no cached writer must not leave one behind, so the
|
||||
// registry keeps holding only writers a delivery created and an
|
||||
// eviction can reach.
|
||||
func TestArchiveSweep_LeavesNoRegistryEntry(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
env := setupSweeperTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
path := env.seedArchiveRows(
|
||||
t, tgt,
|
||||
t, webhookID,
|
||||
time.Now().Add(-48*time.Hour),
|
||||
time.Now().Add(-time.Minute),
|
||||
)
|
||||
|
||||
require.False(t, env.eng.ExportHasArchiveWriter(tgt.ID))
|
||||
require.False(t, env.eng.ExportHasArchiveWriter(webhookID))
|
||||
|
||||
env.sweeper.ExportSweep(context.Background())
|
||||
|
||||
@@ -366,7 +349,7 @@ func TestArchiveSweep_LeavesNoRegistryEntry(t *testing.T) {
|
||||
"the sweep must still prune an idle archive",
|
||||
)
|
||||
assert.False(
|
||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||
"the sweep must release the registry entry it created",
|
||||
)
|
||||
}
|
||||
@@ -381,31 +364,34 @@ func TestArchiveSweep_KeepsWriterAdoptedByDelivery(
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
env := setupSweeperTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
env.seedArchiveRows(
|
||||
t, tgt, time.Now().Add(-48*time.Hour),
|
||||
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||
)
|
||||
|
||||
webhookDB := testWebhookDB(t)
|
||||
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
|
||||
event.WebhookID = webhookID
|
||||
d := seedDatabaseTargetDelivery(
|
||||
t, webhookDB, event, `{"expiry":"1h"}`,
|
||||
)
|
||||
|
||||
env.sweeper.ExportSweep(context.Background())
|
||||
require.False(t, env.eng.ExportHasArchiveWriter(tgt.ID))
|
||||
require.False(t, env.eng.ExportHasArchiveWriter(webhookID))
|
||||
|
||||
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||
|
||||
assert.True(
|
||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||
"a delivery's writer must stay registered",
|
||||
)
|
||||
|
||||
env.sweeper.ExportSweep(context.Background())
|
||||
|
||||
assert.True(
|
||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||
"a sweep must not drop a writer a delivery owns",
|
||||
)
|
||||
}
|
||||
@@ -437,15 +423,15 @@ func TestArchiveSweep_KeepsWriterAdoptedDuringSweep(
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
env := setupSweeperTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
env.seedArchiveRows(
|
||||
t, tgt, time.Now().Add(-48*time.Hour),
|
||||
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||
)
|
||||
|
||||
sweepWriter, created, err := env.eng.ExportSweepWriterFor(
|
||||
tgt.ID,
|
||||
webhookID,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.True(
|
||||
@@ -456,34 +442,37 @@ func TestArchiveSweep_KeepsWriterAdoptedDuringSweep(
|
||||
// The delivery lands mid-sweep and adopts the entry.
|
||||
webhookDB := testWebhookDB(t)
|
||||
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
|
||||
event.WebhookID = webhookID
|
||||
d := seedDatabaseTargetDelivery(
|
||||
t, webhookDB, event, `{"expiry":"1h"}`,
|
||||
)
|
||||
|
||||
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||
|
||||
adopted := env.eng.ExportArchiveWriterFor(tgt.ID)
|
||||
adopted := env.eng.ExportArchiveWriterFor(webhookID)
|
||||
require.NotNil(t, adopted)
|
||||
require.True(
|
||||
t, sweepWriter.Same(adopted),
|
||||
"the delivery must have adopted the sweep's writer",
|
||||
)
|
||||
require.True(
|
||||
t, env.eng.ExportArchiveHandleOpen(tgt.ID),
|
||||
t, env.eng.ExportArchiveHandleOpen(webhookID),
|
||||
"the delivery leaves the archive handle open",
|
||||
)
|
||||
|
||||
// The sweep finishes.
|
||||
env.eng.ExportReleaseSweepWriter(tgt.ID, sweepWriter)
|
||||
env.eng.ExportReleaseSweepWriter(webhookID, sweepWriter)
|
||||
|
||||
require.True(
|
||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||
"a writer adopted by a delivery during a sweep must "+
|
||||
"stay registered, or its open handle is unreachable",
|
||||
)
|
||||
|
||||
env.eng.EvictTarget(tgt.ID)
|
||||
env.eng.EvictWebhook(webhookID)
|
||||
|
||||
assert.False(
|
||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||
"the adopted writer must still be evictable",
|
||||
)
|
||||
assert.False(
|
||||
@@ -492,34 +481,34 @@ func TestArchiveSweep_KeepsWriterAdoptedDuringSweep(
|
||||
)
|
||||
}
|
||||
|
||||
// TestArchiveSweep_ContinuesAfterPerTargetFailure proves a
|
||||
// failure for one target does not abort the sweep for the
|
||||
// TestArchiveSweep_ContinuesAfterPerWebhookFailure proves a
|
||||
// failure for one webhook does not abort the sweep for the
|
||||
// others: an unparseable expiry and an unreadable archive both
|
||||
// have to be logged and stepped over.
|
||||
func TestArchiveSweep_ContinuesAfterPerTargetFailure(
|
||||
func TestArchiveSweep_ContinuesAfterPerWebhookFailure(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
env := setupSweeperTest(t)
|
||||
|
||||
// Seeded first so the sweep reaches them before the healthy
|
||||
// target: targets come back in insertion order.
|
||||
badConfig := env.seedDatabaseTarget(t, `{"expiry":"!!!"}`)
|
||||
// webhook: targets come back in insertion order.
|
||||
badConfigID := env.seedDatabaseTarget(t, `{"expiry":"!!!"}`)
|
||||
env.seedArchiveRows(
|
||||
t, badConfig, time.Now().Add(-48*time.Hour),
|
||||
t, badConfigID, time.Now().Add(-48*time.Hour),
|
||||
)
|
||||
|
||||
corrupt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
corruptID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
require.NoError(t, os.WriteFile(
|
||||
env.archivePath(corrupt),
|
||||
env.archivePath(corruptID),
|
||||
[]byte("this is not a sqlite database"),
|
||||
0o600,
|
||||
))
|
||||
|
||||
healthy := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
healthyID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
healthyPath := env.seedArchiveRows(
|
||||
t, healthy,
|
||||
t, healthyID,
|
||||
time.Now().Add(-48*time.Hour),
|
||||
time.Now().Add(-time.Minute),
|
||||
)
|
||||
@@ -529,14 +518,14 @@ func TestArchiveSweep_ContinuesAfterPerTargetFailure(
|
||||
assert.Equal(
|
||||
t, []string{sweepRowNew},
|
||||
archivedEventIDs(t, healthyPath),
|
||||
"a failure for an earlier target must not stop the "+
|
||||
"a failure for an earlier webhook must not stop the "+
|
||||
"sweep from pruning the ones after it",
|
||||
)
|
||||
}
|
||||
|
||||
// TestArchiveSweep_OpenExistingDoesNotCreateFile pins the second
|
||||
// of the two no-create guards. The first is the stat in
|
||||
// sweepExpired; this one is the SQLite open mode, which is what
|
||||
// sweepWebhook; this one is the SQLite open mode, which is what
|
||||
// protects the window between that stat and the open. Flipping
|
||||
// the sweep's mode to create-if-missing makes this fail.
|
||||
func TestArchiveSweep_OpenExistingDoesNotCreateFile(
|
||||
@@ -572,13 +561,13 @@ func TestArchiveSweep_OpenExistingDoesNotCreateFile(
|
||||
func TestArchiveSweep_PrunesIdleArchive(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
env := setupSweeperTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
|
||||
now := time.Now()
|
||||
path := env.seedArchiveRows(
|
||||
t, tgt,
|
||||
t, webhookID,
|
||||
now.Add(-48*time.Hour),
|
||||
now.Add(-time.Minute),
|
||||
)
|
||||
@@ -611,11 +600,11 @@ func TestArchiveSweep_PrunesIdleArchive(t *testing.T) {
|
||||
func TestArchiveSweep_LeavesArchiveClosed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
env := setupSweeperTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
path := env.seedArchiveRows(
|
||||
t, tgt, time.Now().Add(-48*time.Hour),
|
||||
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||
)
|
||||
|
||||
w := delivery.NewExportArchiveWriter(
|
||||
@@ -651,32 +640,35 @@ func TestArchiveSweep_ClosesHandleOfRegisteredWriter(
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
env := setupSweeperTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
env.seedArchiveRows(
|
||||
t, tgt, time.Now().Add(-48*time.Hour),
|
||||
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||
)
|
||||
|
||||
webhookDB := testWebhookDB(t)
|
||||
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
|
||||
event.WebhookID = webhookID
|
||||
d := seedDatabaseTargetDelivery(
|
||||
t, webhookDB, event, `{"expiry":"1h"}`,
|
||||
)
|
||||
|
||||
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||
|
||||
require.True(
|
||||
t, env.eng.ExportArchiveHandleOpen(tgt.ID),
|
||||
t, env.eng.ExportArchiveHandleOpen(webhookID),
|
||||
"the delivery must leave the archive handle open",
|
||||
)
|
||||
|
||||
env.sweeper.ExportSweep(context.Background())
|
||||
|
||||
require.True(
|
||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||
"the delivery's registry entry must survive the sweep",
|
||||
)
|
||||
assert.False(
|
||||
t, env.eng.ExportArchiveHandleOpen(tgt.ID),
|
||||
t, env.eng.ExportArchiveHandleOpen(webhookID),
|
||||
"the sweep must leave the archive closed",
|
||||
)
|
||||
}
|
||||
@@ -692,11 +684,11 @@ func TestArchiveSweep_NeverExpiryUntouched(t *testing.T) {
|
||||
`{"expiry":""}`,
|
||||
"",
|
||||
} {
|
||||
env := setupArchiveTest(t)
|
||||
env := setupSweeperTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, configJSON)
|
||||
webhookID := env.seedDatabaseTarget(t, configJSON)
|
||||
path := env.seedArchiveRows(
|
||||
t, tgt,
|
||||
t, webhookID,
|
||||
time.Now().Add(-10000*time.Hour),
|
||||
)
|
||||
|
||||
@@ -707,7 +699,7 @@ func TestArchiveSweep_NeverExpiryUntouched(t *testing.T) {
|
||||
"config %q must keep rows forever", configJSON,
|
||||
)
|
||||
assert.False(
|
||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||
t, env.eng.ExportHasArchiveWriter(webhookID),
|
||||
"config %q must leave no registry entry behind",
|
||||
configJSON,
|
||||
)
|
||||
@@ -730,10 +722,10 @@ func TestArchiveSweep_NeverExpirySkipsBeforeOpening(
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
env := setupSweeperTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"never"}`)
|
||||
path := env.archivePath(tgt)
|
||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"never"}`)
|
||||
path := env.archivePath(webhookID)
|
||||
|
||||
seedUnmigratedArchive(t, path)
|
||||
require.False(t, archiveTableExists(t, path))
|
||||
@@ -776,16 +768,16 @@ func archiveTableExists(t *testing.T, path string) bool {
|
||||
}
|
||||
|
||||
// TestArchiveSweep_DoesNotCreateArchiveFile proves the sweep
|
||||
// never conjures an archive: a database target that has never
|
||||
// received an event must still have no archive file (nor SQLite
|
||||
// sidecar) after a sweep, and no registry entry either.
|
||||
// never conjures an archive: a webhook with a database target
|
||||
// that has never received an event must still have no archive
|
||||
// file (nor SQLite sidecar) after a sweep.
|
||||
func TestArchiveSweep_DoesNotCreateArchiveFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
env := setupSweeperTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
path := env.archivePath(tgt)
|
||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
path := env.archivePath(webhookID)
|
||||
|
||||
require.NoFileExists(t, path)
|
||||
|
||||
@@ -797,11 +789,6 @@ func TestArchiveSweep_DoesNotCreateArchiveFile(t *testing.T) {
|
||||
"the sweep must not create an archive file",
|
||||
)
|
||||
}
|
||||
|
||||
assert.False(
|
||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||
"the sweep must leave no registry entry behind",
|
||||
)
|
||||
}
|
||||
|
||||
// TestArchiveSweep_DoesNotCreateAfterWriterExists covers the
|
||||
@@ -813,11 +800,11 @@ func TestArchiveSweep_DoesNotCreateAfterWriterExists(
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
env := setupSweeperTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
|
||||
path, err := env.eng.ExportEnsureArchiveWriter(tgt.ID)
|
||||
path, err := env.eng.ExportEnsureArchiveWriter(webhookID)
|
||||
require.NoError(t, err)
|
||||
require.NoFileExists(t, path)
|
||||
|
||||
@@ -832,17 +819,17 @@ func TestArchiveSweep_DoesNotCreateAfterWriterExists(
|
||||
func TestArchiveSweep_SkipsDeletedWebhookTargets(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
env := setupSweeperTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
path := env.seedArchiveRows(
|
||||
t, tgt, time.Now().Add(-48*time.Hour),
|
||||
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||
)
|
||||
|
||||
require.NoError(
|
||||
t,
|
||||
env.mainDB.DB().
|
||||
Where("webhook_id = ?", tgt.WebhookID).
|
||||
Where("webhook_id = ?", webhookID).
|
||||
Delete(&database.Target{}).Error,
|
||||
)
|
||||
|
||||
@@ -855,14 +842,14 @@ func TestArchiveSweep_SkipsDeletedWebhookTargets(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestArchiveSweep_ConcurrentWrites proves the sweep serialises
|
||||
// against writes through the target's writer mutex. Run under
|
||||
// -race, an unsynchronised sweep would be caught here.
|
||||
// against writes through the per-webhook writer mutex. Run
|
||||
// under -race, an unsynchronised sweep would be caught here.
|
||||
func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
env := setupSweeperTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
|
||||
webhookDB := testWebhookDB(t)
|
||||
|
||||
@@ -875,10 +862,13 @@ func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
||||
|
||||
for range sweepConcurrentWrites {
|
||||
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||
event.WebhookID = webhookID
|
||||
|
||||
deliveries = append(
|
||||
deliveries,
|
||||
seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
|
||||
seedDatabaseTargetDelivery(
|
||||
t, webhookDB, event, `{"expiry":"1h"}`,
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -904,7 +894,7 @@ func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
||||
|
||||
wg.Wait()
|
||||
|
||||
assert.FileExists(t, env.archivePath(tgt))
|
||||
assert.FileExists(t, env.archivePath(webhookID))
|
||||
}
|
||||
|
||||
// TestArchiveSweeper_StopsCleanly proves the background loop
|
||||
@@ -912,11 +902,11 @@ func TestArchiveSweep_ConcurrentWrites(t *testing.T) {
|
||||
func TestArchiveSweeper_StopsCleanly(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
env := setupSweeperTest(t)
|
||||
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
webhookID := env.seedDatabaseTarget(t, `{"expiry":"1h"}`)
|
||||
env.seedArchiveRows(
|
||||
t, tgt, time.Now().Add(-48*time.Hour),
|
||||
t, webhookID, time.Now().Add(-48*time.Hour),
|
||||
)
|
||||
|
||||
env.sweeper.ExportSetInterval(time.Millisecond)
|
||||
@@ -940,7 +930,7 @@ func TestArchiveSweeper_StopHookHonoursStopTimeout(
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
env := setupSweeperTest(t)
|
||||
|
||||
lc := &recordingLifecycle{}
|
||||
env.sweeper.ExportRegisterHooks(lc)
|
||||
|
||||
@@ -1,177 +0,0 @@
|
||||
package delivery_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
)
|
||||
|
||||
// targetTotals reads one target's totals from a webhook database, all
|
||||
// zero when it has no row.
|
||||
func targetTotals(
|
||||
t *testing.T, db *gorm.DB, targetID string,
|
||||
) database.TargetTotals {
|
||||
t.Helper()
|
||||
|
||||
var rows []database.TargetTotals
|
||||
|
||||
require.NoError(t, db.Where("target_id = ?", targetID).
|
||||
Find(&rows).Error)
|
||||
|
||||
if len(rows) == 0 {
|
||||
return database.TargetTotals{TargetID: targetID}
|
||||
}
|
||||
|
||||
return rows[0]
|
||||
}
|
||||
|
||||
// TestUpdateDeliveryStatus_FinishTimeAndTargetTotals pins what a status
|
||||
// write records for the webhook page's statistics: the time a delivery
|
||||
// finished, set only when it becomes delivered or failed, and one more
|
||||
// on its target's delivered or failed total.
|
||||
func TestUpdateDeliveryStatus_FinishTimeAndTargetTotals(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
status database.DeliveryStatus
|
||||
finished bool
|
||||
delivered int64
|
||||
failed int64
|
||||
}{
|
||||
{database.DeliveryStatusRetrying, false, 0, 0},
|
||||
{database.DeliveryStatusDelivered, true, 1, 0},
|
||||
{database.DeliveryStatusFailed, true, 0, 1},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(string(tt.status), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := testWebhookDB(t)
|
||||
e := testEngine(t, 1)
|
||||
event := seedEvent(t, db, `{}`)
|
||||
targetID := uuid.New().String()
|
||||
d := seedDelivery(
|
||||
t, db, event.ID, targetID,
|
||||
database.DeliveryStatusPending,
|
||||
)
|
||||
|
||||
before := time.Now()
|
||||
|
||||
require.NoError(t, e.ExportUpdateDeliveryStatus(
|
||||
db, &d, tt.status,
|
||||
))
|
||||
|
||||
var stored database.Delivery
|
||||
|
||||
require.NoError(t, db.First(&stored, "id = ?", d.ID).Error)
|
||||
assert.Equal(t, tt.status, stored.Status)
|
||||
|
||||
if tt.finished {
|
||||
require.NotNil(t, stored.FinishedAt)
|
||||
assert.False(t, stored.FinishedAt.Before(before))
|
||||
} else {
|
||||
assert.Nil(t, stored.FinishedAt)
|
||||
}
|
||||
|
||||
assert.Equal(t, database.TargetTotals{
|
||||
TargetID: targetID,
|
||||
Delivered: tt.delivered,
|
||||
Failed: tt.failed,
|
||||
}, targetTotals(t, db, targetID))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestUpdateDeliveryStatus_DeletedDeliveryIsNotCounted covers a
|
||||
// delivery retention deleted while the engine still held it. Failing
|
||||
// it afterwards writes no row, so it adds no failure either: retention
|
||||
// has already counted what it removed.
|
||||
func TestUpdateDeliveryStatus_DeletedDeliveryIsNotCounted(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := testWebhookDB(t)
|
||||
e := testEngine(t, 1)
|
||||
event := seedEvent(t, db, `{}`)
|
||||
targetID := uuid.New().String()
|
||||
d := seedDelivery(
|
||||
t, db, event.ID, targetID,
|
||||
database.DeliveryStatusRetrying,
|
||||
)
|
||||
|
||||
require.NoError(t, db.Unscoped().
|
||||
Delete(&database.Delivery{}, "id = ?", d.ID).Error)
|
||||
|
||||
require.NoError(t, e.ExportUpdateDeliveryStatus(
|
||||
db, &d, database.DeliveryStatusFailed,
|
||||
))
|
||||
|
||||
assert.Equal(t, database.TargetTotals{TargetID: targetID},
|
||||
targetTotals(t, db, targetID))
|
||||
}
|
||||
|
||||
// TestUpdateDeliveryStatus_FinishedDeliveryIsNotSettledAgain covers a
|
||||
// delivery settled a second time, as recovery can do when a worker has
|
||||
// settled it since recovery read it. Neither status writes over the
|
||||
// first, and the totals do not move.
|
||||
func TestUpdateDeliveryStatus_FinishedDeliveryIsNotSettledAgain(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
finished := []database.DeliveryStatus{
|
||||
database.DeliveryStatusDelivered,
|
||||
database.DeliveryStatusFailed,
|
||||
}
|
||||
|
||||
for _, first := range finished {
|
||||
t.Run(string(first), func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := testWebhookDB(t)
|
||||
e := testEngine(t, 1)
|
||||
event := seedEvent(t, db, `{}`)
|
||||
targetID := uuid.New().String()
|
||||
d := seedDelivery(
|
||||
t, db, event.ID, targetID,
|
||||
database.DeliveryStatusRetrying,
|
||||
)
|
||||
|
||||
// The delivery as recovery read it, before the worker
|
||||
// settled it.
|
||||
readBefore := d
|
||||
|
||||
require.NoError(t, e.ExportUpdateDeliveryStatus(
|
||||
db, &d, first,
|
||||
))
|
||||
|
||||
var settled database.Delivery
|
||||
|
||||
require.NoError(t, db.First(&settled, "id = ?", d.ID).Error)
|
||||
require.NotNil(t, settled.FinishedAt)
|
||||
|
||||
totals := targetTotals(t, db, targetID)
|
||||
|
||||
for _, again := range finished {
|
||||
stale := readBefore
|
||||
|
||||
require.NoError(t, e.ExportUpdateDeliveryStatus(
|
||||
db, &stale, again,
|
||||
))
|
||||
}
|
||||
|
||||
var stored database.Delivery
|
||||
|
||||
require.NoError(t, db.First(&stored, "id = ?", d.ID).Error)
|
||||
assert.Equal(t, first, stored.Status)
|
||||
require.NotNil(t, stored.FinishedAt)
|
||||
assert.True(t, settled.FinishedAt.Equal(*stored.FinishedAt))
|
||||
assert.Equal(t, totals, targetTotals(t, db, targetID))
|
||||
})
|
||||
}
|
||||
}
|
||||
+29
-116
@@ -14,7 +14,6 @@ import (
|
||||
"go.uber.org/fx"
|
||||
"gorm.io/gorm"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/globals"
|
||||
"sneak.berlin/go/webhooker/internal/lifecycle"
|
||||
"sneak.berlin/go/webhooker/internal/logger"
|
||||
"sneak.berlin/go/webhooker/internal/metrics"
|
||||
@@ -123,24 +122,21 @@ type Notifier interface {
|
||||
Notify(tasks []Task)
|
||||
}
|
||||
|
||||
// Archives is how the handlers keep the database targets' archive
|
||||
// files in step with the configuration. Deleting a webhook or a
|
||||
// target releases the cached archive writers, whose open file
|
||||
// handles would otherwise outlive them; renaming one renames the
|
||||
// archive files, which are named for the webhook and the target
|
||||
// (see ArchiveFileName).
|
||||
// WebhookEvictor releases the delivery engine's per-webhook
|
||||
// state for a webhook that no longer needs it — currently the
|
||||
// cached archive writer of the database target, whose open
|
||||
// file handle would otherwise outlive the webhook.
|
||||
//
|
||||
// It is deliberately separate from Notifier: archiving lifecycle
|
||||
// is not notification, and a small interface keeps the handlers
|
||||
// package free of any dependency on the engine's internals while
|
||||
// staying trivially fakeable in tests.
|
||||
// It is deliberately separate from Notifier and deliberately
|
||||
// one method wide: archiving lifecycle is not notification, and
|
||||
// a single-method interface keeps the handlers package free of
|
||||
// any dependency on the engine's internals while staying
|
||||
// trivially fakeable in tests.
|
||||
//
|
||||
// Neither eviction deletes an archive file. Both are idempotent
|
||||
// and are no-ops for a webhook or target with no engine state.
|
||||
type Archives interface {
|
||||
// EvictWebhook never deletes an archive file. It is idempotent
|
||||
// and is a no-op for a webhook with no engine state.
|
||||
type WebhookEvictor interface {
|
||||
EvictWebhook(webhookID string)
|
||||
EvictTarget(targetID string)
|
||||
Rename(targetID, webhookName, targetName string) error
|
||||
}
|
||||
|
||||
// EngineParams are the fx dependencies for the delivery
|
||||
@@ -150,10 +146,8 @@ type EngineParams struct {
|
||||
|
||||
DB *database.Database
|
||||
DBManager *database.WebhookDBManager
|
||||
Globals *globals.Globals
|
||||
Logger *logger.Logger
|
||||
SSRFGuard *Guard
|
||||
Metrics *metrics.Set
|
||||
}
|
||||
|
||||
// Engine processes queued deliveries in the background
|
||||
@@ -173,14 +167,10 @@ type Engine struct {
|
||||
retryCh chan Task
|
||||
workers int
|
||||
|
||||
// version is the running build's version, the one the web UI
|
||||
// footer shows. userAgent puts it on every outbound request.
|
||||
version string
|
||||
|
||||
// mtr is the delivery metric set. Production wires the one
|
||||
// registered on the registry /metrics serves; a test can
|
||||
// substitute a set registered on a registry it holds, so it can
|
||||
// gather what its own deliveries recorded.
|
||||
// mtr is the delivery metric set. Production wires the
|
||||
// process-wide one; a test can substitute a set registered on
|
||||
// a private registry so its assertions are not disturbed by
|
||||
// deliveries other tests are making at the same time.
|
||||
mtr *metrics.Set
|
||||
|
||||
// targets maps each target type to its implementation.
|
||||
@@ -191,7 +181,7 @@ type Engine struct {
|
||||
httpTarget *httpTarget
|
||||
|
||||
// dbTarget is retained so the engine can reach the archive
|
||||
// writer registry for eviction, renames and the idle sweep.
|
||||
// writer registry for webhook eviction and the idle sweep.
|
||||
dbTarget *databaseTarget
|
||||
|
||||
// inflight is the set of deliveries this engine currently owns.
|
||||
@@ -214,8 +204,7 @@ func New(
|
||||
deliveryCh: make(chan Task, deliveryChannelSize),
|
||||
retryCh: make(chan Task, retryChannelSize),
|
||||
workers: defaultWorkers,
|
||||
version: params.Globals.Version,
|
||||
mtr: params.Metrics,
|
||||
mtr: metrics.Default(),
|
||||
}
|
||||
|
||||
e.initTargets(&http.Client{
|
||||
@@ -260,44 +249,17 @@ func (e *Engine) Notify(tasks []Task) {
|
||||
}
|
||||
}
|
||||
|
||||
// EvictWebhook implements Archives. The cached archive writer of
|
||||
// every database target of the webhook is dropped from the
|
||||
// registry and its file handle closed. The archive files
|
||||
// themselves are left on disk — they are long-term storage the
|
||||
// operator owns.
|
||||
// EvictWebhook implements WebhookEvictor. It releases the
|
||||
// engine's per-webhook archiving state: the database target's
|
||||
// cached archive writer is dropped from the registry and its
|
||||
// file handle closed. The archive file itself is left on disk
|
||||
// — it is long-term storage the operator owns.
|
||||
func (e *Engine) EvictWebhook(webhookID string) {
|
||||
if e.dbTarget == nil {
|
||||
return
|
||||
}
|
||||
|
||||
e.dbTarget.evictWebhook(webhookID)
|
||||
}
|
||||
|
||||
// EvictTarget implements Archives. It is EvictWebhook for a single
|
||||
// database target, and leaves the archive file on disk the same
|
||||
// way.
|
||||
func (e *Engine) EvictTarget(targetID string) {
|
||||
if e.dbTarget == nil {
|
||||
return
|
||||
}
|
||||
|
||||
e.dbTarget.evict(targetID)
|
||||
}
|
||||
|
||||
// Rename implements Archives. It renames a database target's
|
||||
// archive file to ArchiveFileName(webhookName, targetName,
|
||||
// targetID), under the lock the target's archive writes and the
|
||||
// idle sweep take. It never replaces a file: if one already has the
|
||||
// new name, the error is ErrArchiveNameTaken. The caller renames
|
||||
// before it saves the new name: see databaseTarget.rename.
|
||||
func (e *Engine) Rename(
|
||||
targetID, webhookName, targetName string,
|
||||
) error {
|
||||
if e.dbTarget == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return e.dbTarget.rename(targetID, webhookName, targetName)
|
||||
e.dbTarget.evict(webhookID)
|
||||
}
|
||||
|
||||
// ScheduleRetry schedules a task to be re-enqueued onto the
|
||||
@@ -338,13 +300,6 @@ func (e *Engine) ScheduleRetry(
|
||||
})
|
||||
}
|
||||
|
||||
// userAgent is the User-Agent header of every http and slack
|
||||
// delivery request: the program name and the running build's
|
||||
// version.
|
||||
func (e *Engine) userAgent() string {
|
||||
return "webhooker/" + e.version
|
||||
}
|
||||
|
||||
// registerHooks wires the engine's start and stop into the fx
|
||||
// lifecycle. The start hook's context is deliberately ignored
|
||||
// (see start for why the worker pool must not inherit it); the
|
||||
@@ -411,8 +366,7 @@ func (e *Engine) start() {
|
||||
// Once the pool has drained it closes the archive writers, so a
|
||||
// clean stop leaves no archive -wal behind. Nothing else holds a
|
||||
// writer for long by then: the archive sweeper stops before the
|
||||
// engine, and deleting or renaming a webhook or target only closes
|
||||
// or moves one. If the pool did
|
||||
// engine, and deleting a webhook only closes one. If the pool did
|
||||
// not drain in time, the writers are left open, as a kill would
|
||||
// leave them. Closing them would wait for any write in progress,
|
||||
// and a worker still running would then open new writers that
|
||||
@@ -577,11 +531,6 @@ func (e *Engine) processRetryTask(
|
||||
return
|
||||
}
|
||||
|
||||
// Set before anything below can fail the delivery: the failure is
|
||||
// added to this target's totals.
|
||||
d.EventID = task.EventID
|
||||
d.TargetID = task.TargetID
|
||||
|
||||
if d.Status != database.DeliveryStatusRetrying {
|
||||
e.log.Debug(
|
||||
"skipping retry for delivery "+
|
||||
@@ -613,6 +562,8 @@ func (e *Engine) processRetryTask(
|
||||
}
|
||||
|
||||
target := buildTargetFromTask(task)
|
||||
d.EventID = task.EventID
|
||||
d.TargetID = task.TargetID
|
||||
d.Event = event
|
||||
d.Target = target
|
||||
|
||||
@@ -1603,9 +1554,8 @@ func (e *Engine) updateDeliveryStatus(
|
||||
targetType database.TargetType,
|
||||
status database.DeliveryStatus,
|
||||
) error {
|
||||
err := webhookDB.Transaction(func(tx *gorm.DB) error {
|
||||
return writeDeliveryStatus(tx, d, status)
|
||||
})
|
||||
err := webhookDB.Model(d).
|
||||
Update("status", status).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf(
|
||||
"updating delivery %s to status %s: %w",
|
||||
@@ -1624,43 +1574,6 @@ func (e *Engine) updateDeliveryStatus(
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeDeliveryStatus writes a delivery's new status. A delivery that
|
||||
// becomes delivered or failed also gets the time it finished, and is
|
||||
// added to its target's delivered or failed total. That write changes
|
||||
// only a delivery not yet delivered or failed, and the total moves
|
||||
// only when it changed a row: retention may have deleted the delivery
|
||||
// while the engine was working on it, and a recovery path may settle
|
||||
// a delivery that a worker has already settled.
|
||||
func writeDeliveryStatus(
|
||||
tx *gorm.DB,
|
||||
d *database.Delivery,
|
||||
status database.DeliveryStatus,
|
||||
) error {
|
||||
if !status.Terminal() {
|
||||
return tx.Model(d).Update("status", status).Error
|
||||
}
|
||||
|
||||
res := tx.Model(d).
|
||||
Where("status NOT IN ?", []database.DeliveryStatus{
|
||||
database.DeliveryStatusDelivered,
|
||||
database.DeliveryStatusFailed,
|
||||
}).
|
||||
Updates(map[string]any{
|
||||
"status": status,
|
||||
"finished_at": time.Now(),
|
||||
})
|
||||
if res.Error != nil || res.RowsAffected == 0 {
|
||||
return res.Error
|
||||
}
|
||||
|
||||
add := database.TargetTotals{TargetID: d.TargetID, Delivered: 1}
|
||||
if status == database.DeliveryStatusFailed {
|
||||
add = database.TargetTotals{TargetID: d.TargetID, Failed: 1}
|
||||
}
|
||||
|
||||
return database.AddTargetTotals(tx, add)
|
||||
}
|
||||
|
||||
// settleStatus moves a delivery to its outcome status and reports a
|
||||
// failed write through bookkeepingFailed, which leaves the row
|
||||
// recoverable. It exists so the target call sites read as one
|
||||
|
||||
@@ -2,6 +2,7 @@ package delivery_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -9,7 +10,6 @@ import (
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/fx"
|
||||
"gorm.io/gorm/clause"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/delivery"
|
||||
)
|
||||
@@ -272,35 +272,22 @@ func TestEngine_StopHookHonoursStopTimeout(t *testing.T) {
|
||||
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
||||
}
|
||||
|
||||
// deliverToArchive gives the setup's webhook a database target,
|
||||
// runs one delivery to it through the running engine, and returns
|
||||
// the target's ID and archive file path. The archive writer holds
|
||||
// the file open afterwards.
|
||||
func deliverToArchive(t *testing.T, s iSetup) (string, string) {
|
||||
// deliverToArchive runs one delivery to a database target through
|
||||
// the running engine and returns the webhook's archive file path.
|
||||
// The archive writer holds the file open afterwards.
|
||||
func deliverToArchive(t *testing.T, s iSetup) string {
|
||||
t.Helper()
|
||||
|
||||
iCreateWebhook(t, s.MainDB, s.WebhookID, "hook")
|
||||
|
||||
tgt := &database.Target{
|
||||
WebhookID: s.WebhookID,
|
||||
Name: "archive",
|
||||
Type: database.TargetTypeDatabase,
|
||||
}
|
||||
require.NoError(
|
||||
t, s.MainDB.Omit(clause.Associations).Create(tgt).Error,
|
||||
)
|
||||
|
||||
deliveryID, task := seedLogTask(t, s)
|
||||
task.TargetID = tgt.ID
|
||||
task.TargetType = database.TargetTypeDatabase
|
||||
|
||||
s.Engine.Notify([]delivery.Task{task})
|
||||
|
||||
iWaitForDelivered(t, s.WebhookDB, deliveryID)
|
||||
|
||||
return tgt.ID, filepath.Join(
|
||||
return filepath.Join(
|
||||
filepath.Dir(s.DBMgr.DBPath(s.WebhookID)),
|
||||
"archive-hook-archive-"+tgt.ID+".db",
|
||||
fmt.Sprintf("archive-%s.db", s.WebhookID),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -317,7 +304,7 @@ func TestEngine_StopHookClosesArchives(t *testing.T) {
|
||||
|
||||
lc := startEngineViaHook(t, s.Engine)
|
||||
|
||||
_, path := deliverToArchive(t, s)
|
||||
path := deliverToArchive(t, s)
|
||||
require.FileExists(
|
||||
t, path+"-wal",
|
||||
"an open archive should have a -wal for the stop to remove",
|
||||
@@ -351,7 +338,7 @@ func TestEngine_StopHookTimeoutLeavesArchivesOpen(t *testing.T) {
|
||||
|
||||
lc := startEngineViaHook(t, s.Engine)
|
||||
|
||||
targetID, _ := deliverToArchive(t, s)
|
||||
deliverToArchive(t, s)
|
||||
|
||||
release := make(chan struct{})
|
||||
|
||||
@@ -365,7 +352,7 @@ func TestEngine_StopHookTimeoutLeavesArchivesOpen(t *testing.T) {
|
||||
requireStopHookExpires(t, lc.hooks[0], "delivery engine")
|
||||
|
||||
require.True(
|
||||
t, s.Engine.ExportArchiveHandleOpen(targetID),
|
||||
t, s.Engine.ExportArchiveHandleOpen(s.WebhookID),
|
||||
"a stop that timed out must not close archive writers",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -57,10 +57,7 @@ func testWebhookDB(t *testing.T) *gorm.DB {
|
||||
&database.Event{},
|
||||
&database.Delivery{},
|
||||
&database.DeliveryResult{},
|
||||
&database.EventTotals{},
|
||||
&database.TargetTotals{},
|
||||
))
|
||||
require.NoError(t, db.Create(&database.EventTotals{}).Error)
|
||||
|
||||
return db
|
||||
}
|
||||
@@ -351,15 +348,23 @@ func TestDeliverDatabase_ImmediateSuccess(
|
||||
|
||||
db := testWebhookDB(t)
|
||||
|
||||
// The database target archives for real, so the engine needs
|
||||
// the target in the main database and a data directory.
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, "")
|
||||
// The database target archives for real now, so the engine
|
||||
// needs a webhook DB manager to locate the data directory.
|
||||
e := delivery.NewTestEngineWithDB(
|
||||
nil,
|
||||
database.NewTestWebhookDBManager(t.TempDir()),
|
||||
slog.New(slog.NewTextHandler(
|
||||
os.Stderr,
|
||||
&slog.HandlerOptions{Level: slog.LevelDebug},
|
||||
)),
|
||||
&http.Client{Timeout: 5 * time.Second},
|
||||
1,
|
||||
)
|
||||
|
||||
event := seedEvent(t, db, `{"db":"target"}`)
|
||||
d := seedDatabaseTargetDelivery(t, db, event, tgt)
|
||||
d := seedDatabaseTargetDelivery(t, db, event, "")
|
||||
|
||||
env.eng.ExportDeliverDatabase(db, d)
|
||||
e.ExportDeliverDatabase(db, d)
|
||||
|
||||
var updated database.Delivery
|
||||
|
||||
@@ -1239,6 +1244,11 @@ func TestDoHTTPRequest_ForwardsHeaders(t *testing.T) {
|
||||
testContentType,
|
||||
receivedHeaders.Get("Content-Type"),
|
||||
)
|
||||
|
||||
assert.Equal(t,
|
||||
"webhooker/1.0",
|
||||
receivedHeaders.Get("User-Agent"),
|
||||
)
|
||||
}
|
||||
|
||||
// The event's stored inbound headers carry the same Content-Type the
|
||||
@@ -1307,7 +1317,6 @@ func TestApplyRequestHeaders_SendsOneContentType(t *testing.T) {
|
||||
ContentType: tc.event,
|
||||
},
|
||||
cfg,
|
||||
"webhooker/dev",
|
||||
)
|
||||
|
||||
assert.Equal(t,
|
||||
@@ -1324,27 +1333,32 @@ func TestProcessDelivery_RoutesToCorrectHandler(
|
||||
|
||||
db := testWebhookDB(t)
|
||||
|
||||
// The database target archives for real, so the engine needs
|
||||
// the target in the main database and a data directory.
|
||||
env := setupArchiveTest(t)
|
||||
archive := env.seedDatabaseTarget(t, "")
|
||||
// The database target archives for real now, so the engine
|
||||
// needs a webhook DB manager to locate the data directory.
|
||||
e := delivery.NewTestEngineWithDB(
|
||||
nil,
|
||||
database.NewTestWebhookDBManager(t.TempDir()),
|
||||
slog.New(slog.NewTextHandler(
|
||||
os.Stderr,
|
||||
&slog.HandlerOptions{Level: slog.LevelDebug},
|
||||
)),
|
||||
&http.Client{Timeout: 5 * time.Second},
|
||||
1,
|
||||
)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
targetType database.TargetType
|
||||
targetID string
|
||||
wantStatus database.DeliveryStatus
|
||||
}{
|
||||
{
|
||||
"database target",
|
||||
database.TargetTypeDatabase,
|
||||
archive.ID,
|
||||
database.DeliveryStatusDelivered,
|
||||
},
|
||||
{
|
||||
"log target",
|
||||
database.TargetTypeLog,
|
||||
uuid.New().String(),
|
||||
database.DeliveryStatusDelivered,
|
||||
},
|
||||
}
|
||||
@@ -1354,7 +1368,7 @@ func TestProcessDelivery_RoutesToCorrectHandler(
|
||||
t.Parallel()
|
||||
|
||||
runRoutingSubtest(
|
||||
t, db, env.eng, tt.targetType, tt.targetID,
|
||||
t, db, e, tt.targetType,
|
||||
tt.wantStatus,
|
||||
)
|
||||
})
|
||||
@@ -1366,7 +1380,6 @@ func runRoutingSubtest(
|
||||
db *gorm.DB,
|
||||
e *delivery.Engine,
|
||||
targetType database.TargetType,
|
||||
targetID string,
|
||||
wantStatus database.DeliveryStatus,
|
||||
) {
|
||||
t.Helper()
|
||||
@@ -1374,7 +1387,8 @@ func runRoutingSubtest(
|
||||
event := seedEvent(t, db, `{"routing":"test"}`)
|
||||
|
||||
dlv := seedDelivery(
|
||||
t, db, event.ID, targetID,
|
||||
t, db, event.ID,
|
||||
uuid.New().String(),
|
||||
database.DeliveryStatusPending,
|
||||
)
|
||||
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"go.uber.org/fx"
|
||||
"gorm.io/gorm"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
@@ -41,6 +40,11 @@ const (
|
||||
ExportPendingSweepMinAge = pendingSweepMinAge
|
||||
)
|
||||
|
||||
// ExportIsBlockedIP exposes isBlockedIP for testing.
|
||||
func ExportIsBlockedIP(ip net.IP) bool {
|
||||
return isBlockedIP(ip)
|
||||
}
|
||||
|
||||
// NewTestGuard builds an SSRF Guard from an explicit egress
|
||||
// allowlist, without going through config. Passing no prefixes
|
||||
// yields the default guard, which blocks every private/reserved
|
||||
@@ -66,11 +70,6 @@ func ExportBlockedNetworks() []*net.IPNet {
|
||||
return blockedNetworks
|
||||
}
|
||||
|
||||
// ExportBlockedPublicNetworks exposes blockedPublicNetworks.
|
||||
func ExportBlockedPublicNetworks() []*net.IPNet {
|
||||
return blockedPublicNetworks
|
||||
}
|
||||
|
||||
// ExportIsForwardableHeader exposes isForwardableHeader.
|
||||
func ExportIsForwardableHeader(name string) bool {
|
||||
return isForwardableHeader(name)
|
||||
@@ -83,9 +82,8 @@ func ExportApplyRequestHeaders(
|
||||
req *http.Request,
|
||||
event *database.Event,
|
||||
cfg *HTTPTargetConfig,
|
||||
userAgent string,
|
||||
) []string {
|
||||
return applyRequestHeaders(req, event, cfg, userAgent)
|
||||
return applyRequestHeaders(req, event, cfg)
|
||||
}
|
||||
|
||||
// ExportTruncate exposes truncate for testing.
|
||||
@@ -152,16 +150,6 @@ func (e *Engine) ExportDeliverSlack(
|
||||
)
|
||||
}
|
||||
|
||||
// ExportUpdateDeliveryStatus exposes updateDeliveryStatus. It passes no
|
||||
// target type, so no metric moves.
|
||||
func (e *Engine) ExportUpdateDeliveryStatus(
|
||||
webhookDB *gorm.DB,
|
||||
d *database.Delivery,
|
||||
status database.DeliveryStatus,
|
||||
) error {
|
||||
return e.updateDeliveryStatus(webhookDB, d, "", status)
|
||||
}
|
||||
|
||||
// ExportProcessNewTask exposes processNewTask.
|
||||
func (e *Engine) ExportProcessNewTask(
|
||||
ctx context.Context, task *Task,
|
||||
@@ -401,7 +389,7 @@ func NewTestEngine(
|
||||
deliveryCh: make(chan Task, deliveryChannelSize),
|
||||
retryCh: make(chan Task, retryChannelSize),
|
||||
workers: workers,
|
||||
mtr: metrics.New(prometheus.NewRegistry()),
|
||||
mtr: metrics.Default(),
|
||||
}
|
||||
e.initTargets(client)
|
||||
|
||||
@@ -416,7 +404,7 @@ func NewTestEngineSmallRetry(
|
||||
e := &Engine{
|
||||
log: log,
|
||||
retryCh: make(chan Task, 1),
|
||||
mtr: metrics.New(prometheus.NewRegistry()),
|
||||
mtr: metrics.Default(),
|
||||
}
|
||||
e.initTargets(nil)
|
||||
|
||||
@@ -439,7 +427,7 @@ func NewTestEngineWithDB(
|
||||
deliveryCh: make(chan Task, deliveryChannelSize),
|
||||
retryCh: make(chan Task, retryChannelSize),
|
||||
workers: workers,
|
||||
mtr: metrics.New(prometheus.NewRegistry()),
|
||||
mtr: metrics.Default(),
|
||||
}
|
||||
e.initTargets(client)
|
||||
|
||||
@@ -447,7 +435,8 @@ func NewTestEngineWithDB(
|
||||
}
|
||||
|
||||
// ExportSetMetrics substitutes the engine's metric set, so a test can
|
||||
// assert on collectors registered on a registry it holds.
|
||||
// assert on collectors registered on a private registry instead of
|
||||
// the process-wide ones every other test is also moving.
|
||||
func (e *Engine) ExportSetMetrics(mtr *metrics.Set) {
|
||||
e.mtr = mtr
|
||||
}
|
||||
@@ -474,7 +463,7 @@ func NewTestCircuitBreaker(
|
||||
type ExportArchivedEvent = archivedEvent
|
||||
|
||||
// ExportArchiveWriter wraps an archiveWriter so black-box tests
|
||||
// can exercise the archive file mechanics.
|
||||
// can exercise the per-webhook archive file mechanics.
|
||||
type ExportArchiveWriter struct {
|
||||
w *archiveWriter
|
||||
}
|
||||
@@ -549,12 +538,6 @@ func (e *ExportArchiveWriter) Evict() {
|
||||
e.w.evict()
|
||||
}
|
||||
|
||||
// Rename gives the archive file a new name in the same directory,
|
||||
// as a rename of the webhook or target does.
|
||||
func (e *ExportArchiveWriter) Rename(name string) error {
|
||||
return e.w.rename(name)
|
||||
}
|
||||
|
||||
// HandleOpen reports whether the writer currently holds an open
|
||||
// archive handle.
|
||||
func (e *ExportArchiveWriter) HandleOpen() bool {
|
||||
@@ -574,16 +557,16 @@ func (e *ExportArchiveWriter) Same(
|
||||
}
|
||||
|
||||
// ExportArchiveWriterFor returns the archive writer the registry
|
||||
// currently caches for a database target, or nil when none is
|
||||
// cached. It never creates one, so a test can hold a reference to
|
||||
// the very writer an eviction is about to detach.
|
||||
// currently caches for a webhook, or nil when none is cached. It
|
||||
// never creates one, so a test can hold a reference to the very
|
||||
// writer an eviction is about to detach.
|
||||
func (e *Engine) ExportArchiveWriterFor(
|
||||
targetID string,
|
||||
webhookID string,
|
||||
) *ExportArchiveWriter {
|
||||
e.dbTarget.mu.Lock()
|
||||
defer e.dbTarget.mu.Unlock()
|
||||
|
||||
w, ok := e.dbTarget.writers[targetID]
|
||||
w, ok := e.dbTarget.writers[webhookID]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
@@ -592,26 +575,26 @@ func (e *Engine) ExportArchiveWriterFor(
|
||||
}
|
||||
|
||||
// ExportHasArchiveWriter reports whether the database target
|
||||
// type currently caches an archive writer for a target.
|
||||
// currently caches an archive writer for a webhook.
|
||||
func (e *Engine) ExportHasArchiveWriter(
|
||||
targetID string,
|
||||
webhookID string,
|
||||
) bool {
|
||||
e.dbTarget.mu.Lock()
|
||||
defer e.dbTarget.mu.Unlock()
|
||||
|
||||
_, ok := e.dbTarget.writers[targetID]
|
||||
_, ok := e.dbTarget.writers[webhookID]
|
||||
|
||||
return ok
|
||||
}
|
||||
|
||||
// ExportArchiveHandleOpen reports whether the cached archive
|
||||
// writer for a target holds an open database handle. It
|
||||
// writer for a webhook holds an open database handle. It
|
||||
// returns false when no writer is cached.
|
||||
func (e *Engine) ExportArchiveHandleOpen(
|
||||
targetID string,
|
||||
webhookID string,
|
||||
) bool {
|
||||
e.dbTarget.mu.Lock()
|
||||
w, ok := e.dbTarget.writers[targetID]
|
||||
w, ok := e.dbTarget.writers[webhookID]
|
||||
e.dbTarget.mu.Unlock()
|
||||
|
||||
if !ok {
|
||||
@@ -625,12 +608,12 @@ func (e *Engine) ExportArchiveHandleOpen(
|
||||
}
|
||||
|
||||
// ExportEnsureArchiveWriter creates (if needed) and returns the
|
||||
// archive file path of the cached writer for a target, so a
|
||||
// archive file path of the cached writer for a webhook, so a
|
||||
// test can prime the registry the way a delivery would.
|
||||
func (e *Engine) ExportEnsureArchiveWriter(
|
||||
targetID string,
|
||||
webhookID string,
|
||||
) (string, error) {
|
||||
w, err := e.dbTarget.writerFor(targetID)
|
||||
w, err := e.dbTarget.writerFor(webhookID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -638,14 +621,14 @@ func (e *Engine) ExportEnsureArchiveWriter(
|
||||
return w.path, nil
|
||||
}
|
||||
|
||||
// ExportSweepWriterFor takes a target's registry writer exactly
|
||||
// ExportSweepWriterFor takes a webhook's registry writer exactly
|
||||
// as the idle sweep does, reporting whether the sweep had to
|
||||
// create the entry. It lets a test drive the registry through the
|
||||
// sweep's own entry point instead of choreographing goroutines.
|
||||
func (e *Engine) ExportSweepWriterFor(
|
||||
targetID string,
|
||||
webhookID string,
|
||||
) (*ExportArchiveWriter, bool, error) {
|
||||
w, created, err := e.dbTarget.sweepWriterFor(targetID)
|
||||
w, created, err := e.dbTarget.sweepWriterFor(webhookID)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
@@ -656,9 +639,9 @@ func (e *Engine) ExportSweepWriterFor(
|
||||
// ExportReleaseSweepWriter releases a sweep-created registry entry
|
||||
// exactly as a finished sweep does.
|
||||
func (e *Engine) ExportReleaseSweepWriter(
|
||||
targetID string, w *ExportArchiveWriter,
|
||||
webhookID string, w *ExportArchiveWriter,
|
||||
) {
|
||||
e.dbTarget.releaseSweepWriter(targetID, w.w)
|
||||
e.dbTarget.releaseSweepWriter(webhookID, w.w)
|
||||
}
|
||||
|
||||
// NewTestArchiveSweeper builds an ArchiveSweeper backed by the
|
||||
|
||||
@@ -35,8 +35,9 @@ const (
|
||||
)
|
||||
|
||||
// mIsolate gives the setup's engine a metric set registered on a
|
||||
// registry this test holds, so its exact assertions can gather from
|
||||
// it.
|
||||
// private registry. The process-wide collectors are moved by every
|
||||
// other delivery test running in parallel, so exact assertions are
|
||||
// only possible against a registry this test owns.
|
||||
func mIsolate(
|
||||
t *testing.T, s iSetup,
|
||||
) *prometheus.Registry {
|
||||
|
||||
@@ -375,7 +375,6 @@ func TestApplyRequestHeaders_ReportsOriginScopedNames(t *testing.T) {
|
||||
"Content-Type": testContentType,
|
||||
},
|
||||
},
|
||||
"webhooker/dev",
|
||||
)
|
||||
|
||||
assert.Equal(t,
|
||||
|
||||
+44
-99
@@ -17,6 +17,10 @@ const (
|
||||
// dnsResolutionTimeout is the maximum time to wait for
|
||||
// DNS resolution during SSRF validation.
|
||||
dnsResolutionTimeout = 5 * time.Second
|
||||
|
||||
// azureWireServer is Azure's WireServer, a public address that
|
||||
// serves VM credentials.
|
||||
azureWireServer = "168.63.129.16"
|
||||
)
|
||||
|
||||
// Sentinel errors for SSRF validation.
|
||||
@@ -25,64 +29,42 @@ var (
|
||||
errNoIPs = errors.New(
|
||||
"hostname resolved to no IP addresses",
|
||||
)
|
||||
// ErrBlockedPrivateOrReservedIP reports an address in the
|
||||
// default blocklist's private and reserved ranges,
|
||||
// blockedNetworks.
|
||||
ErrBlockedPrivateOrReservedIP = errors.New(
|
||||
// ErrBlockedIP reports a private or reserved address the
|
||||
// default blocklist refuses, one that ALLOWED_EGRESS_CIDRS
|
||||
// can open.
|
||||
ErrBlockedIP = errors.New(
|
||||
"blocked private or reserved address",
|
||||
)
|
||||
// errBlockedPublicMetadata reports a public address on the
|
||||
// default blocklist, one in blockedPublicNetworks.
|
||||
errBlockedPublicMetadata = errors.New(
|
||||
errBlockedWireServer = errors.New(
|
||||
"blocked cloud metadata address",
|
||||
)
|
||||
errBlockedMetadata = errors.New(
|
||||
"blocked link-local, cloud instance metadata or " +
|
||||
"unspecified address: ALLOWED_EGRESS_CIDRS cannot open it",
|
||||
"blocked link-local or cloud instance metadata " +
|
||||
"address: ALLOWED_EGRESS_CIDRS cannot open it",
|
||||
)
|
||||
errInvalidScheme = errors.New(
|
||||
"only http and https are allowed",
|
||||
)
|
||||
)
|
||||
|
||||
// blockedNetworks and blockedPublicNetworks together are the
|
||||
// default blocklist: the private and reserved IP ranges, plus
|
||||
// the public cloud metadata addresses, that are blocked to
|
||||
// prevent SSRF attacks. An operator can permit specific blocks
|
||||
// out of this set with ALLOWED_EGRESS_CIDRS; see Guard.
|
||||
//
|
||||
// blockedNetworks holds the private and reserved IP ranges.
|
||||
// blockedNetworks is the default blocklist: the private and
|
||||
// reserved IP ranges, plus the public cloud metadata addresses,
|
||||
// that are blocked to prevent SSRF attacks. An operator can
|
||||
// permit specific blocks out of this set with
|
||||
// ALLOWED_EGRESS_CIDRS; see Guard.
|
||||
//
|
||||
//nolint:gochecknoglobals // package-level network list is appropriate here
|
||||
var blockedNetworks []*net.IPNet
|
||||
|
||||
// blockedPublicNetworks holds the default blocklist's public
|
||||
// addresses, kept apart from blockedNetworks so that they are
|
||||
// refused as cloud metadata addresses, never as private or
|
||||
// reserved ones.
|
||||
//
|
||||
// A public address belongs on the default blocklist only if it
|
||||
// hands credentials, user data or bootstrap material to whatever
|
||||
// can reach it, without the caller presenting anything; it goes
|
||||
// in this list. A provider's other public addresses are not
|
||||
// refused, since reaching them can be legitimate and no list of
|
||||
// them could be complete.
|
||||
//
|
||||
//nolint:gochecknoglobals // package-level network list is appropriate here
|
||||
var blockedPublicNetworks []*net.IPNet
|
||||
|
||||
// alwaysBlockedNetworks are the ranges no configuration can
|
||||
// open, so a supplied CIDR that covers one still leaves it
|
||||
// blocked. An entry is here for one of two reasons: it is a
|
||||
// metadata endpoint (the link-local blocks and the cloud
|
||||
// instance metadata endpoints that live outside them), or it is
|
||||
// an unspecified address. Reaching a metadata endpoint is
|
||||
// credential or user-data theft rather than delivery to an
|
||||
// internal service.
|
||||
// open: the link-local blocks and the cloud instance metadata
|
||||
// endpoints that live outside them. Reaching one is credential
|
||||
// or user-data theft rather than delivery to an internal
|
||||
// service, so a supplied CIDR that covers such an address still
|
||||
// leaves it blocked.
|
||||
//
|
||||
// Inclusion criterion for metadata endpoints — one belongs here
|
||||
// only if BOTH hold, and every metadata entry below satisfies
|
||||
// both:
|
||||
// Inclusion criterion — an address belongs here only if BOTH
|
||||
// hold, and every entry below satisfies both:
|
||||
//
|
||||
// 1. It is a fixed address assigned by the provider, or a
|
||||
// range reserved by IANA — never one the operator chose.
|
||||
@@ -93,8 +75,8 @@ var blockedPublicNetworks []*net.IPNet
|
||||
// not cheaply rotated.
|
||||
//
|
||||
// Both halves are load-bearing, so use them to refuse a
|
||||
// metadata candidate and say why. An endpoint disclosing only
|
||||
// the operator's own inventory (instance id, region, disks, NICs)
|
||||
// candidate and say why. An endpoint disclosing only the
|
||||
// operator's own inventory (instance id, region, disks, NICs)
|
||||
// fails (2): letting a delivery target reach the operator's own
|
||||
// infrastructure is the feature ALLOWED_EGRESS_CIDRS exists to
|
||||
// provide. But (2) is not "IAM credentials only" either —
|
||||
@@ -109,21 +91,12 @@ var blockedPublicNetworks []*net.IPNet
|
||||
// when it clears both halves. Nothing in this list can be
|
||||
// reopened, so putting a public address here leaves the operator
|
||||
// no escape hatch at all — the condition ALLOWED_EGRESS_CIDRS
|
||||
// exists to remove. Default-block it in blockedPublicNetworks
|
||||
// instead, which an allowlist can override.
|
||||
// exists to remove. Default-block it in blockedNetworks instead,
|
||||
// which an allowlist can override.
|
||||
//
|
||||
// This is a criterion, not an enumeration of every metadata
|
||||
// address in existence.
|
||||
//
|
||||
// The unspecified addresses 0.0.0.0 and :: are here for a
|
||||
// separate reason: they disclose nothing, but no host can have
|
||||
// either, and on Linux a connection to one reaches this host's
|
||||
// own loopback. Listing them means an allowlist reaches loopback
|
||||
// only through an entry that covers a loopback address
|
||||
// (127.0.0.0/8, ::1/128, 0.0.0.0/0), never through one that
|
||||
// covers only 0.0.0.0 or :: (0.0.0.0/8, for example). Nothing
|
||||
// else lives at either address, so refusing them costs nothing.
|
||||
//
|
||||
// Every entry is either already in blockedNetworks — this list is
|
||||
// what makes it unconditional — or an alternate encoding of
|
||||
// 169.254.169.254 that Contains does not match against
|
||||
@@ -143,51 +116,24 @@ var alwaysBlockedNetworks []*net.IPNet
|
||||
//nolint:gochecknoinits // init is the idiomatic way to parse CIDRs once at startup
|
||||
func init() {
|
||||
blockedNetworks = mustParseCIDRs([]string{
|
||||
// IPv4 loopback.
|
||||
"127.0.0.0/8",
|
||||
// RFC 1918 private network.
|
||||
"10.0.0.0/8",
|
||||
// RFC 1918 private network.
|
||||
"172.16.0.0/12",
|
||||
// RFC 1918 private network.
|
||||
"192.168.0.0/16",
|
||||
// IPv4 link-local.
|
||||
"169.254.0.0/16",
|
||||
// "This network", holding the IPv4 unspecified address 0.0.0.0.
|
||||
"0.0.0.0/8",
|
||||
// Carrier-grade NAT shared address space.
|
||||
"100.64.0.0/10",
|
||||
// IETF protocol assignments.
|
||||
"192.0.0.0/24",
|
||||
// IPv4 documentation (TEST-NET-1).
|
||||
"192.0.2.0/24",
|
||||
// Benchmarking.
|
||||
"198.18.0.0/15",
|
||||
// IPv4 documentation (TEST-NET-2).
|
||||
"198.51.100.0/24",
|
||||
// IPv4 documentation (TEST-NET-3).
|
||||
"203.0.113.0/24",
|
||||
// IPv4 multicast.
|
||||
"224.0.0.0/4",
|
||||
// Reserved, including the broadcast address.
|
||||
"240.0.0.0/4",
|
||||
// IPv6 loopback.
|
||||
"::1/128",
|
||||
// IPv6 unspecified address.
|
||||
"::/128",
|
||||
// IPv6 unique local addresses.
|
||||
"fc00::/7",
|
||||
// IPv6 link-local.
|
||||
"fe80::/10",
|
||||
// IPv6 multicast.
|
||||
"ff00::/8",
|
||||
// IPv6 documentation.
|
||||
"2001:db8::/32",
|
||||
})
|
||||
|
||||
blockedPublicNetworks = mustParseCIDRs([]string{
|
||||
// Azure WireServer, a public address that serves VM credentials.
|
||||
"168.63.129.16/32",
|
||||
azureWireServer + "/32",
|
||||
})
|
||||
|
||||
// Every entry is named. The set must not grow or shrink
|
||||
@@ -242,14 +188,6 @@ func init() {
|
||||
// allowlist from opening it.
|
||||
"192.0.0.192/32",
|
||||
|
||||
// The unspecified addresses, each of which reaches this
|
||||
// host's loopback on Linux.
|
||||
//
|
||||
// IPv4 unspecified address, inside the blocked 0.0.0.0/8.
|
||||
"0.0.0.0/32",
|
||||
// IPv6 unspecified address.
|
||||
"::/128",
|
||||
|
||||
// 169.254.169.254 as an IPv4-compatible IPv6 address.
|
||||
"::a9fe:a9fe/128",
|
||||
// 169.254.169.254 behind the NAT64 well-known prefix.
|
||||
@@ -289,6 +227,13 @@ func matchesAny(networks []*net.IPNet, ip net.IP) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// isBlockedIP checks whether an IP address falls within
|
||||
// the default blocklist, before any operator allowlist is
|
||||
// considered.
|
||||
func isBlockedIP(ip net.IP) bool {
|
||||
return matchesAny(blockedNetworks, ip)
|
||||
}
|
||||
|
||||
// Guard makes every SSRF decision in the process.
|
||||
//
|
||||
// It holds the operator's ALLOWED_EGRESS_CIDRS allowlist and
|
||||
@@ -386,12 +331,10 @@ func (g *Guard) allows(ip net.IP) bool {
|
||||
// The order is the policy:
|
||||
//
|
||||
// 1. alwaysBlockedNetworks is refused before the allowlist is
|
||||
// consulted, so no configured CIDR reaches link-local, a
|
||||
// cloud metadata endpoint at a non-public address, or an
|
||||
// unspecified address.
|
||||
// consulted, so no configured CIDR reaches link-local or a
|
||||
// cloud metadata endpoint at a non-public address.
|
||||
// 2. The allowlist is consulted next, so a listed private
|
||||
// network, or a listed public address on the default
|
||||
// blocklist, becomes reachable.
|
||||
// network becomes reachable.
|
||||
// 3. Everything else keeps the default blocklist's answer.
|
||||
func (g *Guard) checkIP(ip net.IP) error {
|
||||
if matchesAny(alwaysBlockedNetworks, ip) {
|
||||
@@ -404,15 +347,17 @@ func (g *Guard) checkIP(ip net.IP) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
if matchesAny(blockedNetworks, ip) {
|
||||
// WireServer is on the default blocklist but is a public
|
||||
// address, so its refusal does not call it private or reserved.
|
||||
if ip.Equal(net.ParseIP(azureWireServer)) {
|
||||
return fmt.Errorf(
|
||||
"target IP %s: %w", ip, ErrBlockedPrivateOrReservedIP,
|
||||
"target IP %s: %w", ip, errBlockedWireServer,
|
||||
)
|
||||
}
|
||||
|
||||
if matchesAny(blockedPublicNetworks, ip) {
|
||||
if isBlockedIP(ip) {
|
||||
return fmt.Errorf(
|
||||
"target IP %s: %w", ip, errBlockedPublicMetadata,
|
||||
"target IP %s: %w", ip, ErrBlockedIP,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -23,10 +23,6 @@ const (
|
||||
metadataIP = "169.254.169.254"
|
||||
metadataURL = "http://" + metadataIP + "/latest/meta-data/"
|
||||
|
||||
// linkLocalIPv4 is the IPv4 link-local block, which holds
|
||||
// metadataIP.
|
||||
linkLocalIPv4 = "169.254.0.0/16"
|
||||
|
||||
// loopbackHookURL is a target on this host: blocked by
|
||||
// default, reachable only once an operator allowlists
|
||||
// loopback.
|
||||
@@ -168,13 +164,12 @@ func TestGuardAllowlist_UnlistedPrivateStillRefused(t *testing.T) {
|
||||
|
||||
// TestGuardAllowlist_MetadataAlwaysRefused is the load-bearing
|
||||
// case: cloud instance metadata endpoints are credential theft
|
||||
// rather than delivery to an internal service, and the
|
||||
// unspecified addresses 0.0.0.0 and :: reach this host's loopback
|
||||
// on Linux, so no allowlist reaches any of them. Every guard
|
||||
// below names a CIDR that covers its target — including
|
||||
// 0.0.0.0/0, ::/0, and the ordinary ULA and CGNAT blocks an
|
||||
// operator would really list — and the address must stay
|
||||
// refused anyway, on both the validation and the delivery path.
|
||||
// rather than delivery to an internal service, so no allowlist
|
||||
// reaches one. Every guard below names a CIDR that covers its
|
||||
// target — including 0.0.0.0/0, ::/0, and the ordinary ULA and
|
||||
// CGNAT blocks an operator would really list — and the address
|
||||
// must stay refused anyway, on both the validation and the
|
||||
// delivery path.
|
||||
func TestGuardAllowlist_MetadataAlwaysRefused(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -220,17 +215,15 @@ type metadataAlwaysRefusedCase struct {
|
||||
}
|
||||
|
||||
// metadataAlwaysRefusedCases enumerates every unconditionally
|
||||
// blocked address (link-local, the cloud metadata endpoints and
|
||||
// the unspecified addresses) together with an allowlist entry
|
||||
// that would otherwise reach it. Split by family of address only
|
||||
// to stay under the function-length limit.
|
||||
// blocked address together with an allowlist entry that would
|
||||
// otherwise reach it. Split by family of address only to stay
|
||||
// under the function-length limit.
|
||||
func metadataAlwaysRefusedCases() []metadataAlwaysRefusedCase {
|
||||
cases := linkLocalRefusedCases()
|
||||
cases = append(cases, ulaMetadataRefusedCases()...)
|
||||
cases = append(cases, ipv4MetadataRefusedCases()...)
|
||||
cases = append(cases, encodedMetadataRefusedCases()...)
|
||||
|
||||
return append(cases, unspecifiedRefusedCases()...)
|
||||
return append(cases, encodedMetadataRefusedCases()...)
|
||||
}
|
||||
|
||||
// linkLocalRefusedCases covers the link-local blocks, including
|
||||
@@ -244,7 +237,7 @@ func linkLocalRefusedCases() []metadataAlwaysRefusedCase {
|
||||
},
|
||||
{
|
||||
name: "whole link-local block",
|
||||
allow: linkLocalIPv4,
|
||||
allow: "169.254.0.0/16",
|
||||
target: metadataURL,
|
||||
},
|
||||
{
|
||||
@@ -370,23 +363,6 @@ func encodedMetadataRefusedCases() []metadataAlwaysRefusedCase {
|
||||
}
|
||||
}
|
||||
|
||||
// unspecifiedRefusedCases covers the unspecified addresses, each
|
||||
// of which reaches this host's loopback on Linux.
|
||||
func unspecifiedRefusedCases() []metadataAlwaysRefusedCase {
|
||||
return []metadataAlwaysRefusedCase{
|
||||
{
|
||||
name: "IPv4 unspecified address under 0.0.0.0/0",
|
||||
allow: allowAllIPv4,
|
||||
target: "http://0.0.0.0:8080/hook",
|
||||
},
|
||||
{
|
||||
name: "IPv6 unspecified address under ::/0",
|
||||
allow: allowAllIPv6,
|
||||
target: "http://[::]:8080/hook",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// TestGuardAllowlist_PublicUnaffected asserts the allowlist does
|
||||
// not narrow anything: public addresses were reachable before it
|
||||
// existed and stay reachable, whether or not a list is set.
|
||||
@@ -436,9 +412,6 @@ func TestGuardAllowlist_AzureWireServerReopenable(t *testing.T) {
|
||||
"WireServer must be refused by the default blocklist, "+
|
||||
"which an allowlist can override",
|
||||
)
|
||||
require.NotErrorIs(t, err, delivery.ErrBlockedPrivateOrReservedIP,
|
||||
"WireServer is public, not private or reserved",
|
||||
)
|
||||
|
||||
assertDialRefused(t, defaultGuard, target)
|
||||
|
||||
@@ -523,7 +496,7 @@ func TestAlwaysBlockedNetworks_PinnedSet(t *testing.T) {
|
||||
want := []string{
|
||||
// IPv4 link-local: the 169.254.169.254 metadata
|
||||
// service on AWS, Azure and others.
|
||||
linkLocalIPv4,
|
||||
"169.254.0.0/16",
|
||||
// IPv6 link-local.
|
||||
"fe80::/10",
|
||||
// AWS IPv6 IMDS, inside the ULA space an operator may
|
||||
@@ -544,10 +517,6 @@ func TestAlwaysBlockedNetworks_PinnedSet(t *testing.T) {
|
||||
// Oracle Cloud Classic metadata, inside the blocked
|
||||
// 192.0.0.0/24.
|
||||
"192.0.0.192/32",
|
||||
// The IPv4 and IPv6 unspecified addresses, each of
|
||||
// which reaches this host's loopback on Linux.
|
||||
"0.0.0.0/32",
|
||||
"::/128",
|
||||
// 169.254.169.254 as an IPv4-compatible IPv6 address.
|
||||
"::a9fe:a9fe/128",
|
||||
// 169.254.169.254 behind the NAT64 well-known prefix.
|
||||
@@ -557,94 +526,6 @@ func TestAlwaysBlockedNetworks_PinnedSet(t *testing.T) {
|
||||
assert.Equal(t, want, got)
|
||||
}
|
||||
|
||||
// TestDefaultBlocklist_PinnedSet pins each list of the default
|
||||
// blocklist on its own, the private and reserved ranges in
|
||||
// blockedNetworks and the public addresses in
|
||||
// blockedPublicNetworks, so moving an entry from one list to the
|
||||
// other fails it. For the first address of each entry it then
|
||||
// checks that the default guard refuses it, and that listing the
|
||||
// entry in ALLOWED_EGRESS_CIDRS opens it unless the unconditional
|
||||
// set holds that address.
|
||||
func TestDefaultBlocklist_PinnedSet(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// public marks an entry of blockedPublicNetworks; every other
|
||||
// entry belongs in blockedNetworks.
|
||||
tests := []struct {
|
||||
cidr string
|
||||
public bool
|
||||
reopenable bool
|
||||
}{
|
||||
{cidr: "127.0.0.0/8", reopenable: true},
|
||||
{cidr: "10.0.0.0/8", reopenable: true},
|
||||
{cidr: "172.16.0.0/12", reopenable: true},
|
||||
{cidr: "192.168.0.0/16", reopenable: true},
|
||||
{cidr: linkLocalIPv4, reopenable: false},
|
||||
// Its first address, 0.0.0.0, is in the unconditional set.
|
||||
{cidr: "0.0.0.0/8", reopenable: false},
|
||||
{cidr: "100.64.0.0/10", reopenable: true},
|
||||
{cidr: "192.0.0.0/24", reopenable: true},
|
||||
{cidr: "192.0.2.0/24", reopenable: true},
|
||||
{cidr: "198.18.0.0/15", reopenable: true},
|
||||
{cidr: "198.51.100.0/24", reopenable: true},
|
||||
{cidr: "203.0.113.0/24", reopenable: true},
|
||||
{cidr: "224.0.0.0/4", reopenable: true},
|
||||
{cidr: "240.0.0.0/4", reopenable: true},
|
||||
{cidr: "::1/128", reopenable: true},
|
||||
{cidr: "::/128", reopenable: false},
|
||||
{cidr: "fc00::/7", reopenable: true},
|
||||
{cidr: "fe80::/10", reopenable: false},
|
||||
{cidr: "ff00::/8", reopenable: true},
|
||||
{cidr: "2001:db8::/32", reopenable: true},
|
||||
{cidr: "168.63.129.16/32", public: true, reopenable: true},
|
||||
}
|
||||
|
||||
wantPrivate := make([]string, 0, len(tests))
|
||||
wantPublic := make([]string, 0, len(tests))
|
||||
|
||||
for _, tt := range tests {
|
||||
if tt.public {
|
||||
wantPublic = append(wantPublic, tt.cidr)
|
||||
} else {
|
||||
wantPrivate = append(wantPrivate, tt.cidr)
|
||||
}
|
||||
}
|
||||
|
||||
gotPrivate := make([]string, 0, len(tests))
|
||||
for _, n := range delivery.ExportBlockedNetworks() {
|
||||
gotPrivate = append(gotPrivate, n.String())
|
||||
}
|
||||
|
||||
gotPublic := make([]string, 0, len(tests))
|
||||
for _, n := range delivery.ExportBlockedPublicNetworks() {
|
||||
gotPublic = append(gotPublic, n.String())
|
||||
}
|
||||
|
||||
assert.ElementsMatch(t, wantPrivate, gotPrivate, "blockedNetworks")
|
||||
assert.ElementsMatch(t, wantPublic, gotPublic, "blockedPublicNetworks")
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.cidr, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
prefix := netip.MustParsePrefix(tt.cidr)
|
||||
ip := net.IP(prefix.Addr().AsSlice())
|
||||
|
||||
require.Error(t,
|
||||
delivery.NewTestGuard().ExportCheckIP(ip),
|
||||
"the default guard must refuse %s", ip,
|
||||
)
|
||||
|
||||
err := delivery.NewTestGuard(prefix).ExportCheckIP(ip)
|
||||
if tt.reopenable {
|
||||
assert.NoError(t, err, "listing %s must open it", tt.cidr)
|
||||
} else {
|
||||
assert.Error(t, err, "listing %s must not open it", tt.cidr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// requireLoopback fails the test unless rawURL's host is a
|
||||
// loopback address, so the allowlist test cannot silently stop
|
||||
// exercising a blocked range.
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
"sneak.berlin/go/webhooker/internal/delivery"
|
||||
)
|
||||
|
||||
func TestGuardCheckIP_PrivateRanges(t *testing.T) {
|
||||
func TestIsBlockedIP_PrivateRanges(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
@@ -56,14 +56,12 @@ func TestGuardCheckIP_PrivateRanges(t *testing.T) {
|
||||
"failed to parse IP %s", tt.ip,
|
||||
)
|
||||
|
||||
refused := delivery.NewTestGuard().ExportCheckIP(ip) != nil
|
||||
|
||||
assert.Equal(t,
|
||||
tt.blocked,
|
||||
refused,
|
||||
"default guard refuses %s = %v, want %v",
|
||||
delivery.ExportIsBlockedIP(ip),
|
||||
"isBlockedIP(%s) = %v, want %v",
|
||||
tt.ip,
|
||||
refused,
|
||||
delivery.ExportIsBlockedIP(ip),
|
||||
tt.blocked,
|
||||
)
|
||||
})
|
||||
@@ -101,42 +99,6 @@ func TestValidateTargetURL_Blocked(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestDefaultGuard_RefusesUnspecifiedMulticastAndDocumentation
|
||||
// covers the unspecified addresses and the IPv6 multicast and
|
||||
// documentation ranges: with no allowlist set, each is refused
|
||||
// both when a target is created and when a delivery dials it.
|
||||
func TestDefaultGuard_RefusesUnspecifiedMulticastAndDocumentation(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
guard := delivery.NewTestGuard()
|
||||
|
||||
targets := []string{
|
||||
// The unspecified addresses. On Linux a connection to
|
||||
// either reaches this host's loopback.
|
||||
"http://0.0.0.0:8080/hook",
|
||||
"http://[::]:8080/hook",
|
||||
// IPv6 multicast, all nodes.
|
||||
"http://[ff02::1]/hook",
|
||||
// IPv6 documentation.
|
||||
"http://[2001:db8::1]/hook",
|
||||
}
|
||||
|
||||
for _, target := range targets {
|
||||
t.Run(target, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
require.Error(t,
|
||||
guard.ValidateTargetURL(context.Background(), target),
|
||||
"%s must be refused at target creation", target,
|
||||
)
|
||||
|
||||
assertDialRefused(t, guard, target)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateTargetURL_Allowed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -12,75 +11,22 @@ import (
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
)
|
||||
|
||||
// archiveNameMaxLen is how many characters of a webhook or target
|
||||
// name an archive file name keeps.
|
||||
const archiveNameMaxLen = 40
|
||||
|
||||
// databaseTarget is a no-retry target that archives the full
|
||||
// inbound event into the target's own archive SQLite file, separate
|
||||
// from the per-webhook event database. The event is already
|
||||
// persisted in the per-webhook event DB by the time delivery runs;
|
||||
// the database target additionally writes a durable long-term copy
|
||||
// into the file ArchiveFileName names and then records a single
|
||||
// attempt whose outcome reflects whether the archive write
|
||||
// succeeded. See archiveWriter for the close/reopen, auto-recreate,
|
||||
// and expiry semantics.
|
||||
// databaseTarget is a no-retry target that archives the
|
||||
// full inbound event into a per-webhook archive SQLite file,
|
||||
// separate from the per-webhook event database. The event is
|
||||
// already persisted in the per-webhook event DB by the time
|
||||
// delivery runs; the database target additionally writes a
|
||||
// durable long-term copy into archive-{webhookID}.db and then
|
||||
// records a single attempt whose outcome reflects whether the
|
||||
// archive write succeeded. See archiveWriter for the
|
||||
// close/reopen, auto-recreate, and expiry semantics.
|
||||
type databaseTarget struct {
|
||||
eng *Engine
|
||||
|
||||
// writers holds one archive writer per database target, keyed
|
||||
// by target ID.
|
||||
mu sync.Mutex
|
||||
writers map[string]*archiveWriter
|
||||
}
|
||||
|
||||
// ArchiveFileName returns the file name of a database target's
|
||||
// archive: archive-WEBHOOKNAME-TARGETNAME-TARGETID.db, with both
|
||||
// names passed through archiveNamePart. The target ID keeps the
|
||||
// name unique when two targets' names come out the same.
|
||||
func ArchiveFileName(webhookName, targetName, targetID string) string {
|
||||
return "archive-" + archiveNamePart(webhookName) + "-" +
|
||||
archiveNamePart(targetName) + "-" + targetID + ".db"
|
||||
}
|
||||
|
||||
// archiveNamePart makes a webhook or target name safe to put in a
|
||||
// file name. It is lowercased; ASCII letters and digits are kept,
|
||||
// every other run of characters becomes a single "-", and no "-" is
|
||||
// left at either end. It is cut to archiveNameMaxLen characters, and
|
||||
// a name with nothing left is "unnamed".
|
||||
func archiveNamePart(name string) string {
|
||||
var b strings.Builder
|
||||
|
||||
dash := false
|
||||
|
||||
for _, r := range strings.ToLower(name) {
|
||||
if (r < 'a' || r > 'z') && (r < '0' || r > '9') {
|
||||
dash = b.Len() > 0
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if dash {
|
||||
b.WriteByte('-')
|
||||
|
||||
dash = false
|
||||
}
|
||||
|
||||
b.WriteRune(r)
|
||||
}
|
||||
|
||||
part := b.String()
|
||||
if len(part) > archiveNameMaxLen {
|
||||
part = strings.TrimRight(part[:archiveNameMaxLen], "-")
|
||||
}
|
||||
|
||||
if part == "" {
|
||||
return "unnamed"
|
||||
}
|
||||
|
||||
return part
|
||||
}
|
||||
|
||||
// Deliver implements Target. It archives the event, then
|
||||
// records one successful attempt and marks the delivery
|
||||
// delivered. An archiving error fails the delivery: the
|
||||
@@ -146,7 +92,7 @@ func (t *databaseTarget) Deliver(
|
||||
)
|
||||
}
|
||||
|
||||
// archive writes the full event as a row into the target's
|
||||
// archive writes the full event as a row into the webhook's
|
||||
// archive database, honouring the optional per-target expiry
|
||||
// parsed from the target config JSON.
|
||||
func (t *databaseTarget) archive(d *database.Delivery) error {
|
||||
@@ -160,7 +106,7 @@ func (t *databaseTarget) archive(d *database.Delivery) error {
|
||||
return err
|
||||
}
|
||||
|
||||
w, err := t.writerFor(d.TargetID)
|
||||
w, err := t.writerFor(webhookID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -178,31 +124,30 @@ func (t *databaseTarget) archive(d *database.Delivery) error {
|
||||
return w.write(row, expiry)
|
||||
}
|
||||
|
||||
// writerFor returns the archive writer for a database target,
|
||||
// creating and caching it on first use. Each target has one writer
|
||||
// so its close/reopen debounce state is shared across concurrent
|
||||
// deliveries, and so a rename and the idle sweep take the same lock
|
||||
// as its writes.
|
||||
// writerFor returns the archiveWriter for a webhook, creating
|
||||
// and caching it on first use. Each webhook has one writer so
|
||||
// its close/reopen debounce state is shared across concurrent
|
||||
// deliveries. The archive file lives beside the per-webhook
|
||||
// event database in the data directory.
|
||||
func (t *databaseTarget) writerFor(
|
||||
targetID string,
|
||||
webhookID string,
|
||||
) (*archiveWriter, error) {
|
||||
path, err := t.archivePath(webhookID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
|
||||
w, ok := t.writers[targetID]
|
||||
if t.writers == nil {
|
||||
t.writers = make(map[string]*archiveWriter)
|
||||
}
|
||||
|
||||
w, ok := t.writers[webhookID]
|
||||
if !ok {
|
||||
var err error
|
||||
|
||||
w, err = t.newWriter(targetID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if t.writers == nil {
|
||||
t.writers = make(map[string]*archiveWriter)
|
||||
}
|
||||
|
||||
t.writers[targetID] = w
|
||||
w = newArchiveWriter(path, t.eng.log)
|
||||
t.writers[webhookID] = w
|
||||
}
|
||||
|
||||
// A delivery claims the entry: even if the idle sweep created
|
||||
@@ -214,39 +159,40 @@ func (t *databaseTarget) writerFor(
|
||||
}
|
||||
|
||||
// sweepWriterFor returns the archive writer the idle sweep should
|
||||
// prune a target's archive through, together with whether the sweep
|
||||
// itself created the registry entry.
|
||||
// prune a webhook through, together with whether the sweep itself
|
||||
// created the registry entry.
|
||||
//
|
||||
// The sweep must route its prune through the registered writer so
|
||||
// the writer's mutex orders it against concurrent writes, but it
|
||||
// must never leave a registry entry behind: a sweep that ran
|
||||
// concurrently with the target's deletion would otherwise
|
||||
// concurrently with the webhook's deletion would otherwise
|
||||
// re-create an entry that nothing will ever evict again, which is
|
||||
// exactly the leak eviction exists to prevent. An entry the sweep
|
||||
// creates is therefore marked sweep-owned and handed back to
|
||||
// releaseSweepWriter when the sweep is done.
|
||||
func (t *databaseTarget) sweepWriterFor(
|
||||
targetID string,
|
||||
webhookID string,
|
||||
) (*archiveWriter, bool, error) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
|
||||
w, ok := t.writers[targetID]
|
||||
if ok {
|
||||
return w, false, nil
|
||||
}
|
||||
|
||||
w, err := t.newWriter(targetID)
|
||||
path, err := t.archivePath(webhookID)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
|
||||
if t.writers == nil {
|
||||
t.writers = make(map[string]*archiveWriter)
|
||||
}
|
||||
|
||||
w, ok := t.writers[webhookID]
|
||||
if ok {
|
||||
return w, false, nil
|
||||
}
|
||||
|
||||
w = newArchiveWriter(path, t.eng.log)
|
||||
w.sweepOwned = true
|
||||
t.writers[targetID] = w
|
||||
t.writers[webhookID] = w
|
||||
|
||||
return w, true, nil
|
||||
}
|
||||
@@ -263,95 +209,57 @@ func (t *databaseTarget) sweepWriterFor(
|
||||
// delivery that adopted the writer keeps a registered, evictable
|
||||
// one.
|
||||
func (t *databaseTarget) releaseSweepWriter(
|
||||
targetID string, w *archiveWriter,
|
||||
webhookID string, w *archiveWriter,
|
||||
) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
|
||||
cur, ok := t.writers[targetID]
|
||||
cur, ok := t.writers[webhookID]
|
||||
if !ok || cur != w || !cur.sweepOwned {
|
||||
return
|
||||
}
|
||||
|
||||
delete(t.writers, targetID)
|
||||
delete(t.writers, webhookID)
|
||||
}
|
||||
|
||||
// newWriter builds the writer for a database target's archive. The
|
||||
// file lives beside the webhook's event database in the data
|
||||
// directory and is named for the webhook and the target as the main
|
||||
// database has them now; from then on only rename changes the name
|
||||
// the writer uses. It does not touch the archive file.
|
||||
func (t *databaseTarget) newWriter(
|
||||
targetID string,
|
||||
) (*archiveWriter, error) {
|
||||
// archivePath returns the archive file path for a webhook: it
|
||||
// lives beside the per-webhook event database in the data
|
||||
// directory. It does not touch the filesystem.
|
||||
func (t *databaseTarget) archivePath(
|
||||
webhookID string,
|
||||
) (string, error) {
|
||||
if t.eng.dbManager == nil {
|
||||
return nil, errArchiveNoDataDir
|
||||
return "", errArchiveNoDataDir
|
||||
}
|
||||
|
||||
var target database.Target
|
||||
dir := filepath.Dir(t.eng.dbManager.DBPath(webhookID))
|
||||
|
||||
err := t.eng.database.DB().
|
||||
Preload("Webhook").
|
||||
First(&target, "id = ?", targetID).Error
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"loading database target %s: %w", targetID, err,
|
||||
)
|
||||
}
|
||||
|
||||
dir := filepath.Dir(t.eng.dbManager.DBPath(target.WebhookID))
|
||||
name := ArchiveFileName(
|
||||
target.Webhook.Name, target.Name, target.ID,
|
||||
)
|
||||
|
||||
w := newArchiveWriter(filepath.Join(dir, name), t.eng.log)
|
||||
w.webhookID = target.WebhookID
|
||||
|
||||
return w, nil
|
||||
return filepath.Join(
|
||||
dir, fmt.Sprintf("archive-%s.db", webhookID),
|
||||
), nil
|
||||
}
|
||||
|
||||
// rename moves a database target's archive file to the name for
|
||||
// webhookName and targetName. It goes through the target's writer,
|
||||
// so the move holds the lock that writes and the idle sweep take,
|
||||
// and later writes use the new name.
|
||||
//
|
||||
// The writer is created if there is none, and it stays cached. The
|
||||
// handlers rename before they save the new name, so until the save
|
||||
// the main database still has the old one; a delivery in that window
|
||||
// must find this writer rather than build one from the old name.
|
||||
func (t *databaseTarget) rename(
|
||||
targetID, webhookName, targetName string,
|
||||
) error {
|
||||
w, err := t.writerFor(targetID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return w.rename(ArchiveFileName(webhookName, targetName, targetID))
|
||||
}
|
||||
|
||||
// evict drops a database target's archive writer from the registry
|
||||
// and closes its handle, so a deleted target does not leave a
|
||||
// writer (and an open archive handle within its debounce window)
|
||||
// alive for the process lifetime.
|
||||
// evict drops a webhook's archive writer from the registry and
|
||||
// closes its handle, so a deleted webhook does not leave a
|
||||
// writer (and an open archive handle within its debounce
|
||||
// window) alive for the process lifetime.
|
||||
//
|
||||
// The map entry is removed under the registry lock, which is
|
||||
// then released before the handle is closed under the writer's
|
||||
// own lock: that ordering keeps the registry available to other
|
||||
// targets while an in-flight write on this one drains, and
|
||||
// webhooks while an in-flight write on this one drains, and
|
||||
// closing under the writer's lock means eviction can never race
|
||||
// a write.
|
||||
//
|
||||
// Eviction is idempotent and silent for a target with no writer,
|
||||
// which is the common case: only a database target that has
|
||||
// received an event or been renamed has one. It never deletes the
|
||||
// archive file.
|
||||
func (t *databaseTarget) evict(targetID string) {
|
||||
// Eviction is idempotent and silent for a webhook with no
|
||||
// writer, which is the common case: a webhook with no database
|
||||
// target never creates one. It never deletes the archive file.
|
||||
func (t *databaseTarget) evict(webhookID string) {
|
||||
t.mu.Lock()
|
||||
|
||||
w, ok := t.writers[targetID]
|
||||
w, ok := t.writers[webhookID]
|
||||
if ok {
|
||||
delete(t.writers, targetID)
|
||||
delete(t.writers, webhookID)
|
||||
}
|
||||
|
||||
t.mu.Unlock()
|
||||
@@ -364,41 +272,13 @@ func (t *databaseTarget) evict(targetID string) {
|
||||
|
||||
t.eng.log.Info(
|
||||
"evicted archive writer",
|
||||
"target_id", targetID,
|
||||
"webhook_id", webhookID,
|
||||
"path", w.path,
|
||||
)
|
||||
}
|
||||
|
||||
// evictWebhook evicts, exactly as evict does, the writer of every
|
||||
// database target of a webhook.
|
||||
func (t *databaseTarget) evictWebhook(webhookID string) {
|
||||
t.mu.Lock()
|
||||
|
||||
var gone []*archiveWriter
|
||||
|
||||
for targetID, w := range t.writers {
|
||||
if w.webhookID == webhookID {
|
||||
delete(t.writers, targetID)
|
||||
|
||||
gone = append(gone, w)
|
||||
}
|
||||
}
|
||||
|
||||
t.mu.Unlock()
|
||||
|
||||
for _, w := range gone {
|
||||
w.evict()
|
||||
|
||||
t.eng.log.Info(
|
||||
"evicted archive writer",
|
||||
"webhook_id", webhookID,
|
||||
"path", w.path,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// evictAll evicts every cached archive writer, exactly as evict
|
||||
// does for one target. The engine calls it at shutdown, once its
|
||||
// does for one webhook. The engine calls it at shutdown, once its
|
||||
// workers have returned. Closing the last handle on an archive
|
||||
// moves the contents of its -wal into the .db and removes the
|
||||
// -wal, so a clean stop leaves each archive as a single file.
|
||||
@@ -415,25 +295,38 @@ func (t *databaseTarget) evictAll() {
|
||||
}
|
||||
}
|
||||
|
||||
// sweepArchive prunes one database target's archive of rows older
|
||||
// than expiry, without requiring a write. A missing archive file is
|
||||
// left missing (see sweepExpired), so a sweep never creates an
|
||||
// archive for a target that has never received an event.
|
||||
// sweepWebhook prunes one webhook's archive of rows older than
|
||||
// expiry, without requiring a write. It returns nil (nothing to
|
||||
// do) when the archive file does not exist, so a sweep never
|
||||
// creates an archive for a webhook that has a database target
|
||||
// but has never received an event.
|
||||
//
|
||||
// It also never leaves a registry entry behind: an entry it had
|
||||
// to create to reach the writer's mutex is released again once
|
||||
// the prune is done, so a sweep racing a target deletion cannot
|
||||
// the prune is done, so a sweep racing a webhook deletion cannot
|
||||
// resurrect the writer the eviction just dropped.
|
||||
func (t *databaseTarget) sweepArchive(
|
||||
targetID string, expiry time.Duration,
|
||||
func (t *databaseTarget) sweepWebhook(
|
||||
webhookID string, expiry time.Duration,
|
||||
) error {
|
||||
w, created, err := t.sweepWriterFor(targetID)
|
||||
path, err := t.archivePath(webhookID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check before taking a writer at all: a webhook whose
|
||||
// archive has never been created gets no writer, no handle,
|
||||
// and no file.
|
||||
if !fileExists(path) {
|
||||
return nil
|
||||
}
|
||||
|
||||
w, created, err := t.sweepWriterFor(webhookID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if created {
|
||||
defer t.releaseSweepWriter(targetID, w)
|
||||
defer t.releaseSweepWriter(webhookID, w)
|
||||
}
|
||||
|
||||
return w.sweepExpired(expiry)
|
||||
|
||||
@@ -4,10 +4,8 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -43,7 +41,7 @@ const (
|
||||
|
||||
var (
|
||||
// errArchiveMissingWebhookID is returned when an event to
|
||||
// archive has no webhook id to record in its archive row.
|
||||
// archive has no webhook id to key its archive file on.
|
||||
errArchiveMissingWebhookID = errors.New(
|
||||
"cannot archive event without a webhook id",
|
||||
)
|
||||
@@ -63,19 +61,13 @@ var (
|
||||
)
|
||||
|
||||
// errArchiveWriterEvicted is returned when a writer that has
|
||||
// been evicted (its target or its webhook was deleted) is used
|
||||
// again. An evicted writer is no longer in the registry, so
|
||||
// reopening its file would leak a handle nothing owns.
|
||||
// been evicted (its webhook was deleted, or its last database
|
||||
// target was removed) is used again. An evicted writer is no
|
||||
// longer in the registry, so reopening its file would leak a
|
||||
// handle nothing owns.
|
||||
errArchiveWriterEvicted = errors.New(
|
||||
"archive writer has been evicted",
|
||||
)
|
||||
|
||||
// ErrArchiveNameTaken is returned when an archive cannot be
|
||||
// renamed because a file already has the new name. That file may
|
||||
// be an archive with rows of its own, so it is never replaced.
|
||||
ErrArchiveNameTaken = errors.New(
|
||||
"a file already has the archive's new name",
|
||||
)
|
||||
)
|
||||
|
||||
// databaseTargetConfig is the optional per-target JSON config
|
||||
@@ -88,7 +80,7 @@ type databaseTargetConfig struct {
|
||||
}
|
||||
|
||||
// archivedEvent is one fully captured webhook event stored in a
|
||||
// database target's archive for long-term retention. It is a
|
||||
// per-webhook archive database for long-term retention. It is a
|
||||
// self-contained copy — independent of the per-webhook event
|
||||
// database, which may prune events under its own retention.
|
||||
type archivedEvent struct {
|
||||
@@ -178,8 +170,8 @@ func ValidateArchiveExpiry(expiry string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// archiveWriter owns one database target's archive SQLite file.
|
||||
// It serialises writes, and after each write closes and reopens
|
||||
// archiveWriter owns one per-webhook archive SQLite file. It
|
||||
// serialises writes, and after each write closes and reopens
|
||||
// the file (debounced to at most once per debounce window) so
|
||||
// an operator can move the file away for offline archiving. The
|
||||
// next write recreates a moved or removed file, because the
|
||||
@@ -195,21 +187,16 @@ type archiveWriter struct {
|
||||
reopens int
|
||||
|
||||
// evicted marks a writer that has been removed from the
|
||||
// registry. Its handle is closed and it must never open the
|
||||
// file again: nothing holds it any more, so a reopen would
|
||||
// leak the handle for the process lifetime.
|
||||
// per-webhook registry. Its handle is closed and it must
|
||||
// never open the file again: nothing holds it any more, so a
|
||||
// reopen would leak the handle for the process lifetime.
|
||||
evicted bool
|
||||
|
||||
// webhookID is the webhook the archive's target belongs to,
|
||||
// so deleting the webhook can find its writers. It is set
|
||||
// when the writer is created and never changes.
|
||||
webhookID string
|
||||
|
||||
// sweepOwned marks a registry entry that the idle sweep
|
||||
// created because no writer was cached for the target. The
|
||||
// created because no writer was cached for the webhook. The
|
||||
// sweep removes such an entry again when it is done, so a
|
||||
// sweep can never leave — or resurrect — a registry entry
|
||||
// for a target that has been deleted. A delivery that adopts
|
||||
// for a webhook that has been deleted. A delivery that adopts
|
||||
// the writer clears the flag, handing the entry to the
|
||||
// registry proper.
|
||||
//
|
||||
@@ -398,78 +385,11 @@ func (w *archiveWriter) sweepExpired(expiry time.Duration) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// rename gives the archive file a new name in the same directory,
|
||||
// and the writer uses the file under that name from now on. The
|
||||
// handle is closed first, which folds the -wal into the .db; any
|
||||
// -wal or -shm still beside the file (left by a crash) is moved with
|
||||
// it, because SQLite finds them by name. A missing file is not an
|
||||
// error: the operator may have moved it away, and the next write
|
||||
// creates it under the new name.
|
||||
//
|
||||
// If a file already has the new name, nothing is moved and the
|
||||
// error is ErrArchiveNameTaken. If one file fails to move, those
|
||||
// already moved are moved back before the error is returned, so the
|
||||
// archive is never split across two names.
|
||||
func (w *archiveWriter) rename(name string) error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
|
||||
if w.evicted {
|
||||
return fmt.Errorf(
|
||||
"%w: %s", errArchiveWriterEvicted, w.path,
|
||||
)
|
||||
}
|
||||
|
||||
path := filepath.Join(filepath.Dir(w.path), name)
|
||||
if path == w.path {
|
||||
return nil
|
||||
}
|
||||
|
||||
suffixes := []string{"", "-wal", "-shm"}
|
||||
|
||||
for _, suffix := range suffixes {
|
||||
if fileExists(path + suffix) {
|
||||
return fmt.Errorf(
|
||||
"%w: %s", ErrArchiveNameTaken, name+suffix,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
w.close()
|
||||
|
||||
for i, suffix := range suffixes {
|
||||
err := os.Rename(w.path+suffix, path+suffix)
|
||||
if err == nil || errors.Is(err, fs.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, moved := range suffixes[:i] {
|
||||
backErr := os.Rename(path+moved, w.path+moved)
|
||||
if backErr != nil && !errors.Is(backErr, fs.ErrNotExist) {
|
||||
w.log.Error(
|
||||
"failed to move archive file back",
|
||||
"from", path+moved,
|
||||
"to", w.path+moved,
|
||||
"error", backErr,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf(
|
||||
"renaming archive %s to %s: %w", w.path+suffix, path+suffix, err,
|
||||
)
|
||||
}
|
||||
|
||||
w.path = path
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// evict closes the writer's handle and marks it unusable. It is
|
||||
// called when the writer leaves the registry, because its target
|
||||
// or its webhook was deleted, or at shutdown. The archive FILE is
|
||||
// deliberately left on disk: it is long-term storage an operator
|
||||
// may still want.
|
||||
// called when the writer leaves the registry, either because the
|
||||
// webhook was deleted or because its last database target was
|
||||
// removed. The archive FILE is deliberately left on disk: it is
|
||||
// long-term storage an operator may still want.
|
||||
func (w *archiveWriter) evict() {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
|
||||
@@ -17,109 +17,85 @@ import (
|
||||
"sneak.berlin/go/webhooker/internal/delivery"
|
||||
)
|
||||
|
||||
// deliverTo archives one event to a database target, leaving the
|
||||
// target's writer cached with its handle open.
|
||||
func deliverTo(
|
||||
t *testing.T, env *archiveEnv, tgt *database.Target,
|
||||
) {
|
||||
// evictTestEngine builds an engine backed by a temporary data
|
||||
// directory and returns it along with that directory.
|
||||
func evictTestEngine(t *testing.T) (*delivery.Engine, string) {
|
||||
t.Helper()
|
||||
|
||||
webhookDB := testWebhookDB(t)
|
||||
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
||||
dataDir := t.TempDir()
|
||||
|
||||
env.eng.ExportDeliverDatabase(
|
||||
webhookDB, seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
|
||||
eng := delivery.NewTestEngineWithDB(
|
||||
nil,
|
||||
database.NewTestWebhookDBManager(dataDir),
|
||||
archiveTestLogger(),
|
||||
&http.Client{Timeout: 5 * time.Second},
|
||||
1,
|
||||
)
|
||||
|
||||
return eng, dataDir
|
||||
}
|
||||
|
||||
// TestEvictWebhook_ClosesAndRemovesWriter proves that evicting
|
||||
// a webhook drops the archive writers of its database targets
|
||||
// from the registry and closes their open handles, rather than
|
||||
// leaving them alive for the process lifetime, and leaves another
|
||||
// webhook's writer alone.
|
||||
// a webhook drops its archive writer from the registry and
|
||||
// closes the open archive handle, rather than leaving both
|
||||
// alive for the process lifetime.
|
||||
func TestEvictWebhook_ClosesAndRemovesWriter(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
first := env.seedDatabaseTarget(t, "")
|
||||
second := env.addDatabaseTarget(t, first.WebhookID, "")
|
||||
other := env.seedDatabaseTarget(t, "")
|
||||
eng, dataDir := evictTestEngine(t)
|
||||
|
||||
for _, tgt := range []*database.Target{first, second, other} {
|
||||
deliverTo(t, env, tgt)
|
||||
webhookDB := testWebhookDB(t)
|
||||
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
|
||||
|
||||
require.True(
|
||||
t, env.eng.ExportArchiveHandleOpen(tgt.ID),
|
||||
"the writer should hold an open handle after a write",
|
||||
)
|
||||
}
|
||||
eng.ExportDeliverDatabase(webhookDB, d)
|
||||
|
||||
env.eng.EvictWebhook(first.WebhookID)
|
||||
webhookID := event.WebhookID
|
||||
|
||||
for _, tgt := range []*database.Target{first, second} {
|
||||
assert.False(
|
||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||
"eviction should remove the registry entry",
|
||||
)
|
||||
assert.False(
|
||||
t, env.eng.ExportArchiveHandleOpen(tgt.ID),
|
||||
"eviction should close the archive handle",
|
||||
)
|
||||
assert.FileExists(
|
||||
t, env.archivePath(tgt),
|
||||
"eviction must not delete the archive file",
|
||||
)
|
||||
}
|
||||
|
||||
assert.True(
|
||||
t, env.eng.ExportArchiveHandleOpen(other.ID),
|
||||
"another webhook's writer must be left alone",
|
||||
require.True(
|
||||
t, eng.ExportHasArchiveWriter(webhookID),
|
||||
"a delivery should have cached an archive writer",
|
||||
)
|
||||
require.True(
|
||||
t, eng.ExportArchiveHandleOpen(webhookID),
|
||||
"the writer should hold an open handle after a write",
|
||||
)
|
||||
}
|
||||
|
||||
// TestEvictTarget_LeavesOtherTargets proves that evicting one
|
||||
// database target leaves the writer of another target of the same
|
||||
// webhook in place.
|
||||
func TestEvictTarget_LeavesOtherTargets(t *testing.T) {
|
||||
t.Parallel()
|
||||
eng.EvictWebhook(webhookID)
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
doomed := env.seedDatabaseTarget(t, "")
|
||||
kept := env.addDatabaseTarget(t, doomed.WebhookID, "")
|
||||
assert.False(
|
||||
t, eng.ExportHasArchiveWriter(webhookID),
|
||||
"eviction should remove the registry entry",
|
||||
)
|
||||
assert.False(
|
||||
t, eng.ExportArchiveHandleOpen(webhookID),
|
||||
"eviction should close the archive handle",
|
||||
)
|
||||
|
||||
deliverTo(t, env, doomed)
|
||||
deliverTo(t, env, kept)
|
||||
|
||||
env.eng.EvictTarget(doomed.ID)
|
||||
|
||||
assert.False(t, env.eng.ExportHasArchiveWriter(doomed.ID))
|
||||
archivePath := filepath.Join(
|
||||
dataDir, fmt.Sprintf("archive-%s.db", webhookID),
|
||||
)
|
||||
assert.FileExists(
|
||||
t, env.archivePath(doomed),
|
||||
t, archivePath,
|
||||
"eviction must not delete the archive file",
|
||||
)
|
||||
assert.True(
|
||||
t, env.eng.ExportArchiveHandleOpen(kept.ID),
|
||||
"the other target's writer must be left alone",
|
||||
)
|
||||
}
|
||||
|
||||
// TestEvictWebhook_UnknownWebhookIsNoOp proves eviction is safe
|
||||
// for the common case of a webhook or target that never had an
|
||||
// archive writer, and that repeating it does not panic.
|
||||
// for the common case of a webhook that never had a database
|
||||
// target, and that repeating it does not panic.
|
||||
func TestEvictWebhook_UnknownWebhookIsNoOp(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
eng, _ := evictTestEngine(t)
|
||||
|
||||
assert.NotPanics(t, func() {
|
||||
env.eng.EvictWebhook("no-such-webhook")
|
||||
env.eng.EvictWebhook("no-such-webhook")
|
||||
env.eng.EvictTarget("no-such-target")
|
||||
env.eng.EvictTarget("no-such-target")
|
||||
eng.EvictWebhook("no-such-webhook")
|
||||
eng.EvictWebhook("no-such-webhook")
|
||||
})
|
||||
|
||||
assert.False(
|
||||
t, env.eng.ExportHasArchiveWriter("no-such-target"),
|
||||
t, eng.ExportHasArchiveWriter("no-such-webhook"),
|
||||
"eviction must not create a writer",
|
||||
)
|
||||
}
|
||||
@@ -313,14 +289,17 @@ func TestEvictWebhook_RacingWriteDoesNotReopenHandle(
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, "")
|
||||
eng, _ := evictTestEngine(t)
|
||||
|
||||
webhookDB := testWebhookDB(t)
|
||||
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
|
||||
|
||||
// Prime the registry so the test can hold the very writer the
|
||||
// eviction is about to detach.
|
||||
deliverTo(t, env, tgt)
|
||||
eng.ExportDeliverDatabase(webhookDB, d)
|
||||
|
||||
w := env.eng.ExportArchiveWriterFor(tgt.ID)
|
||||
w := eng.ExportArchiveWriterFor(event.WebhookID)
|
||||
require.NotNil(t, w)
|
||||
require.True(t, w.HandleOpen())
|
||||
|
||||
@@ -330,7 +309,7 @@ func TestEvictWebhook_RacingWriteDoesNotReopenHandle(
|
||||
// eviction has to contend for the writer's mutex.
|
||||
race.awaitFirstWrite()
|
||||
|
||||
env.eng.EvictWebhook(tgt.WebhookID)
|
||||
eng.EvictWebhook(event.WebhookID)
|
||||
|
||||
sawEvicted, otherErr := race.wait()
|
||||
|
||||
@@ -345,33 +324,41 @@ func TestEvictWebhook_RacingWriteDoesNotReopenHandle(
|
||||
"been evicted",
|
||||
)
|
||||
assert.False(
|
||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||
t, eng.ExportHasArchiveWriter(event.WebhookID),
|
||||
"the registry entry must stay gone",
|
||||
)
|
||||
}
|
||||
|
||||
// TestEvictWebhook_LaterDeliveryRecreatesWriter proves eviction
|
||||
// does not break archiving for a target that is still alive: a
|
||||
// does not break archiving for a webhook that is still alive: a
|
||||
// subsequent delivery gets a brand new writer from the registry.
|
||||
// It says nothing about the evicted writer itself — that is what
|
||||
// TestEvictedWriter_WriteDoesNotReopenFile covers.
|
||||
func TestEvictWebhook_LaterDeliveryRecreatesWriter(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, "")
|
||||
eng, _ := evictTestEngine(t)
|
||||
|
||||
deliverTo(t, env, tgt)
|
||||
require.True(t, env.eng.ExportHasArchiveWriter(tgt.ID))
|
||||
webhookDB := testWebhookDB(t)
|
||||
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
|
||||
|
||||
env.eng.EvictWebhook(tgt.WebhookID)
|
||||
eng.ExportDeliverDatabase(webhookDB, d)
|
||||
require.True(
|
||||
t, eng.ExportHasArchiveWriter(event.WebhookID),
|
||||
)
|
||||
|
||||
// A fresh delivery for the same target gets a brand new
|
||||
eng.EvictWebhook(event.WebhookID)
|
||||
|
||||
// A fresh delivery for the same webhook gets a brand new
|
||||
// writer from the registry, so archiving keeps working.
|
||||
deliverTo(t, env, tgt)
|
||||
second := seedDatabaseTargetDelivery(
|
||||
t, webhookDB, event, "",
|
||||
)
|
||||
eng.ExportDeliverDatabase(webhookDB, second)
|
||||
|
||||
assert.True(
|
||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||
t, eng.ExportHasArchiveWriter(event.WebhookID),
|
||||
"a later delivery should recreate the writer",
|
||||
)
|
||||
}
|
||||
@@ -383,16 +370,19 @@ func TestEvictWebhook_LaterDeliveryRecreatesWriter(t *testing.T) {
|
||||
func TestEngineStop_WriteAfterStopIsRefused(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, "")
|
||||
eng, _ := evictTestEngine(t)
|
||||
|
||||
deliverTo(t, env, tgt)
|
||||
webhookDB := testWebhookDB(t)
|
||||
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
|
||||
|
||||
w := env.eng.ExportArchiveWriterFor(tgt.ID)
|
||||
eng.ExportDeliverDatabase(webhookDB, d)
|
||||
|
||||
w := eng.ExportArchiveWriterFor(event.WebhookID)
|
||||
require.NotNil(t, w)
|
||||
require.True(t, w.HandleOpen())
|
||||
|
||||
require.NoError(t, env.eng.ExportStop(context.Background()))
|
||||
require.NoError(t, eng.ExportStop(context.Background()))
|
||||
|
||||
err := w.Write(evictTestRow("ev-after-stop"), 0)
|
||||
|
||||
@@ -405,7 +395,7 @@ func TestEngineStop_WriteAfterStopIsRefused(t *testing.T) {
|
||||
"a refused write must not reopen the archive",
|
||||
)
|
||||
assert.False(
|
||||
t, env.eng.ExportHasArchiveWriter(tgt.ID),
|
||||
t, eng.ExportHasArchiveWriter(event.WebhookID),
|
||||
"the stop should empty the registry",
|
||||
)
|
||||
|
||||
|
||||
@@ -4,12 +4,13 @@ import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/sqlite"
|
||||
@@ -73,18 +74,25 @@ func removeArchiveFiles(t *testing.T, path string) {
|
||||
|
||||
// TestDeliverDatabase_ArchivesEvent verifies that delivering to
|
||||
// a database target marks the delivery delivered and archives
|
||||
// the full event into the target's own archive file.
|
||||
// the full event into a separate per-webhook archive file.
|
||||
func TestDeliverDatabase_ArchivesEvent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, "")
|
||||
dataDir := t.TempDir()
|
||||
dbMgr := database.NewTestWebhookDBManager(dataDir)
|
||||
|
||||
e := delivery.NewTestEngineWithDB(
|
||||
nil, dbMgr,
|
||||
archiveTestLogger(),
|
||||
&http.Client{Timeout: 5 * time.Second},
|
||||
1,
|
||||
)
|
||||
|
||||
webhookDB := testWebhookDB(t)
|
||||
event := seedEvent(t, webhookDB, `{"archived":true}`)
|
||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
|
||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, "")
|
||||
|
||||
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||
e.ExportDeliverDatabase(webhookDB, d)
|
||||
|
||||
var updated database.Delivery
|
||||
|
||||
@@ -97,7 +105,8 @@ func TestDeliverDatabase_ArchivesEvent(t *testing.T) {
|
||||
)
|
||||
|
||||
archivePath := filepath.Join(
|
||||
env.dataDir, "archive-sweep-test-archive-"+tgt.ID+".db",
|
||||
dataDir,
|
||||
fmt.Sprintf("archive-%s.db", event.WebhookID),
|
||||
)
|
||||
assert.FileExists(t, archivePath)
|
||||
|
||||
@@ -279,31 +288,31 @@ func TestParseArchiveExpiry(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// seedDatabaseTargetDelivery seeds a pending delivery of an event
|
||||
// to a database target and returns the in-memory delivery the
|
||||
// target handler is invoked with.
|
||||
// seedDatabaseTargetDelivery seeds a pending delivery for a
|
||||
// database target with the given config JSON and returns the
|
||||
// in-memory delivery the target handler is invoked with.
|
||||
func seedDatabaseTargetDelivery(
|
||||
t *testing.T,
|
||||
webhookDB *gorm.DB,
|
||||
event database.Event,
|
||||
tgt *database.Target,
|
||||
config string,
|
||||
) *database.Delivery {
|
||||
t.Helper()
|
||||
|
||||
dlv := seedDelivery(
|
||||
t, webhookDB, event.ID, tgt.ID,
|
||||
t, webhookDB, event.ID, uuid.New().String(),
|
||||
database.DeliveryStatusPending,
|
||||
)
|
||||
|
||||
d := &database.Delivery{
|
||||
EventID: event.ID,
|
||||
TargetID: tgt.ID,
|
||||
TargetID: dlv.TargetID,
|
||||
Status: database.DeliveryStatusPending,
|
||||
Event: event,
|
||||
Target: database.Target{
|
||||
Name: tgt.Name,
|
||||
Name: "test-db",
|
||||
Type: database.TargetTypeDatabase,
|
||||
Config: tgt.Config,
|
||||
Config: config,
|
||||
},
|
||||
}
|
||||
d.ID = dlv.ID
|
||||
@@ -321,14 +330,22 @@ func TestDeliverDatabase_ArchiveFailureFailsDelivery(
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, `{"expiry":"nonsense"}`)
|
||||
dataDir := t.TempDir()
|
||||
|
||||
e := delivery.NewTestEngineWithDB(
|
||||
nil, database.NewTestWebhookDBManager(dataDir),
|
||||
archiveTestLogger(),
|
||||
&http.Client{Timeout: 5 * time.Second},
|
||||
1,
|
||||
)
|
||||
|
||||
webhookDB := testWebhookDB(t)
|
||||
event := seedEvent(t, webhookDB, `{"archived":false}`)
|
||||
d := seedDatabaseTargetDelivery(t, webhookDB, event, tgt)
|
||||
d := seedDatabaseTargetDelivery(
|
||||
t, webhookDB, event, `{"expiry":"nonsense"}`,
|
||||
)
|
||||
|
||||
env.eng.ExportDeliverDatabase(webhookDB, d)
|
||||
e.ExportDeliverDatabase(webhookDB, d)
|
||||
|
||||
var updated database.Delivery
|
||||
|
||||
@@ -356,7 +373,10 @@ func TestDeliverDatabase_ArchiveFailureFailsDelivery(
|
||||
)
|
||||
|
||||
assert.NoFileExists(t,
|
||||
env.archivePath(tgt),
|
||||
filepath.Join(
|
||||
dataDir,
|
||||
fmt.Sprintf("archive-%s.db", event.WebhookID),
|
||||
),
|
||||
"no archive file should exist for a failed config",
|
||||
)
|
||||
}
|
||||
@@ -380,290 +400,3 @@ func TestValidateArchiveExpiry(t *testing.T) {
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// TestArchiveFileName pins the archive file name and the rules
|
||||
// that make a webhook or target name safe to put in it.
|
||||
func TestArchiveFileName(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const id = "3f2a1c9e-8d4b-4c1a-9e2f-0a1b2c3d4e5f"
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
webhook string
|
||||
target string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
"plain names", "orders", "archive",
|
||||
"archive-orders-archive-" + id + ".db",
|
||||
},
|
||||
{
|
||||
"lowercased", "Orders", "Main Archive",
|
||||
"archive-orders-main-archive-" + id + ".db",
|
||||
},
|
||||
{
|
||||
"a run of other characters is one dash",
|
||||
`a /\..b`, "c__--d",
|
||||
"archive-a-b-c-d-" + id + ".db",
|
||||
},
|
||||
{
|
||||
"no dash at either end", " --orders!! ", "(archive)",
|
||||
"archive-orders-archive-" + id + ".db",
|
||||
},
|
||||
{
|
||||
"path separators", "../../etc/passwd", "a/b",
|
||||
"archive-etc-passwd-a-b-" + id + ".db",
|
||||
},
|
||||
{
|
||||
"letters outside ASCII are dropped",
|
||||
"Bestellungen Größe", "café",
|
||||
"archive-bestellungen-gr-e-caf-" + id + ".db",
|
||||
},
|
||||
{
|
||||
"nothing left is unnamed", "", "!!!",
|
||||
"archive-unnamed-unnamed-" + id + ".db",
|
||||
},
|
||||
{
|
||||
"cut to 40 characters", strings.Repeat("a", 50), "x",
|
||||
"archive-" + strings.Repeat("a", 40) + "-x-" + id + ".db",
|
||||
},
|
||||
{
|
||||
"no dash left by the cut",
|
||||
strings.Repeat("a", 39) + " b", "x",
|
||||
"archive-" + strings.Repeat("a", 39) + "-x-" + id + ".db",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
assert.Equal(
|
||||
t, tc.want,
|
||||
delivery.ArchiveFileName(tc.webhook, tc.target, id),
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeliverDatabase_EachTargetHasItsOwnArchive proves two
|
||||
// database targets of one webhook archive into separate files.
|
||||
func TestDeliverDatabase_EachTargetHasItsOwnArchive(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
first := env.seedDatabaseTarget(t, "")
|
||||
second := env.addDatabaseTarget(t, first.WebhookID, "")
|
||||
|
||||
webhookDB := testWebhookDB(t)
|
||||
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||
|
||||
for _, tgt := range []*database.Target{first, second} {
|
||||
env.eng.ExportDeliverDatabase(
|
||||
webhookDB,
|
||||
seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
|
||||
)
|
||||
}
|
||||
|
||||
require.NotEqual(
|
||||
t, env.archivePath(first), env.archivePath(second),
|
||||
)
|
||||
assert.Equal(
|
||||
t, []string{event.ID},
|
||||
archivedEventIDs(t, env.archivePath(first)),
|
||||
)
|
||||
assert.Equal(
|
||||
t, []string{event.ID},
|
||||
archivedEventIDs(t, env.archivePath(second)),
|
||||
)
|
||||
}
|
||||
|
||||
// TestRename_MovesTheFile proves a rename moves the archive, rows
|
||||
// and all, and that later writes go to the new name.
|
||||
func TestRename_MovesTheFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, "")
|
||||
oldPath := env.archivePath(tgt)
|
||||
|
||||
webhookDB := testWebhookDB(t)
|
||||
first := seedEvent(t, webhookDB, `{"n":1}`)
|
||||
env.eng.ExportDeliverDatabase(
|
||||
webhookDB, seedDatabaseTargetDelivery(t, webhookDB, first, tgt),
|
||||
)
|
||||
require.FileExists(t, oldPath)
|
||||
|
||||
require.NoError(
|
||||
t, env.eng.Rename(tgt.ID, "Orders", "Long Term"),
|
||||
)
|
||||
|
||||
newPath := filepath.Join(
|
||||
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
||||
)
|
||||
|
||||
assert.NoFileExists(t, oldPath)
|
||||
assert.Equal(t, []string{first.ID}, archivedEventIDs(t, newPath))
|
||||
|
||||
second := seedEvent(t, webhookDB, `{"n":2}`)
|
||||
env.eng.ExportDeliverDatabase(
|
||||
webhookDB,
|
||||
seedDatabaseTargetDelivery(t, webhookDB, second, tgt),
|
||||
)
|
||||
|
||||
assert.ElementsMatch(
|
||||
t, []string{first.ID, second.ID},
|
||||
archivedEventIDs(t, newPath),
|
||||
)
|
||||
assert.NoFileExists(
|
||||
t, oldPath, "a write after the rename must use the new name",
|
||||
)
|
||||
}
|
||||
|
||||
// TestRename_NeverReplacesAFile plants a file at the new name, once
|
||||
// the .db alone, once a lone -wal and once a lone -shm, and proves
|
||||
// each time that the rename is refused, the planted file survives,
|
||||
// and the archive keeps its name and its rows.
|
||||
func TestRename_NeverReplacesAFile(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, suffix := range archiveFileSuffixes() {
|
||||
t.Run("planted .db"+suffix, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, "")
|
||||
oldPath := env.archivePath(tgt)
|
||||
|
||||
webhookDB := testWebhookDB(t)
|
||||
first := seedEvent(t, webhookDB, `{"n":1}`)
|
||||
env.eng.ExportDeliverDatabase(
|
||||
webhookDB,
|
||||
seedDatabaseTargetDelivery(t, webhookDB, first, tgt),
|
||||
)
|
||||
|
||||
newPath := filepath.Join(
|
||||
env.dataDir, "archive-orders-long-term-"+tgt.ID+".db",
|
||||
)
|
||||
plantedPath := newPath + suffix
|
||||
require.NoError(
|
||||
t, os.WriteFile(plantedPath, []byte("planted"), 0o600),
|
||||
)
|
||||
|
||||
require.ErrorIs(
|
||||
t, env.eng.Rename(tgt.ID, "Orders", "Long Term"),
|
||||
delivery.ErrArchiveNameTaken,
|
||||
)
|
||||
|
||||
//nolint:gosec // reads the file the test planted under t.TempDir()
|
||||
planted, err := os.ReadFile(plantedPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "planted", string(planted))
|
||||
|
||||
second := seedEvent(t, webhookDB, `{"n":2}`)
|
||||
env.eng.ExportDeliverDatabase(
|
||||
webhookDB,
|
||||
seedDatabaseTargetDelivery(t, webhookDB, second, tgt),
|
||||
)
|
||||
|
||||
assert.ElementsMatch(
|
||||
t, []string{first.ID, second.ID},
|
||||
archivedEventIDs(t, oldPath),
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRename_BeforeTheNameIsSaved covers the order the handlers
|
||||
// use: they rename before they save the new name, so a delivery in
|
||||
// between must write under the new name although the main database
|
||||
// still has the old one. It also shows that renaming an archive that
|
||||
// does not exist yet is not an error.
|
||||
func TestRename_BeforeTheNameIsSaved(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupArchiveTest(t)
|
||||
tgt := env.seedDatabaseTarget(t, "")
|
||||
|
||||
require.NoError(
|
||||
t, env.eng.Rename(tgt.ID, "Orders", "Archive"),
|
||||
)
|
||||
|
||||
webhookDB := testWebhookDB(t)
|
||||
event := seedEvent(t, webhookDB, `{"n":1}`)
|
||||
env.eng.ExportDeliverDatabase(
|
||||
webhookDB, seedDatabaseTargetDelivery(t, webhookDB, event, tgt),
|
||||
)
|
||||
|
||||
assert.FileExists(
|
||||
t,
|
||||
filepath.Join(
|
||||
env.dataDir, "archive-orders-archive-"+tgt.ID+".db",
|
||||
),
|
||||
)
|
||||
assert.NoFileExists(t, env.archivePath(tgt))
|
||||
}
|
||||
|
||||
// TestArchiveWriter_RenameMovesSidecars proves a rename carries
|
||||
// the -wal and -shm a crash can leave beside an archive no handle
|
||||
// has opened since. SQLite finds them by name, so a -wal left
|
||||
// behind would lose the transactions it holds.
|
||||
func TestArchiveWriter_RenameMovesSidecars(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
oldPath := filepath.Join(dir, "archive-old.db")
|
||||
newPath := filepath.Join(dir, "archive-new.db")
|
||||
|
||||
for _, suffix := range archiveFileSuffixes() {
|
||||
require.NoError(
|
||||
t, os.WriteFile(oldPath+suffix, []byte(suffix), 0o600),
|
||||
)
|
||||
}
|
||||
|
||||
w := delivery.NewExportArchiveWriter(
|
||||
oldPath, archiveTestLogger(), 0,
|
||||
)
|
||||
|
||||
require.NoError(t, w.Rename("archive-new.db"))
|
||||
|
||||
for _, suffix := range archiveFileSuffixes() {
|
||||
assert.NoFileExists(t, oldPath+suffix)
|
||||
assert.FileExists(t, newPath+suffix)
|
||||
}
|
||||
|
||||
assert.Equal(t, newPath, w.Path())
|
||||
}
|
||||
|
||||
// TestArchiveWriter_RenameMovesBackOnFailure makes the -wal fail to
|
||||
// move after the .db has moved, and proves the .db is moved back, so
|
||||
// the archive is never split across two names. The new name is 255
|
||||
// bytes, the longest a file name may be, so the .db can take it but
|
||||
// the -wal, four bytes longer, cannot.
|
||||
func TestArchiveWriter_RenameMovesBackOnFailure(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
dir := t.TempDir()
|
||||
oldPath := filepath.Join(dir, "archive-old.db")
|
||||
newName := strings.Repeat("a", 252) + ".db"
|
||||
|
||||
for _, suffix := range archiveFileSuffixes() {
|
||||
require.NoError(
|
||||
t, os.WriteFile(oldPath+suffix, []byte(suffix), 0o600),
|
||||
)
|
||||
}
|
||||
|
||||
w := delivery.NewExportArchiveWriter(
|
||||
oldPath, archiveTestLogger(), 0,
|
||||
)
|
||||
|
||||
require.Error(t, w.Rename(newName))
|
||||
|
||||
for _, suffix := range archiveFileSuffixes() {
|
||||
assert.FileExists(t, oldPath+suffix)
|
||||
}
|
||||
|
||||
assert.NoFileExists(t, filepath.Join(dir, newName))
|
||||
assert.Equal(t, oldPath, w.Path())
|
||||
}
|
||||
|
||||
@@ -442,9 +442,7 @@ func (t *httpTarget) doHTTPRequest(
|
||||
)
|
||||
}
|
||||
|
||||
originScoped := applyRequestHeaders(
|
||||
req, event, cfg, t.eng.userAgent(),
|
||||
)
|
||||
originScoped := applyRequestHeaders(req, event, cfg)
|
||||
|
||||
client := t.clientForRequest(cfg, originScoped)
|
||||
|
||||
@@ -564,13 +562,10 @@ func isForwardableHeader(name string) bool {
|
||||
// Content-Type goes out once: a Content-Type configured on the target
|
||||
// wins, otherwise the event's ContentType, otherwise none. The inbound
|
||||
// Content-Type in the event's headers is never forwarded.
|
||||
//
|
||||
// userAgent is set last, over any configured or inbound User-Agent.
|
||||
func applyRequestHeaders(
|
||||
req *http.Request,
|
||||
event *database.Event,
|
||||
cfg *HTTPTargetConfig,
|
||||
userAgent string,
|
||||
) []string {
|
||||
if event.ContentType != "" {
|
||||
req.Header.Set(
|
||||
@@ -585,7 +580,7 @@ func applyRequestHeaders(
|
||||
originScoped[http.CanonicalHeaderKey(k)] = struct{}{}
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", userAgent)
|
||||
req.Header.Set("User-Agent", "webhooker/1.0")
|
||||
|
||||
// A Content-Type configured on the target describes the body
|
||||
// being sent rather than the sender. A 307/308 preserves the
|
||||
|
||||
@@ -136,7 +136,7 @@ func (t *slackTarget) attempt(
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", t.eng.userAgent())
|
||||
req.Header.Set("User-Agent", "webhooker/1.0")
|
||||
|
||||
resp, doErr := executeHTTPRequest(t.client, req)
|
||||
durationMs := time.Since(start).Milliseconds()
|
||||
|
||||
@@ -418,38 +418,6 @@ func TestProcessRetryTask_TargetDeleted_MakesNoAttempt(
|
||||
assert.Zero(t, s.Engine.ExportInflightHeld())
|
||||
}
|
||||
|
||||
// TestProcessRetryTask_TargetDeleted_CountsFailureOnTarget verifies
|
||||
// that the failure of a retry abandoned because its target is gone is
|
||||
// added to that target's own totals, not to a row with no target.
|
||||
func TestProcessRetryTask_TargetDeleted_CountsFailureOnTarget(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
s := newISetup(t)
|
||||
|
||||
var hits atomic.Int64
|
||||
|
||||
task, targetID := tRetryChainSetup(
|
||||
t, s, "gone-counted", &hits,
|
||||
)
|
||||
|
||||
require.NoError(t, s.MainDB.Delete(
|
||||
&database.Target{}, "id = ?", targetID,
|
||||
).Error)
|
||||
|
||||
s.Engine.ExportProcessRetryTask(
|
||||
context.Background(), &task,
|
||||
)
|
||||
|
||||
var rows []database.TargetTotals
|
||||
|
||||
require.NoError(t, s.WebhookDB.Find(&rows).Error)
|
||||
assert.Equal(t, []database.TargetTotals{
|
||||
{TargetID: targetID, Failed: 1},
|
||||
}, rows)
|
||||
}
|
||||
|
||||
// TestProcessRetryTask_TargetPresent_StillDelivers is the guard's
|
||||
// mutation check: a liveness check that refused every retry would pass
|
||||
// the test above and break every retry there is.
|
||||
|
||||
@@ -1,91 +0,0 @@
|
||||
package delivery_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/fx/fxtest"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/delivery"
|
||||
"sneak.berlin/go/webhooker/internal/globals"
|
||||
"sneak.berlin/go/webhooker/internal/logger"
|
||||
"sneak.berlin/go/webhooker/internal/metrics"
|
||||
)
|
||||
|
||||
// Both the http and the slack target send webhooker/ and the version
|
||||
// in Globals, the value the web UI footer shows. A User-Agent
|
||||
// configured on the target or carried in by the sender does not
|
||||
// replace it.
|
||||
func TestUserAgent_IsTheBuildVersion(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const want = "webhooker/1.2.3-test"
|
||||
|
||||
userAgents := make(chan string, 1)
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
userAgents <- r.Header.Get("User-Agent")
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
},
|
||||
))
|
||||
defer ts.Close()
|
||||
|
||||
g := &globals.Globals{Version: "1.2.3-test"}
|
||||
lc := fxtest.NewLifecycle(t)
|
||||
|
||||
log, err := logger.New(lc, logger.LoggerParams{Globals: g})
|
||||
require.NoError(t, err)
|
||||
|
||||
e := delivery.New(lc, delivery.EngineParams{
|
||||
Globals: g,
|
||||
Logger: log,
|
||||
// httptest listens on loopback, which the default guard
|
||||
// refuses.
|
||||
SSRFGuard: delivery.NewTestGuard(
|
||||
netip.MustParsePrefix("127.0.0.0/8"),
|
||||
),
|
||||
Metrics: metrics.New(prometheus.NewRegistry()),
|
||||
})
|
||||
|
||||
statusCode, _, _, err := e.ExportDoHTTPRequest(
|
||||
context.Background(),
|
||||
&delivery.HTTPTargetConfig{
|
||||
URL: ts.URL,
|
||||
Headers: map[string]string{"User-Agent": "configured/1"},
|
||||
},
|
||||
&database.Event{Headers: `{"User-Agent":["curl/8"]}`},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, http.StatusOK, statusCode)
|
||||
require.Len(t, userAgents, 1, "the http target sent no request")
|
||||
assert.Equal(t, want, <-userAgents, "http target")
|
||||
|
||||
db := testWebhookDB(t)
|
||||
targetID := uuid.New().String()
|
||||
|
||||
slackCfg, err := json.Marshal(
|
||||
delivery.SlackTargetConfig{WebhookURL: ts.URL},
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
event := seedEvent(t, db, `{"action":"test"}`)
|
||||
dlv := seedDelivery(
|
||||
t, db, event.ID, targetID, database.DeliveryStatusPending,
|
||||
)
|
||||
|
||||
e.ExportDeliverSlack(context.Background(), db, buildSlackDelivery(
|
||||
dlv, event, targetID, "test-slack", string(slackCfg),
|
||||
))
|
||||
require.Len(t, userAgents, 1, "the slack target sent no request")
|
||||
assert.Equal(t, want, <-userAgents, "slack target")
|
||||
}
|
||||
@@ -137,10 +137,6 @@ func bootAtDebug(t *testing.T, dataDir string) string {
|
||||
|
||||
app := fxtest.New(
|
||||
t,
|
||||
// fx's own log is discarded, not sent to t.Logf: a hook still
|
||||
// running after a start or stop timeout would write there after
|
||||
// the test has returned.
|
||||
fx.NopLogger,
|
||||
fx.Provide(
|
||||
globals.New,
|
||||
logger.New,
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
// SQL — parameters and all — for every statement that returns an
|
||||
// error, including gorm.ErrRecordNotFound. Two of this service's
|
||||
// lookups miss by design on unauthenticated routes: the entrypoint
|
||||
// lookup on /h/{uuid}, whose path segment the client picks
|
||||
// lookup on /webhook/{uuid}, whose path segment the client picks
|
||||
// outright, and the user lookup behind the login form, whose username
|
||||
// the client picks outright. Under the default logger each of those
|
||||
// misses printed an unbounded, attacker-chosen string, at no level the
|
||||
|
||||
+33
-63
@@ -2,56 +2,19 @@ package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/logfield"
|
||||
"sneak.berlin/go/webhooker/internal/middleware"
|
||||
)
|
||||
|
||||
// loginDestination returns where a successful login sends the
|
||||
// browser: next when it is a path on this site, otherwise "/", which
|
||||
// leads to the webhook list.
|
||||
//
|
||||
// A browser reads "//host" as another site, reads "\" as "/", and
|
||||
// drops tabs and newlines before reading at all. So the value must
|
||||
// start with exactly one "/" and hold no "\" or control character
|
||||
// anywhere: http.Redirect cleans "/a/../\host" down to "/\host". It
|
||||
// is checked after percent-decoding, so an encoded form of any of
|
||||
// these is refused too.
|
||||
func loginDestination(next string) string {
|
||||
if len(next) > middleware.MaxNextBytes {
|
||||
return "/"
|
||||
}
|
||||
|
||||
decoded, err := url.PathUnescape(next)
|
||||
if err != nil ||
|
||||
!strings.HasPrefix(decoded, "/") ||
|
||||
strings.HasPrefix(decoded, "//") ||
|
||||
strings.Contains(decoded, `\`) ||
|
||||
strings.ContainsFunc(decoded, unicode.IsControl) {
|
||||
return "/"
|
||||
}
|
||||
|
||||
return next
|
||||
}
|
||||
|
||||
// HandleLoginPage returns a handler for the login page (GET)
|
||||
func (h *Handlers) HandleLoginPage() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
next := loginDestination(
|
||||
r.URL.Query().Get(middleware.NextParam),
|
||||
)
|
||||
|
||||
// Check if already logged in
|
||||
sess, err := h.session.Get(r)
|
||||
if err == nil && h.session.IsAuthenticated(sess) {
|
||||
http.Redirect( //nolint:gosec // checked by loginDestination
|
||||
w, r, next, http.StatusSeeOther,
|
||||
)
|
||||
http.Redirect(w, r, "/", http.StatusSeeOther)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -59,7 +22,6 @@ func (h *Handlers) HandleLoginPage() http.HandlerFunc {
|
||||
// Render login page
|
||||
data := map[string]any{
|
||||
tmplKeyError: "",
|
||||
tmplKeyNext: next,
|
||||
}
|
||||
|
||||
h.renderTemplate(w, r, "login.html", data)
|
||||
@@ -74,7 +36,7 @@ func (h *Handlers) HandleLoginSubmit() http.HandlerFunc {
|
||||
err := r.ParseForm()
|
||||
if err != nil {
|
||||
h.log.Error("failed to parse form", "error", err)
|
||||
h.renderError(w, r, http.StatusBadRequest)
|
||||
http.Error(w, "Bad request", http.StatusBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -115,13 +77,8 @@ func (h *Handlers) HandleLoginSubmit() http.HandlerFunc {
|
||||
"user_id", user.ID,
|
||||
)
|
||||
|
||||
// The form value is the client's to set, so it is checked
|
||||
// again here rather than trusted from the rendered page.
|
||||
http.Redirect( //nolint:gosec // checked by loginDestination
|
||||
w, r,
|
||||
loginDestination(r.PostFormValue(middleware.NextParam)),
|
||||
http.StatusSeeOther,
|
||||
)
|
||||
// Redirect to home page
|
||||
http.Redirect(w, r, "/", http.StatusSeeOther)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -134,12 +91,10 @@ func (h *Handlers) renderLoginError(
|
||||
) {
|
||||
data := map[string]any{
|
||||
tmplKeyError: msg,
|
||||
tmplKeyNext: loginDestination(
|
||||
r.PostFormValue(middleware.NextParam),
|
||||
),
|
||||
}
|
||||
|
||||
h.renderTemplateStatus(w, r, "login.html", data, status)
|
||||
w.WriteHeader(status)
|
||||
h.renderTemplate(w, r, "login.html", data)
|
||||
}
|
||||
|
||||
// authenticateUser looks up and verifies a user's credentials.
|
||||
@@ -148,10 +103,9 @@ func (h *Handlers) renderLoginError(
|
||||
// The credential check runs BEFORE any rate-limit budget is
|
||||
// consulted, and only a failed check spends budget. That is what
|
||||
// keeps the single administrative path reachable: behind the reverse
|
||||
// proxy this deployment requires, when TRUSTED_PROXIES does not cover
|
||||
// it, every client shares one bucket, so a limiter spent on arrival
|
||||
// lets any stranger deny the operator's own correct password
|
||||
// indefinitely.
|
||||
// proxy this deployment requires, with TRUSTED_PROXIES unset, every
|
||||
// client shares one bucket, so a limiter spent on arrival lets any
|
||||
// stranger deny the operator's own correct password indefinitely.
|
||||
//
|
||||
// Verifying first means every login POST costs an Argon2id hash, so
|
||||
// the work is taken under a bounded number of verification slots.
|
||||
@@ -211,7 +165,11 @@ func (h *Handlers) authenticateUser(
|
||||
|
||||
valid, err := database.VerifyPassword(password, user.Password)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to verify password", err)
|
||||
h.log.Error("failed to verify password", "error", err)
|
||||
http.Error(
|
||||
w, "Internal server error",
|
||||
http.StatusInternalServerError,
|
||||
)
|
||||
|
||||
return user, err
|
||||
}
|
||||
@@ -283,14 +241,24 @@ func (h *Handlers) createAuthenticatedSession(
|
||||
) error {
|
||||
oldSess, err := h.session.Get(r)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to get session", err)
|
||||
h.log.Error("failed to get session", "error", err)
|
||||
http.Error(
|
||||
w, "Internal server error",
|
||||
http.StatusInternalServerError,
|
||||
)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
sess, err := h.session.Regenerate(r, w, oldSess)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to regenerate session", err)
|
||||
h.log.Error(
|
||||
"failed to regenerate session", "error", err,
|
||||
)
|
||||
http.Error(
|
||||
w, "Internal server error",
|
||||
http.StatusInternalServerError,
|
||||
)
|
||||
|
||||
return err
|
||||
}
|
||||
@@ -299,7 +267,11 @@ func (h *Handlers) createAuthenticatedSession(
|
||||
|
||||
err = h.session.Save(r, w, sess)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to save session", err)
|
||||
h.log.Error("failed to save session", "error", err)
|
||||
http.Error(
|
||||
w, "Internal server error",
|
||||
http.StatusInternalServerError,
|
||||
)
|
||||
|
||||
return err
|
||||
}
|
||||
@@ -332,9 +304,7 @@ func (h *Handlers) HandleLogout() http.HandlerFunc {
|
||||
)
|
||||
}
|
||||
|
||||
http.Redirect(
|
||||
w, r, withNotice("/pages/login", signedOut),
|
||||
http.StatusSeeOther,
|
||||
)
|
||||
// Redirect to login page
|
||||
http.Redirect(w, r, "/pages/login", http.StatusSeeOther)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package handlers_test
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"html/template"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
@@ -26,7 +25,7 @@ const (
|
||||
|
||||
// sharedProxyPeer is the whole point of this file. Production is
|
||||
// required to run behind a TLS-terminating reverse proxy, and
|
||||
// when TRUSTED_PROXIES does not cover it every client — attacker
|
||||
// TRUSTED_PROXIES defaults to empty, so every client — attacker
|
||||
// and operator alike — reaches the process from the proxy's
|
||||
// address and shares one rate-limit bucket. Both parties in
|
||||
// these tests therefore use the same RemoteAddr.
|
||||
@@ -116,11 +115,11 @@ func floodFailures(
|
||||
// done-criterion of https://git.eeqj.de/sneak/webhooker/issues/150.
|
||||
//
|
||||
// The attacker and the operator share one rate-limit bucket, because
|
||||
// behind the mandated reverse proxy, when TRUSTED_PROXIES does not
|
||||
// cover it, every client keys on the proxy's address. The attacker
|
||||
// floods the operator's own username — a single-admin product has a
|
||||
// predictable one — far past the failure limit. The operator must
|
||||
// still be able to log in with the correct password.
|
||||
// behind the mandated reverse proxy with TRUSTED_PROXIES unset every
|
||||
// client keys on the proxy's address. The attacker floods the
|
||||
// operator's own username — a single-admin product has a predictable
|
||||
// one — far past the failure limit. The operator must still be able
|
||||
// to log in with the correct password.
|
||||
//
|
||||
// This fails if credentials stop being verified ahead of the limiter.
|
||||
func TestLogin_StrangersFloodCannotLockOutTheOperator(t *testing.T) {
|
||||
@@ -405,60 +404,6 @@ func TestLogin_MissingCredentialsRejectedBeforeAnyHash(t *testing.T) {
|
||||
)
|
||||
}
|
||||
|
||||
// TestLogin_FormErrorAnswersItsStatusWithThePage proves that the login
|
||||
// form shown again with an error still answers 400 with the whole page.
|
||||
func TestLogin_FormErrorAnswersItsStatusWithThePage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var h *handlers.Handlers
|
||||
|
||||
app := newTestApp(t, &h)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
w := submitLogin(h, sharedProxyPeer, "", "")
|
||||
|
||||
assert.Equal(t, http.StatusBadRequest, w.Code)
|
||||
assert.Contains(
|
||||
t, w.Body.String(), "Username and password are required",
|
||||
)
|
||||
assert.Contains(
|
||||
t, w.Body.String(), "</html>",
|
||||
"the page must render to completion",
|
||||
)
|
||||
}
|
||||
|
||||
// TestLogin_FormErrorRenderFailureAnswers500 proves that a login form
|
||||
// error page whose template fails answers 500 with the error page and
|
||||
// none of the form page, rather than the 400 it meant to send.
|
||||
func TestLogin_FormErrorRenderFailureAnswers500(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var h *handlers.Handlers
|
||||
|
||||
app := newTestApp(t, &h)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
// The page prints its error message and then fails.
|
||||
h.AddTemplateForTest("login.html", template.Must(
|
||||
template.New("login").Funcs(template.FuncMap{
|
||||
"fail": func() (string, error) { return "", errMidRender },
|
||||
}).Parse(`{{.Error}}{{fail}}`),
|
||||
))
|
||||
|
||||
w := submitLogin(h, sharedProxyPeer, "", "")
|
||||
|
||||
assert.Equal(t, http.StatusInternalServerError, w.Code)
|
||||
assert.NotContains(
|
||||
t, w.Body.String(), "Username and password are required",
|
||||
"the response must carry no part of the aborted page",
|
||||
)
|
||||
assert.Contains(t, w.Body.String(), "500 Internal Server Error")
|
||||
}
|
||||
|
||||
// TestLogin_SuccessCreatesSession is the control for the tests above:
|
||||
// the success path they assert on really does authenticate.
|
||||
func TestLogin_SuccessCreatesSession(t *testing.T) {
|
||||
@@ -509,202 +454,6 @@ func TestLogin_SuccessCreatesSession(t *testing.T) {
|
||||
)
|
||||
}
|
||||
|
||||
// TestLogin_ReturnsOnlyToAPathOnThisSite is the security half of
|
||||
// https://git.eeqj.de/sneak/webhooker/issues/384: the page a login
|
||||
// returns to is client-chosen, so anything that is not a path on this
|
||||
// site, plain or percent-encoded, must land on "/", the webhook list.
|
||||
func TestLogin_ReturnsOnlyToAPathOnThisSite(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
db *database.Database
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &db)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
seedOperator(t, db)
|
||||
|
||||
cases := []struct{ next, want string }{
|
||||
{"/hook/abc/events?page=2", "/hook/abc/events?page=2"},
|
||||
{"", "/"},
|
||||
{"https://evil.example/", "/"},
|
||||
{"https%3A%2F%2Fevil.example%2F", "/"},
|
||||
{"//evil.example/", "/"},
|
||||
{"%2F%2Fevil.example/", "/"},
|
||||
{"/%2Fevil.example/", "/"},
|
||||
{`/\evil.example/`, "/"},
|
||||
{"%2F%5Cevil.example/", "/"},
|
||||
{"/%5Cevil.example/", "/"},
|
||||
{`/a/../\evil.example/`, "/"},
|
||||
{"/\t/evil.example/", "/"},
|
||||
{"/%09/evil.example/", "/"},
|
||||
{"/\n/evil.example/", "/"},
|
||||
{"/%0A/evil.example/", "/"},
|
||||
{"/\r/evil.example/", "/"},
|
||||
{"/%0D/evil.example/", "/"},
|
||||
{"/%00/evil.example/", "/"},
|
||||
{"/%7F/evil.example/", "/"},
|
||||
{"%252F%252Fevil.example/", "/"},
|
||||
{"https%253A%252F%252Fevil.example%252F", "/"},
|
||||
{"/" + strings.Repeat("a", 4096), "/"},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
form := url.Values{}
|
||||
form.Set("username", operatorUser)
|
||||
form.Set("password", operatorPassword)
|
||||
form.Set("next", c.next)
|
||||
|
||||
req := httptest.NewRequestWithContext(
|
||||
context.Background(),
|
||||
http.MethodPost,
|
||||
"/pages/login",
|
||||
strings.NewReader(form.Encode()),
|
||||
)
|
||||
req.Header.Set(
|
||||
"Content-Type", "application/x-www-form-urlencoded",
|
||||
)
|
||||
req.RemoteAddr = sharedProxyPeer
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
h.HandleLoginSubmit().ServeHTTP(w, req)
|
||||
|
||||
assert.Equal(t, http.StatusSeeOther, w.Code, "next %q", c.next)
|
||||
assert.Equal(
|
||||
t, c.want, w.Header().Get("Location"), "next %q", c.next,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLogin_WrongPasswordKeepsTheRequestedPage: after a wrong
|
||||
// password the login page is shown again with the same next, so the
|
||||
// next attempt still returns to the page that was asked for.
|
||||
func TestLogin_WrongPasswordKeepsTheRequestedPage(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
db *database.Database
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &db)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
seedOperator(t, db)
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("username", operatorUser)
|
||||
form.Set("password", "wrong")
|
||||
form.Set("next", "/hook/abc")
|
||||
|
||||
req := httptest.NewRequestWithContext(
|
||||
context.Background(),
|
||||
http.MethodPost,
|
||||
"/pages/login",
|
||||
strings.NewReader(form.Encode()),
|
||||
)
|
||||
req.Header.Set(
|
||||
"Content-Type", "application/x-www-form-urlencoded",
|
||||
)
|
||||
req.RemoteAddr = sharedProxyPeer
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
h.HandleLoginSubmit().ServeHTTP(w, req)
|
||||
|
||||
assert.Equal(t, http.StatusUnauthorized, w.Code)
|
||||
assert.Contains(
|
||||
t, w.Body.String(), `name="next" value="/hook/abc"`,
|
||||
)
|
||||
}
|
||||
|
||||
// loginPageGet renders the login page as a GET with the given next
|
||||
// value and cookies.
|
||||
func loginPageGet(
|
||||
h *handlers.Handlers, next string, cookies []*http.Cookie,
|
||||
) *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequestWithContext(
|
||||
context.Background(), http.MethodGet,
|
||||
"/pages/login?"+url.Values{"next": {next}}.Encode(), nil,
|
||||
)
|
||||
|
||||
for _, c := range cookies {
|
||||
req.AddCookie(c)
|
||||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
h.HandleLoginPage().ServeHTTP(w, req)
|
||||
|
||||
return w
|
||||
}
|
||||
|
||||
// TestLoginPage_CarriesOnlyAPathOnThisSite covers the login page
|
||||
// itself: its form carries the requested page only when it is a path
|
||||
// on this site, and a browser already logged in goes straight there,
|
||||
// or to "/" when it is not.
|
||||
func TestLoginPage_CarriesOnlyAPathOnThisSite(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
assert.Contains(
|
||||
t, loginPageGet(h, "/hook/abc", nil).Body.String(),
|
||||
`name="next" value="/hook/abc"`,
|
||||
)
|
||||
assert.Contains(
|
||||
t, loginPageGet(h, "//evil.example/", nil).Body.String(),
|
||||
`name="next" value="/"`,
|
||||
)
|
||||
|
||||
cookies := authenticatedCookies(t, sess, "test-user-id", "testuser")
|
||||
|
||||
cases := []struct{ next, want string }{
|
||||
{"/hook/abc", "/hook/abc"},
|
||||
{"//evil.example/", "/"},
|
||||
{`/\evil.example/`, "/"},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
w := loginPageGet(h, c.next, cookies)
|
||||
|
||||
assert.Equal(t, http.StatusSeeOther, w.Code, "next %q", c.next)
|
||||
assert.Equal(
|
||||
t, c.want, w.Header().Get("Location"), "next %q", c.next,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLoginPage_HasNoLinkToItself: the navigation bar on the login
|
||||
// page offers no link to the login page.
|
||||
func TestLoginPage_HasNoLinkToItself(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var h *handlers.Handlers
|
||||
|
||||
app := newTestApp(t, &h)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
w := loginPageGet(h, "", nil)
|
||||
|
||||
require.Equal(t, http.StatusOK, w.Code)
|
||||
assert.NotContains(t, w.Body.String(), `href="/pages/login"`)
|
||||
}
|
||||
|
||||
// TestLogin_UsernameAtLimitCanLogIn shows that a username of exactly
|
||||
// database.MaxUsernameBytes still fits in the session cookie. Past
|
||||
// what the cookie can carry, a correct login answers 500.
|
||||
|
||||
@@ -11,37 +11,72 @@ import (
|
||||
"sneak.berlin/go/webhooker/internal/delivery"
|
||||
)
|
||||
|
||||
// The outcomes of a replay POST, as the notice codes its redirect
|
||||
// carries. noticeFor holds the line each one shows.
|
||||
// replayOutcomeParam is the query parameter the replay POST redirects
|
||||
// with and the event log page reads its banner from.
|
||||
const replayOutcomeParam = "replay"
|
||||
|
||||
// replayOutcomeCode is the outcome of a replay POST. The redirect
|
||||
// carries one of these fixed codes rather than a message, so nothing a
|
||||
// client submits can reach the rendered page through it.
|
||||
type replayOutcomeCode string
|
||||
|
||||
const (
|
||||
// replayQueued reports that a new delivery was created and handed
|
||||
// to the delivery engine.
|
||||
replayQueued noticeCode = "replay-queued"
|
||||
replayQueued replayOutcomeCode = "queued"
|
||||
|
||||
// replayTargetDeleted reports a target that once existed and has
|
||||
// since been deleted. Deletes are soft and deliveries carry no
|
||||
// foreign key to the target row, so the history survives its
|
||||
// target and this is the ordinary case for an old event.
|
||||
replayTargetDeleted noticeCode = "replay-target-deleted"
|
||||
replayTargetDeleted replayOutcomeCode = "target-deleted"
|
||||
|
||||
// replayTargetMissing reports a target id that names no row at
|
||||
// all, deleted or otherwise.
|
||||
replayTargetMissing noticeCode = "replay-target-missing"
|
||||
replayTargetMissing replayOutcomeCode = "target-missing"
|
||||
|
||||
// replayTargetInactive reports a target the operator has
|
||||
// deactivated. A deactivated target receives no new deliveries, so
|
||||
// a replay to it would be a delivery they switched off.
|
||||
replayTargetInactive noticeCode = "replay-target-inactive"
|
||||
replayTargetInactive replayOutcomeCode = "target-inactive"
|
||||
|
||||
// replayNotTerminal reports a delivery the engine has not finished
|
||||
// with.
|
||||
replayNotTerminal noticeCode = "replay-not-terminal"
|
||||
replayNotTerminal replayOutcomeCode = "not-terminal"
|
||||
|
||||
// replayInFlight reports that an earlier replay of this event to
|
||||
// this target is still running.
|
||||
replayInFlight noticeCode = "replay-in-flight"
|
||||
replayInFlight replayOutcomeCode = "in-flight"
|
||||
)
|
||||
|
||||
// replayOutcome returns the banner the event log page shows for an
|
||||
// outcome code, and whether the replay was queued. An unrecognised
|
||||
// code yields no banner.
|
||||
func replayOutcome(code string) (string, bool) {
|
||||
switch replayOutcomeCode(code) {
|
||||
case replayQueued:
|
||||
return "Replay queued: a new delivery was created against " +
|
||||
"the target's current configuration.", true
|
||||
case replayTargetDeleted:
|
||||
return "Not replayed: the target this delivery was for has " +
|
||||
"been deleted. Recreate the target, then replay.", false
|
||||
case replayTargetMissing:
|
||||
return "Not replayed: the target this delivery was for no " +
|
||||
"longer exists.", false
|
||||
case replayTargetInactive:
|
||||
return "Not replayed: the target this delivery was for is " +
|
||||
"deactivated. Activate it, then replay.", false
|
||||
case replayNotTerminal:
|
||||
return "Not replayed: this delivery has not finished yet.",
|
||||
false
|
||||
case replayInFlight:
|
||||
return "Not replayed: a delivery of this event to this " +
|
||||
"target is already in flight.", false
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
// HandleDeliveryReplay re-sends a finished delivery's event to its
|
||||
// target.
|
||||
//
|
||||
@@ -70,7 +105,9 @@ func (h *Handlers) HandleDeliveryReplay() http.HandlerFunc {
|
||||
// middleware, which runs before CSRF parses the form.
|
||||
err := r.ParseForm()
|
||||
if err != nil {
|
||||
h.renderError(w, r, http.StatusBadRequest)
|
||||
http.Error(
|
||||
w, "Bad request", http.StatusBadRequest,
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -87,14 +124,14 @@ func (h *Handlers) replayDelivery(
|
||||
webhook database.Webhook,
|
||||
) {
|
||||
if !h.dbMgr.DBExists(webhook.ID) {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
http.NotFound(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to get webhook database", err)
|
||||
h.serverError(w, "failed to get webhook database", err)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -105,14 +142,14 @@ func (h *Handlers) replayDelivery(
|
||||
}
|
||||
|
||||
if !original.Status.Terminal() {
|
||||
redirectToEventLog(w, r, webhook, replayNotTerminal)
|
||||
h.finishReplay(w, r, webhook, replayNotTerminal)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
target, code := h.replayTarget(webhook.ID, original.TargetID)
|
||||
if target == nil {
|
||||
redirectToEventLog(w, r, webhook, code)
|
||||
h.finishReplay(w, r, webhook, code)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -136,7 +173,7 @@ func (h *Handlers) loadReplaySource(
|
||||
&original, "id = ?", chi.URLParam(r, "deliveryID"),
|
||||
).Error
|
||||
if err != nil {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
http.NotFound(w, r)
|
||||
|
||||
return nil, false
|
||||
}
|
||||
@@ -158,14 +195,14 @@ func (h *Handlers) queueReplay(
|
||||
)
|
||||
if err != nil {
|
||||
h.serverError(
|
||||
w, r, "failed to count in-flight deliveries", err,
|
||||
w, "failed to count in-flight deliveries", err,
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if inFlight > 0 {
|
||||
redirectToEventLog(w, r, webhook, replayInFlight)
|
||||
h.finishReplay(w, r, webhook, replayInFlight)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -175,7 +212,7 @@ func (h *Handlers) queueReplay(
|
||||
err = webhookDB.
|
||||
First(&event, "id = ?", original.EventID).Error
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to load event for replay", err)
|
||||
h.serverError(w, "failed to load event for replay", err)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -185,7 +222,7 @@ func (h *Handlers) queueReplay(
|
||||
)
|
||||
if err != nil {
|
||||
h.serverError(
|
||||
w, r, "failed to create replay delivery", err,
|
||||
w, "failed to create replay delivery", err,
|
||||
)
|
||||
|
||||
return
|
||||
@@ -203,7 +240,7 @@ func (h *Handlers) queueReplay(
|
||||
"delivery_id", task.DeliveryID,
|
||||
)
|
||||
|
||||
redirectToEventLog(w, r, webhook, replayQueued)
|
||||
h.finishReplay(w, r, webhook, replayQueued)
|
||||
}
|
||||
|
||||
// replayTarget loads the delivery's target as it stands now.
|
||||
@@ -216,7 +253,7 @@ func (h *Handlers) queueReplay(
|
||||
// with the returned code saying why.
|
||||
func (h *Handlers) replayTarget(
|
||||
webhookID, targetID string,
|
||||
) (*database.Target, noticeCode) {
|
||||
) (*database.Target, replayOutcomeCode) {
|
||||
var target database.Target
|
||||
|
||||
err := h.db.DB().Unscoped().Where(
|
||||
@@ -262,9 +299,8 @@ func countInFlightDeliveries(
|
||||
return count, err
|
||||
}
|
||||
|
||||
// createReplayDelivery writes the new pending delivery row, adds it to
|
||||
// its target's totals in the same transaction, and returns the task
|
||||
// that carries it to the delivery engine.
|
||||
// createReplayDelivery writes the new pending delivery row and returns
|
||||
// the task that carries it to the delivery engine.
|
||||
//
|
||||
// The row is written with associations omitted, and neither Event nor
|
||||
// Target is populated on it: GORM's SaveBeforeAssociations would
|
||||
@@ -283,16 +319,7 @@ func createReplayDelivery(
|
||||
Status: database.DeliveryStatusPending,
|
||||
}
|
||||
|
||||
err := webhookDB.Transaction(func(tx *gorm.DB) error {
|
||||
err := tx.Omit(clause.Associations).Create(dlv).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return database.AddTargetTotals(tx, database.TargetTotals{
|
||||
TargetID: dlv.TargetID, Deliveries: 1,
|
||||
})
|
||||
})
|
||||
err := webhookDB.Omit(clause.Associations).Create(dlv).Error
|
||||
if err != nil {
|
||||
return delivery.Task{}, err
|
||||
}
|
||||
@@ -326,16 +353,17 @@ func replayBody(body string) *string {
|
||||
return &body
|
||||
}
|
||||
|
||||
// redirectToEventLog redirects a replay or resubmit back to the event
|
||||
// log it was triggered from, carrying the outcome as its notice and
|
||||
// the page number the form submitted.
|
||||
func redirectToEventLog(
|
||||
// finishReplay redirects back to the event log the replay was
|
||||
// triggered from, carrying the outcome code the page turns into a
|
||||
// banner and the page number the form submitted.
|
||||
func (h *Handlers) finishReplay(
|
||||
w http.ResponseWriter,
|
||||
r *http.Request,
|
||||
webhook database.Webhook,
|
||||
code noticeCode,
|
||||
code replayOutcomeCode,
|
||||
) {
|
||||
dest := withNotice("/hook/"+webhook.ID+"/events", code)
|
||||
dest := "/source/" + webhook.ID + "/logs?" +
|
||||
replayOutcomeParam + "=" + string(code)
|
||||
|
||||
// The page is read from the form rather than the query string:
|
||||
// this is a POST, and its query string is what logs and Referer
|
||||
|
||||
@@ -138,7 +138,7 @@ func postReplay(
|
||||
t.Helper()
|
||||
|
||||
req := postRequest(
|
||||
"/hook/"+webhookID+"/deliveries/"+
|
||||
"/source/"+webhookID+"/deliveries/"+
|
||||
deliveryID+"/replay",
|
||||
authenticatedCookies(
|
||||
t, sess, deleteTestUserID, deleteTestUsername,
|
||||
@@ -212,7 +212,7 @@ func TestHandleDeliveryReplay_AppendsDeliveryAndLeavesOriginal(
|
||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||
assert.Equal(
|
||||
t,
|
||||
"/hook/"+wh.ID+"/events?notice=replay-queued",
|
||||
"/source/"+wh.ID+"/logs?replay=queued",
|
||||
w.Header().Get("Location"),
|
||||
)
|
||||
|
||||
@@ -362,7 +362,7 @@ func TestHandleDeliveryReplay_RefusesDeletedTarget(t *testing.T) {
|
||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||
assert.Equal(
|
||||
t,
|
||||
"/hook/"+wh.ID+"/events?notice=replay-target-deleted",
|
||||
"/source/"+wh.ID+"/logs?replay=target-deleted",
|
||||
w.Header().Get("Location"),
|
||||
)
|
||||
|
||||
@@ -390,7 +390,7 @@ func TestHandleDeliveryReplay_RefusesDeletedTarget(t *testing.T) {
|
||||
require.Equal(t, http.StatusSeeOther, missing.Code)
|
||||
assert.Equal(
|
||||
t,
|
||||
"/hook/"+wh.ID+"/events?notice=replay-target-missing",
|
||||
"/source/"+wh.ID+"/logs?replay=target-missing",
|
||||
missing.Header().Get("Location"),
|
||||
)
|
||||
}
|
||||
@@ -431,7 +431,7 @@ func TestHandleDeliveryReplay_RefusesWhileEarlierReplayInFlight(
|
||||
require.Equal(t, http.StatusSeeOther, first.Code)
|
||||
require.Equal(
|
||||
t,
|
||||
"/hook/"+wh.ID+"/events?notice=replay-queued",
|
||||
"/source/"+wh.ID+"/logs?replay=queued",
|
||||
first.Header().Get("Location"),
|
||||
)
|
||||
|
||||
@@ -439,7 +439,7 @@ func TestHandleDeliveryReplay_RefusesWhileEarlierReplayInFlight(
|
||||
require.Equal(t, http.StatusSeeOther, second.Code)
|
||||
assert.Equal(
|
||||
t,
|
||||
"/hook/"+wh.ID+"/events?notice=replay-in-flight",
|
||||
"/source/"+wh.ID+"/logs?replay=in-flight",
|
||||
second.Header().Get("Location"),
|
||||
)
|
||||
|
||||
@@ -465,7 +465,7 @@ func TestHandleDeliveryReplay_RefusesWhileEarlierReplayInFlight(
|
||||
require.Equal(t, http.StatusSeeOther, pending.Code)
|
||||
assert.Equal(
|
||||
t,
|
||||
"/hook/"+wh.ID+"/events?notice=replay-not-terminal",
|
||||
"/source/"+wh.ID+"/logs?replay=not-terminal",
|
||||
pending.Header().Get("Location"),
|
||||
)
|
||||
}
|
||||
@@ -501,7 +501,7 @@ func TestHandleSourceLogs_RendersReplayControlAndBanner(t *testing.T) {
|
||||
|
||||
assert.Contains(
|
||||
t, body,
|
||||
`action="/hook/`+wh.ID+`/deliveries/`+
|
||||
`action="/source/`+wh.ID+`/deliveries/`+
|
||||
original.ID+`/replay"`,
|
||||
)
|
||||
assert.Contains(t, body, `method="POST"`)
|
||||
@@ -509,7 +509,7 @@ func TestHandleSourceLogs_RendersReplayControlAndBanner(t *testing.T) {
|
||||
assert.Contains(t, body, ">Replay<")
|
||||
|
||||
refused := renderSourceLogsPageWithQuery(
|
||||
t, h, sess, wh.ID, "?notice=replay-target-deleted",
|
||||
t, h, sess, wh.ID, "?replay=target-deleted",
|
||||
)
|
||||
|
||||
assert.Contains(t, refused, "alert-error")
|
||||
@@ -517,7 +517,7 @@ func TestHandleSourceLogs_RendersReplayControlAndBanner(t *testing.T) {
|
||||
|
||||
// An outcome code nobody issued renders no banner at all.
|
||||
unknown := renderSourceLogsPageWithQuery(
|
||||
t, h, sess, wh.ID, "?notice=made-up",
|
||||
t, h, sess, wh.ID, "?replay=made-up",
|
||||
)
|
||||
|
||||
assert.NotContains(t, unknown, "alert-error")
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"html/template"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"sneak.berlin/go/webhooker/internal/handlers"
|
||||
)
|
||||
|
||||
// TestErrorPage_RenderFailureKeepsStatus proves that an error page
|
||||
// which cannot render answers with the status it was reporting, as
|
||||
// plain text, and is not attempted again: a page whose own render
|
||||
// fails reaches the error page, and the error page failing as well
|
||||
// ends there with the 500.
|
||||
func TestErrorPage_RenderFailureKeepsStatus(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var h *handlers.Handlers
|
||||
|
||||
app := newTestApp(t, &h)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
// .Status is an int, so asking it for a field fails the render.
|
||||
failing := `{{.Status.Missing}}`
|
||||
h.AddTemplateForTest("error.html", template.Must(
|
||||
template.New("error").Parse(failing),
|
||||
))
|
||||
h.AddTemplateForTest("failing.html", template.Must(
|
||||
template.New("failing").Parse(`{{.Data.Missing}}`),
|
||||
))
|
||||
|
||||
req := httptest.NewRequestWithContext(
|
||||
context.Background(), http.MethodGet, "/", nil,
|
||||
)
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
h.HandleErrorPage(http.StatusNotFound).ServeHTTP(w, req)
|
||||
|
||||
assert.Equal(t, http.StatusNotFound, w.Code)
|
||||
assert.Equal(t, "Not Found\n", w.Body.String())
|
||||
|
||||
w = httptest.NewRecorder()
|
||||
h.RenderTemplateForTest(w, req, "failing.html", 0)
|
||||
|
||||
assert.Equal(t, http.StatusInternalServerError, w.Code)
|
||||
assert.Equal(t, "Internal Server Error\n", w.Body.String())
|
||||
}
|
||||
@@ -52,7 +52,7 @@ func (h *Handlers) HandleEventBodyDownload() http.HandlerFunc {
|
||||
// steered by a client.
|
||||
eventID, err := uuid.Parse(chi.URLParam(r, "eventID"))
|
||||
if err != nil {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
http.NotFound(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -103,21 +103,21 @@ func (h *Handlers) serveEventBody(
|
||||
eventID string,
|
||||
) {
|
||||
if !h.dbMgr.DBExists(webhook.ID) {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
http.NotFound(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to get webhook database", err)
|
||||
h.serverError(w, "failed to get webhook database", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
body, found, err := eventBody(webhookDB, webhook.ID, eventID)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to read event body", err)
|
||||
h.serverError(w, "failed to read event body", err)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -130,7 +130,7 @@ func (h *Handlers) serveEventBody(
|
||||
// row and the whole body is served, or it does not and the
|
||||
// response is a clean 404.
|
||||
if !found {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
http.NotFound(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -64,8 +64,8 @@ func fetchEventBody(
|
||||
req := httptest.NewRequestWithContext(
|
||||
context.Background(),
|
||||
http.MethodGet,
|
||||
"/hook/"+url.PathEscape(sourceID)+
|
||||
"/events/"+url.PathEscape(eventID)+"/body",
|
||||
"/source/"+url.PathEscape(sourceID)+
|
||||
"/logs/"+url.PathEscape(eventID)+"/body",
|
||||
nil,
|
||||
)
|
||||
|
||||
@@ -490,7 +490,7 @@ func TestHandleSourceLogs_TruncationMarkerLinksToDownload(
|
||||
page := renderSourceLogsPage(t, h, sess, big.ID)
|
||||
assert.Contains(
|
||||
t, page,
|
||||
"/hook/"+big.ID+"/events/"+bigEvt.ID+"/body",
|
||||
"/source/"+big.ID+"/logs/"+bigEvt.ID+"/body",
|
||||
)
|
||||
|
||||
small := seedWebhook(t, db)
|
||||
@@ -501,6 +501,6 @@ func TestHandleSourceLogs_TruncationMarkerLinksToDownload(
|
||||
page = renderSourceLogsPage(t, h, sess, small.ID)
|
||||
assert.NotContains(
|
||||
t, page,
|
||||
"/hook/"+small.ID+"/events/"+smallEvt.ID+"/body",
|
||||
"/source/"+small.ID+"/logs/"+smallEvt.ID+"/body",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package handlers
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/go-chi/chi"
|
||||
"github.com/google/uuid"
|
||||
@@ -10,19 +11,43 @@ import (
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
)
|
||||
|
||||
// The outcomes of a resubmit POST, as the notice codes its redirect
|
||||
// carries. noticeFor holds the line each one shows.
|
||||
// resubmitOutcomeParam is the query parameter the resubmit POST
|
||||
// redirects with and the event log page reads its banner from.
|
||||
const resubmitOutcomeParam = "resubmit"
|
||||
|
||||
// resubmitOutcomeCode is the outcome of a resubmit POST. The redirect
|
||||
// carries one of these fixed codes rather than a message, so nothing a
|
||||
// client submits can reach the rendered page through it.
|
||||
type resubmitOutcomeCode string
|
||||
|
||||
const (
|
||||
// resubmitQueued reports that a new event was stored and its
|
||||
// deliveries handed to the delivery engine.
|
||||
resubmitQueued noticeCode = "resubmit-queued"
|
||||
resubmitQueued resubmitOutcomeCode = "queued"
|
||||
|
||||
// resubmitNoTargets reports a source with no active targets. The
|
||||
// new event is stored either way, exactly as a received event
|
||||
// with no targets is.
|
||||
resubmitNoTargets noticeCode = "resubmit-no-targets"
|
||||
resubmitNoTargets resubmitOutcomeCode = "no-targets"
|
||||
)
|
||||
|
||||
// resubmitOutcome returns the banner the event log page shows for an
|
||||
// outcome code, and whether the resubmit was queued. An unrecognised
|
||||
// code yields no banner.
|
||||
func resubmitOutcome(code string) (string, bool) {
|
||||
switch resubmitOutcomeCode(code) {
|
||||
case resubmitQueued:
|
||||
return "Resubmitted: a new event was created from the stored " +
|
||||
"one and queued to every active target.", true
|
||||
case resubmitNoTargets:
|
||||
return "Resubmitted: a new event was created, but this " +
|
||||
"source has no active targets, so nothing was queued.",
|
||||
true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
// resubmitSource is the stored event a resubmit copies. Its body is
|
||||
// read as bytes rather than as a string so the copy is byte-identical
|
||||
// to what was received, whatever the payload's encoding.
|
||||
@@ -74,7 +99,7 @@ func (h *Handlers) HandleEventResubmit() http.HandlerFunc {
|
||||
// middleware, which runs before CSRF parses the form.
|
||||
err := r.ParseForm()
|
||||
if err != nil {
|
||||
h.renderError(w, r, http.StatusBadRequest)
|
||||
http.Error(w, "Bad request", http.StatusBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -95,20 +120,20 @@ func (h *Handlers) resubmitEvent(
|
||||
// alphabet rather than from the request.
|
||||
eventID, err := uuid.Parse(chi.URLParam(r, "eventID"))
|
||||
if err != nil {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
http.NotFound(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if !h.dbMgr.DBExists(webhook.ID) {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
http.NotFound(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to get webhook database", err)
|
||||
h.serverError(w, "failed to get webhook database", err)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -122,7 +147,7 @@ func (h *Handlers) resubmitEvent(
|
||||
webhookDB, webhook.ID, eventID.String(),
|
||||
)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to load event to resubmit", err)
|
||||
h.serverError(w, "failed to load event to resubmit", err)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -130,7 +155,7 @@ func (h *Handlers) resubmitEvent(
|
||||
// A miss is a 404 whether the event was reaped, belongs to
|
||||
// another webhook, or never existed.
|
||||
if !found {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
http.NotFound(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -182,7 +207,7 @@ func (h *Handlers) queueResubmit(
|
||||
// inactive one is skipped rather than refused.
|
||||
targets, err := h.loadActiveTargets(webhook.ID)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to query targets", err)
|
||||
h.serverError(w, "failed to query targets", err)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -200,7 +225,7 @@ func (h *Handlers) queueResubmit(
|
||||
targets,
|
||||
)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to store resubmitted event", err)
|
||||
h.serverError(w, "failed to store resubmitted event", err)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -220,5 +245,29 @@ func (h *Handlers) queueResubmit(
|
||||
code = resubmitNoTargets
|
||||
}
|
||||
|
||||
redirectToEventLog(w, r, webhook, code)
|
||||
h.finishResubmit(w, r, webhook, code)
|
||||
}
|
||||
|
||||
// finishResubmit redirects back to the event log the resubmit was
|
||||
// triggered from, carrying the outcome code the page turns into a
|
||||
// banner and the page number the form submitted.
|
||||
func (h *Handlers) finishResubmit(
|
||||
w http.ResponseWriter,
|
||||
r *http.Request,
|
||||
webhook database.Webhook,
|
||||
code resubmitOutcomeCode,
|
||||
) {
|
||||
dest := "/source/" + webhook.ID + "/logs?" +
|
||||
resubmitOutcomeParam + "=" + string(code)
|
||||
|
||||
// The page is read from the form rather than the query string:
|
||||
// this is a POST, and its query string is what logs and Referer
|
||||
// headers record.
|
||||
if page := pageOrFirst(
|
||||
r.PostFormValue("page"),
|
||||
); page > 1 {
|
||||
dest += "&page=" + strconv.Itoa(page)
|
||||
}
|
||||
|
||||
http.Redirect(w, r, dest, http.StatusSeeOther)
|
||||
}
|
||||
|
||||
@@ -65,7 +65,7 @@ func postResubmit(
|
||||
t.Helper()
|
||||
|
||||
req := postRequest(
|
||||
"/hook/"+webhookID+"/events/"+eventID+"/resubmit",
|
||||
"/source/"+webhookID+"/events/"+eventID+"/resubmit",
|
||||
authenticatedCookies(
|
||||
t, sess, deleteTestUserID, deleteTestUsername,
|
||||
),
|
||||
@@ -154,7 +154,7 @@ func TestHandleEventResubmit_DeliversToTargetCreatedAfterTheEvent(
|
||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||
assert.Equal(
|
||||
t,
|
||||
"/hook/"+wh.ID+"/events?notice=resubmit-queued",
|
||||
"/source/"+wh.ID+"/logs?resubmit=queued",
|
||||
w.Header().Get("Location"),
|
||||
)
|
||||
|
||||
@@ -282,7 +282,7 @@ func TestHandleEventResubmit_IsRepeatable(t *testing.T) {
|
||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||
assert.Equal(
|
||||
t,
|
||||
"/hook/"+wh.ID+"/events?notice=resubmit-queued",
|
||||
"/source/"+wh.ID+"/logs?resubmit=queued",
|
||||
w.Header().Get("Location"),
|
||||
"a resubmit must not be refused while an earlier "+
|
||||
"one is in flight",
|
||||
@@ -436,7 +436,7 @@ func TestHandleEventResubmit_SkipsInactiveTarget(t *testing.T) {
|
||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||
assert.Equal(
|
||||
t,
|
||||
"/hook/"+wh.ID+"/events?notice=resubmit-queued",
|
||||
"/source/"+wh.ID+"/logs?resubmit=queued",
|
||||
w.Header().Get("Location"),
|
||||
"an inactive target is skipped, not an error",
|
||||
)
|
||||
@@ -482,7 +482,7 @@ func TestHandleEventResubmit_NoActiveTargetsStillStoresEvent(
|
||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||
assert.Equal(
|
||||
t,
|
||||
"/hook/"+wh.ID+"/events?notice=resubmit-no-targets",
|
||||
"/source/"+wh.ID+"/logs?resubmit=no-targets",
|
||||
w.Header().Get("Location"),
|
||||
)
|
||||
|
||||
@@ -598,7 +598,7 @@ func TestHandleSourceLogs_ShowsResubmitProvenance(t *testing.T) {
|
||||
)
|
||||
assert.Contains(
|
||||
t, body,
|
||||
"/hook/"+wh.ID+"/events/"+original.ID+"/resubmit",
|
||||
"/source/"+wh.ID+"/events/"+original.ID+"/resubmit",
|
||||
"the log must offer the resubmit action per event",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,14 +1,10 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"html/template"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
)
|
||||
|
||||
@@ -67,43 +63,12 @@ func (s *Handlers) LoadEventLogViewsForTest(
|
||||
page int,
|
||||
) []EventLogView {
|
||||
views, _, _ := s.loadEventsWithDeliveries(
|
||||
w, newRequestForTest(), webhook, nil, page,
|
||||
w, webhook, nil, page,
|
||||
)
|
||||
|
||||
return views
|
||||
}
|
||||
|
||||
// WebhookStatsForTest returns the figures the statistics pane on a
|
||||
// webhook's page shows, from the webhook's entrypoints and targets
|
||||
// loaded as that page loads them.
|
||||
func (s *Handlers) WebhookStatsForTest(webhookID string) *WebhookStats {
|
||||
var entrypoints []database.Entrypoint
|
||||
|
||||
s.db.DB().Where("webhook_id = ?", webhookID).Find(&entrypoints)
|
||||
|
||||
var targets []database.Target
|
||||
|
||||
s.db.DB().Where("webhook_id = ?", webhookID).Find(&targets)
|
||||
|
||||
return s.loadWebhookStats(webhookID, entrypoints, targets)
|
||||
}
|
||||
|
||||
// FinishedByTargetForTest exposes finishedByTarget for use in the
|
||||
// handlers_test package.
|
||||
func FinishedByTargetForTest(
|
||||
webhookDB *gorm.DB, since time.Time,
|
||||
) ([]TargetFinished, error) {
|
||||
return finishedByTarget(webhookDB, since)
|
||||
}
|
||||
|
||||
// newRequestForTest is the request the helpers here pass on for
|
||||
// callers that have none: it is used only to render the error page.
|
||||
func newRequestForTest() *http.Request {
|
||||
return httptest.NewRequestWithContext(
|
||||
context.Background(), http.MethodGet, "/", nil,
|
||||
)
|
||||
}
|
||||
|
||||
// AddTemplateForTest registers a template under a page name so that
|
||||
// the handlers_test package can drive the render path with a
|
||||
// template of its own.
|
||||
@@ -157,5 +122,5 @@ func (s *Handlers) BuildDatabaseTargetConfigForTest(
|
||||
w http.ResponseWriter,
|
||||
expiry string,
|
||||
) (string, error) {
|
||||
return s.buildDatabaseTargetConfig(w, newRequestForTest(), expiry)
|
||||
return s.buildDatabaseTargetConfig(w, expiry)
|
||||
}
|
||||
|
||||
@@ -306,7 +306,7 @@ func postWebhook(
|
||||
t.Helper()
|
||||
|
||||
req := httptest.NewRequestWithContext(
|
||||
context.Background(), http.MethodPost, "/h/x",
|
||||
context.Background(), http.MethodPost, "/webhook/x",
|
||||
strings.NewReader("{}"),
|
||||
)
|
||||
|
||||
|
||||
+44
-163
@@ -10,12 +10,9 @@ import (
|
||||
"html/template"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/webhooker/internal/config"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/delivery"
|
||||
"sneak.berlin/go/webhooker/internal/globals"
|
||||
@@ -39,9 +36,6 @@ const (
|
||||
tmplKeyError = "Error"
|
||||
// tmplKeyWebhook is the template data key for a webhook.
|
||||
tmplKeyWebhook = "Webhook"
|
||||
// tmplKeyNext is the template data key for the page to return
|
||||
// to after login.
|
||||
tmplKeyNext = "Next"
|
||||
)
|
||||
|
||||
// errInvalidPassword is returned when a password does not match.
|
||||
@@ -59,17 +53,14 @@ type HandlersParams struct {
|
||||
|
||||
Logger *logger.Logger
|
||||
Globals *globals.Globals
|
||||
Config *config.Config
|
||||
Database *database.Database
|
||||
WebhookDBMgr *database.WebhookDBManager
|
||||
Healthcheck *healthcheck.Healthcheck
|
||||
Session *session.Session
|
||||
Middleware *middleware.Middleware
|
||||
Notifier delivery.Notifier
|
||||
Archives delivery.Archives
|
||||
Evictor delivery.WebhookEvictor
|
||||
SSRFGuard *delivery.Guard
|
||||
Metrics *metrics.Set
|
||||
Registry *prometheus.Registry
|
||||
}
|
||||
|
||||
// Handlers provides HTTP handler methods for all application
|
||||
@@ -83,7 +74,7 @@ type Handlers struct {
|
||||
session *session.Session
|
||||
mw *middleware.Middleware
|
||||
notifier delivery.Notifier
|
||||
archives delivery.Archives
|
||||
evictor delivery.WebhookEvictor
|
||||
mtr *metrics.Set
|
||||
templates map[string]*template.Template
|
||||
|
||||
@@ -92,14 +83,6 @@ type Handlers struct {
|
||||
// is one delivery will actually attempt.
|
||||
ssrf *delivery.Guard
|
||||
|
||||
// renameMu makes the webhook edit, the target edit and target
|
||||
// creation run one at a time, each held from loading the stored
|
||||
// names through the archive rename, the save and any move back.
|
||||
// Interleaved, one could rename an archive between another's
|
||||
// rename and save, leaving the file named for one edit and the
|
||||
// stored names from the other.
|
||||
renameMu sync.Mutex
|
||||
|
||||
// dummyVerifications counts the equivalent-cost verifications
|
||||
// charged for usernames that do not exist. It exists so a test
|
||||
// can prove that path runs without measuring wall-clock time.
|
||||
@@ -108,23 +91,18 @@ type Handlers struct {
|
||||
|
||||
// parsePageTemplate parses a page-specific template set from the
|
||||
// embedded FS. Each page template is combined with the shared
|
||||
// base, htmlheader, navbar and notice templates, and with any further
|
||||
// files the page includes. The page file must be listed first so that
|
||||
// its root action ({{template "base" .}}) becomes the template set's
|
||||
// entry point.
|
||||
func parsePageTemplate(
|
||||
pageFile string, included ...string,
|
||||
) *template.Template {
|
||||
files := append([]string{
|
||||
pageFile,
|
||||
"base.html",
|
||||
"htmlheader.html",
|
||||
"navbar.html",
|
||||
"notice.html",
|
||||
}, included...)
|
||||
|
||||
// base, htmlheader, and navbar templates. The page file must be
|
||||
// listed first so that its root action ({{template "base" .}})
|
||||
// becomes the template set's entry point.
|
||||
func parsePageTemplate(pageFile string) *template.Template {
|
||||
return template.Must(
|
||||
template.ParseFS(templates.Templates, files...),
|
||||
template.ParseFS(
|
||||
templates.Templates,
|
||||
pageFile,
|
||||
"base.html",
|
||||
"htmlheader.html",
|
||||
"navbar.html",
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -143,22 +121,20 @@ func New(
|
||||
s.session = params.Session
|
||||
s.mw = params.Middleware
|
||||
s.notifier = params.Notifier
|
||||
s.archives = params.Archives
|
||||
s.mtr = params.Metrics
|
||||
s.evictor = params.Evictor
|
||||
s.mtr = metrics.Default()
|
||||
s.ssrf = params.SSRFGuard
|
||||
|
||||
// Parse all page templates once at startup
|
||||
s.templates = map[string]*template.Template{
|
||||
"login.html": parsePageTemplate("login.html"),
|
||||
"profile.html": parsePageTemplate("profile.html"),
|
||||
"settings.html": parsePageTemplate("settings.html"),
|
||||
"sources_list.html": parsePageTemplate("sources_list.html"),
|
||||
"sources_new.html": parsePageTemplate("sources_new.html"),
|
||||
"source_detail.html": parsePageTemplate("source_detail.html", "webhook_stats.html"),
|
||||
"source_detail.html": parsePageTemplate("source_detail.html"),
|
||||
"source_edit.html": parsePageTemplate("source_edit.html"),
|
||||
"source_logs.html": parsePageTemplate("source_logs.html"),
|
||||
"target_edit.html": parsePageTemplate("target_edit.html"),
|
||||
"error.html": parsePageTemplate("error.html"),
|
||||
}
|
||||
|
||||
lc.Append(fx.Hook{
|
||||
@@ -170,16 +146,6 @@ func New(
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// HandleErrorPage returns a handler that answers every request with
|
||||
// the error page for status. The router uses it for unknown paths, the
|
||||
// CSRF middleware for a refused form, and each admin page route
|
||||
// group's recoverer for a panic.
|
||||
func (s *Handlers) HandleErrorPage(status int) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
s.renderError(w, r, status)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Handlers) respondJSON(
|
||||
w http.ResponseWriter,
|
||||
_ *http.Request,
|
||||
@@ -197,78 +163,15 @@ func (s *Handlers) respondJSON(
|
||||
}
|
||||
}
|
||||
|
||||
// serverError logs an error and answers with the 500 error page.
|
||||
// serverError logs an error and sends a 500 response.
|
||||
func (s *Handlers) serverError(
|
||||
w http.ResponseWriter, r *http.Request, msg string, err error,
|
||||
w http.ResponseWriter, msg string, err error,
|
||||
) {
|
||||
s.log.Error(msg, "error", err)
|
||||
s.renderError(w, r, http.StatusInternalServerError)
|
||||
}
|
||||
|
||||
// renderError answers with status and the error page: the normal
|
||||
// layout, one fixed line explaining the status, and a link back to the
|
||||
// webhook list, or to sign-in when nobody is signed in.
|
||||
//
|
||||
// It renders the page itself rather than through renderTemplate,
|
||||
// whose own failure comes here. If the error page cannot render
|
||||
// either, the answer is the same status in plain text: never a second
|
||||
// attempt, and never a different status.
|
||||
func (s *Handlers) renderError(
|
||||
w http.ResponseWriter,
|
||||
r *http.Request,
|
||||
status int,
|
||||
) {
|
||||
// The page names the signed-in user, and some error pages are
|
||||
// served outside the routes where NoCache runs.
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
|
||||
// No notice: one would say an action worked above a page saying
|
||||
// the request failed.
|
||||
data := s.pageData(r, map[string]any{
|
||||
"Status": status,
|
||||
"StatusText": http.StatusText(status),
|
||||
"Message": errorPageText(status),
|
||||
}, nil)
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
err := s.templates["error.html"].Execute(&buf, data)
|
||||
if err != nil {
|
||||
s.log.Error("failed to render error page", "error", err)
|
||||
http.Error(w, http.StatusText(status), status)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.WriteHeader(status)
|
||||
|
||||
_, err = buf.WriteTo(w)
|
||||
if err != nil {
|
||||
s.log.Error("failed to write error page", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// errorPageText is the line the error page shows for status. It is
|
||||
// fixed per status, so the page tells the reader no more than the
|
||||
// plain-text answers it replaced did.
|
||||
func errorPageText(status int) string {
|
||||
switch status {
|
||||
case http.StatusBadRequest:
|
||||
return "The request could not be read."
|
||||
case http.StatusForbidden:
|
||||
return "The request was refused. If it came from a form " +
|
||||
"left open for a long time, reload the page and try " +
|
||||
"again."
|
||||
case http.StatusNotFound:
|
||||
return "There is nothing here. It may have been deleted, " +
|
||||
"or the address may be wrong."
|
||||
case http.StatusServiceUnavailable:
|
||||
return "The server is busy. Please try again in a moment."
|
||||
default: // http.StatusInternalServerError
|
||||
return "Something went wrong on the server. Please try " +
|
||||
"again."
|
||||
}
|
||||
http.Error(
|
||||
w, "Internal server error",
|
||||
http.StatusInternalServerError,
|
||||
)
|
||||
}
|
||||
|
||||
// UserInfo represents user information for templates
|
||||
@@ -282,7 +185,6 @@ type templateDataWrapper struct {
|
||||
User *UserInfo
|
||||
CSRFToken string
|
||||
Version string
|
||||
Notice *notice
|
||||
Data any
|
||||
}
|
||||
|
||||
@@ -309,26 +211,12 @@ func (s *Handlers) getUserInfo(
|
||||
}
|
||||
|
||||
// renderTemplate renders a pre-parsed template with common
|
||||
// data and answers 200.
|
||||
// data
|
||||
func (s *Handlers) renderTemplate(
|
||||
w http.ResponseWriter,
|
||||
r *http.Request,
|
||||
pageTemplate string,
|
||||
data any,
|
||||
) {
|
||||
s.renderTemplateStatus(w, r, pageTemplate, data, http.StatusOK)
|
||||
}
|
||||
|
||||
// renderTemplateStatus is renderTemplate answering with status, for a
|
||||
// form shown again with an error. Call it instead of WriteHeader
|
||||
// followed by renderTemplate: the status is written only once the page
|
||||
// has rendered, so a failed render can still answer 500.
|
||||
func (s *Handlers) renderTemplateStatus(
|
||||
w http.ResponseWriter,
|
||||
r *http.Request,
|
||||
pageTemplate string,
|
||||
data any,
|
||||
status int,
|
||||
) {
|
||||
tmpl, ok := s.templates[pageTemplate]
|
||||
if !ok {
|
||||
@@ -336,22 +224,14 @@ func (s *Handlers) renderTemplateStatus(
|
||||
"template not found",
|
||||
"template", pageTemplate,
|
||||
)
|
||||
s.renderError(w, r, http.StatusInternalServerError)
|
||||
http.Error(
|
||||
w, "Internal server error",
|
||||
http.StatusInternalServerError,
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
s.executeTemplate(
|
||||
w, r, tmpl, s.pageData(r, data, noticeFor(r)), status,
|
||||
)
|
||||
}
|
||||
|
||||
// pageData adds the fields the shared layout renders to a page's own
|
||||
// data. The layout shows the notice, when there is one, above the
|
||||
// page.
|
||||
func (s *Handlers) pageData(
|
||||
r *http.Request, data any, pageNotice *notice,
|
||||
) any {
|
||||
userInfo := s.getUserInfo(r)
|
||||
csrfToken := middleware.CSRFToken(r)
|
||||
|
||||
@@ -365,34 +245,33 @@ func (s *Handlers) pageData(
|
||||
m["User"] = userInfo
|
||||
m["CSRFToken"] = csrfToken
|
||||
m["Version"] = version
|
||||
m["Notice"] = pageNotice
|
||||
s.executeTemplate(w, tmpl, m)
|
||||
|
||||
return m
|
||||
return
|
||||
}
|
||||
|
||||
return templateDataWrapper{
|
||||
wrapper := templateDataWrapper{
|
||||
User: userInfo,
|
||||
CSRFToken: csrfToken,
|
||||
Version: version,
|
||||
Notice: pageNotice,
|
||||
Data: data,
|
||||
}
|
||||
|
||||
s.executeTemplate(w, tmpl, wrapper)
|
||||
}
|
||||
|
||||
// executeTemplate renders the template into a buffer and writes status
|
||||
// and the page to the response only once rendering has fully
|
||||
// succeeded. Executing straight into the ResponseWriter commits a
|
||||
// partial body and the status before a mid-render error can be
|
||||
// reported, leaving no way to serve a 500. Buffering makes a page's
|
||||
// rendered size resident memory per concurrent viewer, so every page
|
||||
// owes it a bound: the event log caps each stored body at
|
||||
// maxRenderedBodyBytes for exactly this reason.
|
||||
// executeTemplate renders the template into a buffer and writes to
|
||||
// the response only once rendering has fully succeeded. Executing
|
||||
// straight into the ResponseWriter commits a partial body and a 200
|
||||
// status before a mid-render error can be reported, leaving no way
|
||||
// to serve a 500. Buffering makes a page's rendered size resident
|
||||
// memory per concurrent viewer, so every page owes it a bound: the
|
||||
// event log caps each stored body at maxRenderedBodyBytes for exactly
|
||||
// this reason.
|
||||
func (s *Handlers) executeTemplate(
|
||||
w http.ResponseWriter,
|
||||
r *http.Request,
|
||||
tmpl *template.Template,
|
||||
data any,
|
||||
status int,
|
||||
) {
|
||||
var buf bytes.Buffer
|
||||
|
||||
@@ -401,13 +280,15 @@ func (s *Handlers) executeTemplate(
|
||||
s.log.Error(
|
||||
"failed to execute template", "error", err,
|
||||
)
|
||||
s.renderError(w, r, http.StatusInternalServerError)
|
||||
http.Error(
|
||||
w, "Internal server error",
|
||||
http.StatusInternalServerError,
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
w.WriteHeader(status)
|
||||
|
||||
_, err = buf.WriteTo(w)
|
||||
if err != nil {
|
||||
|
||||
@@ -3,7 +3,6 @@ package handlers_test
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"html/template"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -21,7 +20,6 @@ import (
|
||||
"sneak.berlin/go/webhooker/internal/handlers"
|
||||
"sneak.berlin/go/webhooker/internal/healthcheck"
|
||||
"sneak.berlin/go/webhooker/internal/logger"
|
||||
"sneak.berlin/go/webhooker/internal/metrics"
|
||||
"sneak.berlin/go/webhooker/internal/middleware"
|
||||
"sneak.berlin/go/webhooker/internal/session"
|
||||
)
|
||||
@@ -53,103 +51,23 @@ func (n *recordingNotifier) Tasks() []delivery.Task {
|
||||
return out
|
||||
}
|
||||
|
||||
// recordingArchives is a delivery.Archives that records what it
|
||||
// was asked to do, so a test can prove that a deletion or rename
|
||||
// path reached the delivery engine. After FailRenames, every
|
||||
// rename of that target fails with the given error. After
|
||||
// BlockNextRename, the next rename is recorded and then waits.
|
||||
type recordingArchives struct {
|
||||
mu sync.Mutex
|
||||
evicted []string
|
||||
evictedTargets []string
|
||||
renames []archiveRename
|
||||
renameErrs map[string]error
|
||||
entered chan struct{}
|
||||
release chan struct{}
|
||||
// recordingEvictor is a delivery.WebhookEvictor that records
|
||||
// the webhook ids it was asked to evict, so a test can prove
|
||||
// that a deletion path reached the delivery engine.
|
||||
type recordingEvictor struct {
|
||||
mu sync.Mutex
|
||||
evicted []string
|
||||
}
|
||||
|
||||
// errInjectedRename is the failure a test hands FailRenames.
|
||||
var errInjectedRename = errors.New("injected rename failure")
|
||||
|
||||
// errNameTaken is what the delivery engine returns when a file
|
||||
// already has an archive's new name, here archive-taken.db.
|
||||
var errNameTaken = fmt.Errorf(
|
||||
"%w: archive-taken.db", delivery.ErrArchiveNameTaken,
|
||||
)
|
||||
|
||||
// archiveRename is one recorded Rename call.
|
||||
type archiveRename struct {
|
||||
TargetID string
|
||||
WebhookName string
|
||||
TargetName string
|
||||
}
|
||||
|
||||
func (r *recordingArchives) EvictWebhook(webhookID string) {
|
||||
func (r *recordingEvictor) EvictWebhook(webhookID string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
r.evicted = append(r.evicted, webhookID)
|
||||
}
|
||||
|
||||
func (r *recordingArchives) EvictTarget(targetID string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
r.evictedTargets = append(r.evictedTargets, targetID)
|
||||
}
|
||||
|
||||
func (r *recordingArchives) Rename(
|
||||
targetID, webhookName, targetName string,
|
||||
) error {
|
||||
r.mu.Lock()
|
||||
|
||||
r.renames = append(r.renames, archiveRename{
|
||||
TargetID: targetID,
|
||||
WebhookName: webhookName,
|
||||
TargetName: targetName,
|
||||
})
|
||||
err := r.renameErrs[targetID]
|
||||
entered, release := r.entered, r.release
|
||||
r.entered, r.release = nil, nil
|
||||
|
||||
r.mu.Unlock()
|
||||
|
||||
if entered != nil {
|
||||
close(entered)
|
||||
<-release
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// BlockNextRename makes the next rename, once recorded, wait until
|
||||
// the returned release is called. The returned channel is closed
|
||||
// when that rename starts waiting.
|
||||
func (r *recordingArchives) BlockNextRename() (<-chan struct{}, func()) {
|
||||
entered := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
|
||||
r.mu.Lock()
|
||||
r.entered, r.release = entered, release
|
||||
r.mu.Unlock()
|
||||
|
||||
return entered, func() { close(release) }
|
||||
}
|
||||
|
||||
// FailRenames makes every later rename of targetID fail with err.
|
||||
func (r *recordingArchives) FailRenames(targetID string, err error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
if r.renameErrs == nil {
|
||||
r.renameErrs = map[string]error{}
|
||||
}
|
||||
|
||||
r.renameErrs[targetID] = err
|
||||
}
|
||||
|
||||
// Evicted returns a copy of the recorded webhook ids.
|
||||
func (r *recordingArchives) Evicted() []string {
|
||||
func (r *recordingEvictor) Evicted() []string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
@@ -159,62 +77,22 @@ func (r *recordingArchives) Evicted() []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// EvictedTargets returns a copy of the recorded target ids.
|
||||
func (r *recordingArchives) EvictedTargets() []string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
out := make([]string, len(r.evictedTargets))
|
||||
copy(out, r.evictedTargets)
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// Renames returns a copy of the recorded renames.
|
||||
func (r *recordingArchives) Renames() []archiveRename {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
out := make([]archiveRename, len(r.renames))
|
||||
copy(out, r.renames)
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// newTestApp returns an app whose RequireStart fails the test when
|
||||
// starting takes longer than fx's default start timeout of 15s. That
|
||||
// limit catches a start that hangs, not a busy host: measured with make
|
||||
// test on 2026-10-02 at host load 58-69 on 48 cores, the slowest of this
|
||||
// package's starts took 0.49s.
|
||||
func newTestApp(
|
||||
t *testing.T,
|
||||
targets ...any,
|
||||
) *fxtest.App {
|
||||
t.Helper()
|
||||
|
||||
return newTestAppWithConfig(
|
||||
t, &config.Config{DataDir: t.TempDir()}, targets...,
|
||||
)
|
||||
}
|
||||
|
||||
// newTestAppWithConfig is newTestApp over a caller-supplied Config.
|
||||
func newTestAppWithConfig(
|
||||
t *testing.T,
|
||||
cfg *config.Config,
|
||||
targets ...any,
|
||||
) *fxtest.App {
|
||||
t.Helper()
|
||||
|
||||
return fxtest.New(
|
||||
t,
|
||||
// fx's own log is discarded, not sent to t.Logf: a hook still
|
||||
// running after a start or stop timeout would write there after
|
||||
// the test has returned.
|
||||
fx.NopLogger,
|
||||
fx.Provide(
|
||||
globals.New,
|
||||
logger.New,
|
||||
func() *config.Config { return cfg },
|
||||
func() *config.Config {
|
||||
return &config.Config{
|
||||
DataDir: t.TempDir(),
|
||||
}
|
||||
},
|
||||
database.New,
|
||||
database.NewWebhookDBManager,
|
||||
healthcheck.New,
|
||||
@@ -225,14 +103,12 @@ func newTestAppWithConfig(
|
||||
func(n *recordingNotifier) delivery.Notifier {
|
||||
return n
|
||||
},
|
||||
func() *recordingArchives {
|
||||
return &recordingArchives{}
|
||||
func() *recordingEvictor {
|
||||
return &recordingEvictor{}
|
||||
},
|
||||
func(r *recordingArchives) delivery.Archives {
|
||||
func(r *recordingEvictor) delivery.WebhookEvictor {
|
||||
return r
|
||||
},
|
||||
metrics.NewRegistry,
|
||||
metrics.New,
|
||||
middleware.New,
|
||||
delivery.NewGuard,
|
||||
handlers.New,
|
||||
@@ -300,7 +176,7 @@ func TestHandleIndex_Authenticated(t *testing.T) {
|
||||
|
||||
assert.Equal(t, http.StatusSeeOther, w2.Code)
|
||||
assert.Equal(
|
||||
t, "/hooks", w2.Header().Get("Location"),
|
||||
t, "/sources", w2.Header().Get("Location"),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -431,14 +307,10 @@ func TestRenderTemplateMidRenderErrorSendsNoPartialBody(t *testing.T) {
|
||||
t, http.StatusInternalServerError, w.Code,
|
||||
"a failed render must report a 500",
|
||||
)
|
||||
assert.NotContains(
|
||||
t, w.Body.String(), partialPageMarker,
|
||||
assert.Equal(
|
||||
t, "Internal server error\n", w.Body.String(),
|
||||
"the response must carry no part of the aborted page",
|
||||
)
|
||||
assert.Contains(
|
||||
t, w.Body.String(), "500 Internal Server Error",
|
||||
"a failed render must answer with the error page",
|
||||
)
|
||||
}
|
||||
|
||||
func TestBuildDatabaseTargetConfig_Valid(t *testing.T) {
|
||||
|
||||
@@ -5,13 +5,13 @@ import (
|
||||
)
|
||||
|
||||
// HandleIndex returns a handler for the root path that redirects
|
||||
// based on authentication state: authenticated users go to /hooks
|
||||
// based on authentication state: authenticated users go to /sources
|
||||
// (the dashboard), unauthenticated users go to the login page.
|
||||
func (s *Handlers) HandleIndex() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
sess, err := s.session.Get(r)
|
||||
if err == nil && s.session.IsAuthenticated(sess) {
|
||||
http.Redirect(w, r, "/hooks", http.StatusSeeOther)
|
||||
http.Redirect(w, r, "/sources", http.StatusSeeOther)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ package handlers_test
|
||||
// this package reach a value an UNAUTHENTICATED client picks outright
|
||||
// and of a length it picks outright:
|
||||
//
|
||||
// - the unknown-entrypoint DEBUG line on /h/{uuid}, whose
|
||||
// - the unknown-entrypoint DEBUG line on /webhook/{uuid}, whose
|
||||
// path segment matched no stored entrypoint and so is bounded by
|
||||
// nothing;
|
||||
// - the failed-login DEBUG lines, whose username is a form field.
|
||||
@@ -190,12 +190,12 @@ func assertNoClientText(t *testing.T, buf *bytes.Buffer) {
|
||||
// route pattern.
|
||||
func receiverRouter(h *handlers.Handlers) *chi.Mux {
|
||||
router := chi.NewRouter()
|
||||
router.Post("/h/{uuid}", h.HandleWebhook())
|
||||
router.Post("/webhook/{uuid}", h.HandleWebhook())
|
||||
|
||||
return router
|
||||
}
|
||||
|
||||
// postReceiver sends one POST at /h/<segment>.
|
||||
// postReceiver sends one POST at /webhook/<segment>.
|
||||
//
|
||||
// RawPath is cleared after parsing so chi routes on the decoded path
|
||||
// and the handler sees the raw bytes rather than their percent-escaped
|
||||
@@ -210,7 +210,7 @@ func postReceiver(
|
||||
req := httptest.NewRequestWithContext(
|
||||
context.Background(),
|
||||
http.MethodPost,
|
||||
"/h/"+url.PathEscape(segment),
|
||||
"/webhook/"+url.PathEscape(segment),
|
||||
strings.NewReader(""),
|
||||
)
|
||||
req.URL.RawPath = ""
|
||||
@@ -507,7 +507,7 @@ func TestVerificationCapacity_LogLineDoesNotTrackPathSize(
|
||||
http.StatusServiceUnavailable,
|
||||
postLoginAtPath(
|
||||
t, h,
|
||||
"/hook/"+url.PathEscape(
|
||||
"/source/"+url.PathEscape(
|
||||
oversizedFill(fill),
|
||||
)+"/login",
|
||||
),
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
)
|
||||
|
||||
// HandleMetrics returns the Prometheus scrape handler for the
|
||||
// registry built by metrics.NewRegistry, which the HTTP, delivery, Go
|
||||
// runtime and process collectors register on. It is what
|
||||
// promhttp.Handler builds for the global default registry, including
|
||||
// the promhttp_metric_handler_* series that count scrapes, pointed at
|
||||
// that registry instead.
|
||||
func (s *Handlers) HandleMetrics() http.HandlerFunc {
|
||||
reg := s.params.Registry
|
||||
|
||||
return promhttp.InstrumentMetricHandler(
|
||||
reg, promhttp.HandlerFor(reg, promhttp.HandlerOpts{}),
|
||||
).ServeHTTP
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import "net/http"
|
||||
|
||||
// noticeParam is the query parameter an action's redirect carries its
|
||||
// notice code in.
|
||||
const noticeParam = "notice"
|
||||
|
||||
// noticeCode names one of the fixed lines noticeFor knows. An action
|
||||
// redirects with the code rather than the line, so nothing a client
|
||||
// puts in the URL reaches the page: a code noticeFor does not know
|
||||
// shows nothing.
|
||||
type noticeCode string
|
||||
|
||||
// The codes of the actions on the webhook pages and of signing out.
|
||||
// Replay's codes, with the reasons a replay can be refused, and
|
||||
// resubmit's codes are defined beside those actions.
|
||||
const (
|
||||
webhookCreated noticeCode = "webhook-created"
|
||||
webhookSaved noticeCode = "webhook-saved"
|
||||
webhookDeleted noticeCode = "webhook-deleted"
|
||||
entrypointAdded noticeCode = "entrypoint-added"
|
||||
entrypointDeleted noticeCode = "entrypoint-deleted"
|
||||
entrypointActivated noticeCode = "entrypoint-activated"
|
||||
entrypointDeactivated noticeCode = "entrypoint-deactivated"
|
||||
targetAdded noticeCode = "target-added"
|
||||
targetSaved noticeCode = "target-saved"
|
||||
targetDeleted noticeCode = "target-deleted"
|
||||
targetActivated noticeCode = "target-activated"
|
||||
targetDeactivated noticeCode = "target-deactivated"
|
||||
signedOut noticeCode = "signed-out"
|
||||
)
|
||||
|
||||
// notice is the line templates/notice.html shows above a page to say
|
||||
// what an action did.
|
||||
type notice struct {
|
||||
Text string
|
||||
|
||||
// Failed shows the line as an error: the action was refused.
|
||||
Failed bool
|
||||
}
|
||||
|
||||
// noticeFor returns the notice the request's URL names, or nil when it
|
||||
// names none or an unknown code.
|
||||
func noticeFor(r *http.Request) *notice {
|
||||
n, ok := map[noticeCode]notice{
|
||||
webhookCreated: {Text: "Webhook created."},
|
||||
webhookSaved: {Text: "Webhook saved."},
|
||||
webhookDeleted: {Text: "Webhook deleted."},
|
||||
entrypointAdded: {Text: "Entrypoint added."},
|
||||
entrypointDeleted: {Text: "Entrypoint deleted."},
|
||||
entrypointActivated: {Text: "Entrypoint activated."},
|
||||
entrypointDeactivated: {Text: "Entrypoint deactivated."},
|
||||
targetAdded: {Text: "Target added."},
|
||||
targetSaved: {Text: "Target saved."},
|
||||
targetDeleted: {Text: "Target deleted."},
|
||||
targetActivated: {Text: "Target activated."},
|
||||
targetDeactivated: {Text: "Target deactivated."},
|
||||
signedOut: {Text: "Signed out."},
|
||||
|
||||
replayQueued: {
|
||||
Text: "Replay queued: a new delivery was created " +
|
||||
"against the target's current configuration.",
|
||||
},
|
||||
replayTargetDeleted: {
|
||||
Text: "Not replayed: the target this delivery was for " +
|
||||
"has been deleted. Recreate the target, then replay.",
|
||||
Failed: true,
|
||||
},
|
||||
replayTargetMissing: {
|
||||
Text: "Not replayed: the target this delivery was for " +
|
||||
"no longer exists.",
|
||||
Failed: true,
|
||||
},
|
||||
replayTargetInactive: {
|
||||
Text: "Not replayed: the target this delivery was for " +
|
||||
"is deactivated. Activate it, then replay.",
|
||||
Failed: true,
|
||||
},
|
||||
replayNotTerminal: {
|
||||
Text: "Not replayed: this delivery has not finished yet.",
|
||||
Failed: true,
|
||||
},
|
||||
replayInFlight: {
|
||||
Text: "Not replayed: a delivery of this event to this " +
|
||||
"target is already in flight.",
|
||||
Failed: true,
|
||||
},
|
||||
|
||||
resubmitQueued: {
|
||||
Text: "Resubmitted: a new event was created from the " +
|
||||
"stored one and queued to every active target.",
|
||||
},
|
||||
resubmitNoTargets: {
|
||||
Text: "Resubmitted: a new event was created, but this " +
|
||||
"source has no active targets, so nothing was queued.",
|
||||
},
|
||||
}[noticeCode(r.URL.Query().Get(noticeParam))]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &n
|
||||
}
|
||||
|
||||
// withNotice returns path with code added as its notice.
|
||||
func withNotice(path string, code noticeCode) string {
|
||||
return path + "?" + noticeParam + "=" + string(code)
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi"
|
||||
@@ -36,14 +37,14 @@ func (h *Handlers) HandlePasswordChange() http.HandlerFunc {
|
||||
err := r.ParseForm()
|
||||
if err != nil {
|
||||
h.log.Error("failed to parse form", "error", err)
|
||||
h.renderError(w, r, http.StatusBadRequest)
|
||||
http.Error(w, "Bad request", http.StatusBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
successMessage, errorMessage, handled := h.applyPasswordChange(
|
||||
r.Context(),
|
||||
w,
|
||||
r,
|
||||
sessionUsername,
|
||||
// PostFormValue, not FormValue: the credential must
|
||||
// come from the body, never from the query string.
|
||||
@@ -65,12 +66,12 @@ func (h *Handlers) HandlePasswordChange() http.HandlerFunc {
|
||||
// applyPasswordChange verifies the current password and, on success,
|
||||
// persists a fresh hash for the user, reusing the same helpers that
|
||||
// bootstrap the admin user. It returns the success and error messages
|
||||
// to display on the profile page. On an internal failure it writes the
|
||||
// error page itself and returns handled=false, signalling the caller
|
||||
// to display on the profile page. On an internal failure it writes a
|
||||
// 500 response itself and returns handled=false, signalling the caller
|
||||
// to stop without re-rendering the page.
|
||||
func (h *Handlers) applyPasswordChange(
|
||||
ctx context.Context,
|
||||
w http.ResponseWriter,
|
||||
r *http.Request,
|
||||
username, currentPassword, newPassword, confirmPassword string,
|
||||
) (string, string, bool) {
|
||||
// This endpoint verifies one password and hashes another, at
|
||||
@@ -78,10 +79,15 @@ func (h *Handlers) applyPasswordChange(
|
||||
// endpoint uses. The bound is per hash, not per endpoint: leaving
|
||||
// this path outside it would leave a hole in it. The slot is held
|
||||
// across both hashes.
|
||||
release, ok := h.mw.BeginPasswordVerification(r.Context())
|
||||
release, ok := h.mw.BeginPasswordVerification(ctx)
|
||||
if !ok {
|
||||
h.log.Warn("password verification capacity exhausted")
|
||||
h.renderError(w, r, http.StatusServiceUnavailable)
|
||||
http.Error(
|
||||
w,
|
||||
"The server is busy verifying credentials. "+
|
||||
"Please try again.",
|
||||
http.StatusServiceUnavailable,
|
||||
)
|
||||
|
||||
return "", "", false
|
||||
}
|
||||
@@ -97,7 +103,7 @@ func (h *Handlers) applyPasswordChange(
|
||||
).First(&user).Error
|
||||
if err != nil {
|
||||
h.serverError(
|
||||
w, r, "failed to load user for password change", err,
|
||||
w, "failed to load user for password change", err,
|
||||
)
|
||||
|
||||
return "", "", false
|
||||
@@ -107,7 +113,7 @@ func (h *Handlers) applyPasswordChange(
|
||||
currentPassword, user.Password,
|
||||
)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to verify password", err)
|
||||
h.serverError(w, "failed to verify password", err)
|
||||
|
||||
return "", "", false
|
||||
}
|
||||
@@ -126,7 +132,7 @@ func (h *Handlers) applyPasswordChange(
|
||||
|
||||
hashedPassword, err := database.HashPassword(newPassword)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to hash new password", err)
|
||||
h.serverError(w, "failed to hash new password", err)
|
||||
|
||||
return "", "", false
|
||||
}
|
||||
@@ -135,7 +141,7 @@ func (h *Handlers) applyPasswordChange(
|
||||
"password", hashedPassword,
|
||||
).Error
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to update password", err)
|
||||
h.serverError(w, "failed to update password", err)
|
||||
|
||||
return "", "", false
|
||||
}
|
||||
@@ -156,7 +162,7 @@ func (h *Handlers) profileOwnerOrDeny(
|
||||
) (string, string, bool) {
|
||||
requestedUsername := chi.URLParam(r, "username")
|
||||
if requestedUsername == "" {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
http.NotFound(w, r)
|
||||
|
||||
return "", "", false
|
||||
}
|
||||
@@ -166,7 +172,7 @@ func (h *Handlers) profileOwnerOrDeny(
|
||||
// unexpected retrieval error.
|
||||
sess, err := h.session.Get(r)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to get session", err)
|
||||
h.serverError(w, "failed to get session", err)
|
||||
|
||||
return "", "", false
|
||||
}
|
||||
@@ -174,7 +180,10 @@ func (h *Handlers) profileOwnerOrDeny(
|
||||
sessionUsername, ok := h.session.GetUsername(sess)
|
||||
if !ok {
|
||||
h.log.Error("authenticated session missing username")
|
||||
h.renderError(w, r, http.StatusInternalServerError)
|
||||
http.Error(
|
||||
w, "Internal server error",
|
||||
http.StatusInternalServerError,
|
||||
)
|
||||
|
||||
return "", "", false
|
||||
}
|
||||
@@ -182,14 +191,17 @@ func (h *Handlers) profileOwnerOrDeny(
|
||||
sessionUserID, ok := h.session.GetUserID(sess)
|
||||
if !ok {
|
||||
h.log.Error("authenticated session missing user ID")
|
||||
h.renderError(w, r, http.StatusInternalServerError)
|
||||
http.Error(
|
||||
w, "Internal server error",
|
||||
http.StatusInternalServerError,
|
||||
)
|
||||
|
||||
return "", "", false
|
||||
}
|
||||
|
||||
// Only allow users to act on their own profile.
|
||||
if requestedUsername != sessionUsername {
|
||||
h.renderError(w, r, http.StatusForbidden)
|
||||
http.Error(w, "Forbidden", http.StatusForbidden)
|
||||
|
||||
return "", "", false
|
||||
}
|
||||
|
||||
@@ -128,9 +128,7 @@ func TestUserRoute_Unauthenticated_RedirectedByMiddleware(t *testing.T) {
|
||||
|
||||
var sess *session.Session
|
||||
|
||||
var h *handlers.Handlers
|
||||
|
||||
app := newTestApp(t, &log, &cfg, &sess, &h)
|
||||
app := newTestApp(t, &log, &cfg, &sess)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
@@ -141,7 +139,7 @@ func TestUserRoute_Unauthenticated_RedirectedByMiddleware(t *testing.T) {
|
||||
|
||||
router := chi.NewRouter()
|
||||
router.Route("/user/{username}", func(r chi.Router) {
|
||||
r.Use(mw.CSRF(h.HandleErrorPage(http.StatusForbidden)))
|
||||
r.Use(mw.CSRF())
|
||||
r.Use(mw.RequireAuth())
|
||||
r.Get("/", func(w http.ResponseWriter, _ *http.Request) {
|
||||
handlerReached = true
|
||||
@@ -162,10 +160,7 @@ func TestUserRoute_Unauthenticated_RedirectedByMiddleware(t *testing.T) {
|
||||
"handler must not be reached for unauthenticated request",
|
||||
)
|
||||
assert.Equal(t, http.StatusSeeOther, w.Code)
|
||||
assert.Equal(
|
||||
t, "/pages/login?next=%2Fuser%2Ftestuser",
|
||||
w.Header().Get("Location"),
|
||||
)
|
||||
assert.Equal(t, "/pages/login", w.Header().Get("Location"))
|
||||
}
|
||||
|
||||
// passwordChangeRequest builds a POST request to the password-change
|
||||
|
||||
@@ -313,7 +313,7 @@ func TestHandleWebhook_RecordsBodySize(t *testing.T) {
|
||||
body := strings.Repeat("é", 1024)
|
||||
|
||||
req := httptest.NewRequestWithContext(
|
||||
context.Background(), http.MethodPost, "/h/x",
|
||||
context.Background(), http.MethodPost, "/webhook/x",
|
||||
strings.NewReader(body),
|
||||
)
|
||||
|
||||
|
||||
@@ -1,128 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"sneak.berlin/go/webhooker/internal/config"
|
||||
)
|
||||
|
||||
// notSet is what the Settings page shows for a value that is empty.
|
||||
const notSet = "not set"
|
||||
|
||||
// settingRow is one line of the Settings page: an environment
|
||||
// variable, what it controls, and the value the server loaded for it.
|
||||
type settingRow struct {
|
||||
Name string
|
||||
Description string
|
||||
Value string
|
||||
}
|
||||
|
||||
// HandleSettings returns a handler for the read-only Settings page,
|
||||
// which lists the configuration the server started with.
|
||||
func (h *Handlers) HandleSettings() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
h.renderTemplate(w, r, "settings.html", map[string]any{
|
||||
"Settings": settingRows(h.params.Config),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// settingRows lists every field of cfg under the environment variable
|
||||
// it is read from, with the description the README's configuration
|
||||
// table gives it (less its pointers to other README sections), in the
|
||||
// table's order. METRICS_PASSWORD and SENTRY_DSN are credentials, so
|
||||
// their values never reach the page: only whether they are set.
|
||||
func settingRows(cfg *config.Config) []settingRow {
|
||||
metricsUsername := cfg.MetricsUsername
|
||||
if metricsUsername == "" {
|
||||
metricsUsername = notSet
|
||||
}
|
||||
|
||||
return []settingRow{
|
||||
{"WEBHOOKER_ENVIRONMENT", "dev or prod", cfg.Environment},
|
||||
{"PORT", "HTTP listen port", strconv.Itoa(cfg.Port)},
|
||||
{
|
||||
"BIND_ADDRESS",
|
||||
"IP address the HTTP listener binds. Loopback by default, " +
|
||||
"so the cleartext listener is not published on every " +
|
||||
"interface. The Docker image ships 0.0.0.0 instead",
|
||||
cfg.BindAddress,
|
||||
},
|
||||
{"DATA_DIR", "Directory for all SQLite databases", cfg.DataDir},
|
||||
{"DEBUG", "Enable debug logging", strconv.FormatBool(cfg.Debug)},
|
||||
{
|
||||
"METRICS_USERNAME",
|
||||
"Basic auth username for /metrics. Must be set together " +
|
||||
"with METRICS_PASSWORD; one without the other fails " +
|
||||
"startup",
|
||||
metricsUsername,
|
||||
},
|
||||
{
|
||||
"METRICS_PASSWORD",
|
||||
"Basic auth password for /metrics. Must be set together " +
|
||||
"with METRICS_USERNAME; one without the other fails " +
|
||||
"startup",
|
||||
setOrNotSet(cfg.MetricsPassword),
|
||||
},
|
||||
{
|
||||
"SENTRY_DSN",
|
||||
"Sentry error reporting DSN. Unset leaves error reporting " +
|
||||
"off; a value the Sentry SDK cannot parse fails startup " +
|
||||
"rather than serving with reporting silently off",
|
||||
setOrNotSet(cfg.SentryDSN),
|
||||
},
|
||||
{
|
||||
"RETENTION_SWEEP_INTERVAL",
|
||||
"How often the retention reaper and archive sweeper run " +
|
||||
"(Go duration, must be positive)",
|
||||
cfg.RetentionSweepInterval.String(),
|
||||
},
|
||||
{
|
||||
"SESSION_IDLE_TIMEOUT",
|
||||
"Idle session timeout (Go duration)",
|
||||
cfg.SessionIdleTimeout.String(),
|
||||
},
|
||||
{
|
||||
"RECEIVER_RATE_LIMIT",
|
||||
"Receiver requests/minute per IP per entrypoint " +
|
||||
"(10x that per IP across the route)",
|
||||
strconv.Itoa(cfg.ReceiverRateLimit),
|
||||
},
|
||||
{
|
||||
"TRUSTED_PROXIES",
|
||||
"CIDRs whose forwarded headers are trusted. A set value " +
|
||||
"replaces the default. If any client can reach webhooker, " +
|
||||
"or the proxy in front of it, from an RFC 1918 source " +
|
||||
"address, set it to the proxy's address alone",
|
||||
cidrList(cfg.TrustedProxies),
|
||||
},
|
||||
{
|
||||
"ALLOWED_EGRESS_CIDRS",
|
||||
"CIDRs that delivery targets may reach despite the " +
|
||||
"SSRF blocklist",
|
||||
cidrList(cfg.AllowedEgressCIDRs),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// setOrNotSet is how the Settings page shows a credential: whether it
|
||||
// has a value, never the value itself.
|
||||
func setOrNotSet(value string) string {
|
||||
if value == "" {
|
||||
return notSet
|
||||
}
|
||||
|
||||
return "set"
|
||||
}
|
||||
|
||||
// cidrList renders a CIDR list setting for the Settings page.
|
||||
func cidrList(prefixes []netip.Prefix) string {
|
||||
if len(prefixes) == 0 {
|
||||
return "none"
|
||||
}
|
||||
|
||||
return strings.Join(config.PrefixStrings(prefixes), ", ")
|
||||
}
|
||||
@@ -1,148 +0,0 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"html"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"regexp"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sneak.berlin/go/webhooker/internal/config"
|
||||
"sneak.berlin/go/webhooker/internal/handlers"
|
||||
"sneak.berlin/go/webhooker/internal/session"
|
||||
)
|
||||
|
||||
// settingsShown renders the Settings page over cfg as a logged-in user
|
||||
// and returns the value it shows for each variable name, plus the
|
||||
// whole page.
|
||||
func settingsShown(
|
||||
t *testing.T, cfg *config.Config,
|
||||
) (map[string]string, string) {
|
||||
t.Helper()
|
||||
|
||||
var h *handlers.Handlers
|
||||
|
||||
var sess *session.Session
|
||||
|
||||
app := newTestAppWithConfig(t, cfg, &h, &sess)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
req := httptest.NewRequestWithContext(
|
||||
context.Background(), http.MethodGet, "/settings", nil,
|
||||
)
|
||||
for _, c := range authenticatedCookies(t, sess, "id", "admin") {
|
||||
req.AddCookie(c)
|
||||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
h.HandleSettings().ServeHTTP(w, req)
|
||||
require.Equal(t, http.StatusOK, w.Code)
|
||||
|
||||
body := w.Body.String()
|
||||
|
||||
row := regexp.MustCompile(
|
||||
`<code[^>]*>([A-Z_]+)</code>\s*<code[^>]*>([^<]*)</code>`,
|
||||
)
|
||||
|
||||
shown := map[string]string{}
|
||||
for _, match := range row.FindAllStringSubmatch(body, -1) {
|
||||
shown[match[1]] = html.UnescapeString(match[2])
|
||||
}
|
||||
|
||||
return shown, body
|
||||
}
|
||||
|
||||
func TestSettingsPageShowsLoadedConfiguration(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Each of METRICS_USERNAME, METRICS_PASSWORD and SENTRY_DSN is the
|
||||
// only one of the three set in one of the content tests, so each
|
||||
// row is checked against its own field.
|
||||
cfg := &config.Config{
|
||||
DataDir: t.TempDir(),
|
||||
Debug: true,
|
||||
Environment: config.EnvironmentDev,
|
||||
MetricsUsername: "scraper",
|
||||
MetricsPassword: "",
|
||||
Port: 9123,
|
||||
SentryDSN: "",
|
||||
BindAddress: "192.0.2.10",
|
||||
RetentionSweepInterval: 17 * time.Minute,
|
||||
SessionIdleTimeout: 3 * time.Hour,
|
||||
ReceiverRateLimit: 77,
|
||||
TrustedProxies: []netip.Prefix{
|
||||
netip.MustParsePrefix("10.1.0.0/16"),
|
||||
},
|
||||
AllowedEgressCIDRs: []netip.Prefix{
|
||||
netip.MustParsePrefix("192.168.5.0/24"),
|
||||
netip.MustParsePrefix("fd00::/8"),
|
||||
},
|
||||
}
|
||||
|
||||
shown, body := settingsShown(t, cfg)
|
||||
|
||||
assert.Equal(t, map[string]string{
|
||||
"WEBHOOKER_ENVIRONMENT": "dev",
|
||||
"PORT": "9123",
|
||||
"BIND_ADDRESS": "192.0.2.10",
|
||||
"DATA_DIR": cfg.DataDir,
|
||||
"DEBUG": "true",
|
||||
"METRICS_USERNAME": "scraper",
|
||||
"METRICS_PASSWORD": "not set",
|
||||
"SENTRY_DSN": "not set",
|
||||
"RETENTION_SWEEP_INTERVAL": "17m0s",
|
||||
"SESSION_IDLE_TIMEOUT": "3h0m0s",
|
||||
"RECEIVER_RATE_LIMIT": "77",
|
||||
"TRUSTED_PROXIES": "10.1.0.0/16",
|
||||
"ALLOWED_EGRESS_CIDRS": "192.168.5.0/24, fd00::/8",
|
||||
}, shown)
|
||||
|
||||
assert.Contains(
|
||||
t, body, `href="/settings"`,
|
||||
"the navigation bar links to the page",
|
||||
)
|
||||
}
|
||||
|
||||
func TestSettingsPageShowsUnsetValues(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const metricsPassword = "metrics-password-1f9a"
|
||||
|
||||
shown, body := settingsShown(t, &config.Config{
|
||||
DataDir: t.TempDir(),
|
||||
MetricsPassword: metricsPassword,
|
||||
})
|
||||
|
||||
assert.Equal(t, "not set", shown["METRICS_USERNAME"])
|
||||
assert.Equal(t, "set", shown["METRICS_PASSWORD"])
|
||||
assert.Equal(t, "not set", shown["SENTRY_DSN"])
|
||||
assert.NotContains(t, body, metricsPassword)
|
||||
assert.Equal(t, "none", shown["TRUSTED_PROXIES"])
|
||||
assert.Equal(t, "none", shown["ALLOWED_EGRESS_CIDRS"])
|
||||
}
|
||||
|
||||
func TestSettingsPageShowsSentryDSNOnlyAsSet(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
sentryKey = "dsnkey7c2e"
|
||||
sentryDSN = "https://" + sentryKey + "@errors.example.com/42"
|
||||
)
|
||||
|
||||
shown, body := settingsShown(t, &config.Config{
|
||||
DataDir: t.TempDir(),
|
||||
SentryDSN: sentryDSN,
|
||||
})
|
||||
|
||||
assert.Equal(t, "not set", shown["METRICS_USERNAME"])
|
||||
assert.Equal(t, "not set", shown["METRICS_PASSWORD"])
|
||||
assert.Equal(t, "set", shown["SENTRY_DSN"])
|
||||
assert.NotContains(t, body, sentryKey)
|
||||
}
|
||||
@@ -15,7 +15,6 @@ import (
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/delivery"
|
||||
"sneak.berlin/go/webhooker/internal/handlers"
|
||||
"sneak.berlin/go/webhooker/internal/session"
|
||||
)
|
||||
@@ -80,10 +79,6 @@ func seedTarget(
|
||||
// from a delete statement.
|
||||
var errInjectedDelete = errors.New("injected delete failure")
|
||||
|
||||
// errInjectedSave is the failure failSaveOnTable reports from a
|
||||
// save of an existing row.
|
||||
var errInjectedSave = errors.New("injected save failure")
|
||||
|
||||
// seedEntrypoint inserts an entrypoint for a webhook.
|
||||
func seedEntrypoint(
|
||||
t *testing.T,
|
||||
@@ -151,42 +146,19 @@ func failDeleteOnTable(
|
||||
)
|
||||
}
|
||||
|
||||
// failSaveOnTable is failDeleteOnTable for saves: every update of
|
||||
// an existing row in the named table fails.
|
||||
func failSaveOnTable(
|
||||
t *testing.T,
|
||||
db *database.Database,
|
||||
table string,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
require.NoError(t, db.DB().Callback().Update().
|
||||
Before("gorm:update").
|
||||
Register(
|
||||
"test:fail_save_"+table,
|
||||
func(tx *gorm.DB) {
|
||||
if tx.Statement.Table == table {
|
||||
_ = tx.AddError(errInjectedSave)
|
||||
}
|
||||
},
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
// archivePathFor returns the archive database path the
|
||||
// delivery engine would use for a database target: beside the
|
||||
// webhook's event database in the data directory.
|
||||
// delivery engine would use for a webhook: beside the webhook's
|
||||
// event database in the data directory.
|
||||
func archivePathFor(
|
||||
t *testing.T,
|
||||
mgr *database.WebhookDBManager,
|
||||
wh *database.Webhook,
|
||||
tgt *database.Target,
|
||||
webhookID string,
|
||||
) string {
|
||||
t.Helper()
|
||||
|
||||
return filepath.Join(
|
||||
filepath.Dir(mgr.DBPath(wh.ID)),
|
||||
delivery.ArchiveFileName(wh.Name, tgt.Name, tgt.ID),
|
||||
filepath.Dir(mgr.DBPath(webhookID)),
|
||||
"archive-"+webhookID+".db",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -223,8 +195,8 @@ func postRequest(
|
||||
|
||||
// TestHandleSourceDelete_EvictsArchiveWriter proves that
|
||||
// deleting a webhook reaches the delivery engine and releases
|
||||
// the webhook's archive writers, exercised through the real
|
||||
// deletion handler rather than by calling the engine directly.
|
||||
// the webhook's archive writer, exercised through the real
|
||||
// deletion handler rather than by calling the evictor directly.
|
||||
func TestHandleSourceDelete_EvictsArchiveWriter(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -232,7 +204,7 @@ func TestHandleSourceDelete_EvictsArchiveWriter(t *testing.T) {
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
ev *recordingArchives
|
||||
ev *recordingEvictor
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &ev)
|
||||
@@ -248,7 +220,7 @@ func TestHandleSourceDelete_EvictsArchiveWriter(t *testing.T) {
|
||||
)
|
||||
|
||||
req := postRequest(
|
||||
"/hook/"+wh.ID+"/delete",
|
||||
"/source/"+wh.ID+"/delete",
|
||||
cookies,
|
||||
map[string]string{paramSourceID: wh.ID},
|
||||
)
|
||||
@@ -282,10 +254,9 @@ func TestHandleSourceDelete_KeepsArchiveFile(t *testing.T) {
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
wh := seedWebhook(t, db)
|
||||
tgt := seedTarget(t, db, wh.ID, database.TargetTypeDatabase)
|
||||
|
||||
// Place an archive file where the delivery engine would.
|
||||
archivePath := archivePathFor(t, mgr, wh, tgt)
|
||||
archivePath := archivePathFor(t, mgr, wh.ID)
|
||||
require.NoError(
|
||||
t,
|
||||
writeArchivePlaceholder(archivePath),
|
||||
@@ -296,7 +267,7 @@ func TestHandleSourceDelete_KeepsArchiveFile(t *testing.T) {
|
||||
)
|
||||
|
||||
req := postRequest(
|
||||
"/hook/"+wh.ID+"/delete",
|
||||
"/source/"+wh.ID+"/delete",
|
||||
cookies,
|
||||
map[string]string{paramSourceID: wh.ID},
|
||||
)
|
||||
@@ -352,7 +323,7 @@ func TestHandleSourceDelete_FailedDeleteKeepsEverything(
|
||||
)
|
||||
|
||||
req := postRequest(
|
||||
"/hook/"+wh.ID+"/delete",
|
||||
"/source/"+wh.ID+"/delete",
|
||||
cookies,
|
||||
map[string]string{paramSourceID: wh.ID},
|
||||
)
|
||||
@@ -366,7 +337,7 @@ func TestHandleSourceDelete_FailedDeleteKeepsEverything(
|
||||
)
|
||||
assert.Empty(
|
||||
t, w.Header().Get("Location"),
|
||||
"a failed deletion must not redirect to /hooks",
|
||||
"a failed deletion must not redirect to /sources",
|
||||
)
|
||||
|
||||
assert.Equal(
|
||||
@@ -431,7 +402,7 @@ func TestHandleSourceDelete_RemovesConfigAndEventDatabase(
|
||||
)
|
||||
|
||||
req := postRequest(
|
||||
"/hook/"+wh.ID+"/delete",
|
||||
"/source/"+wh.ID+"/delete",
|
||||
cookies,
|
||||
map[string]string{paramSourceID: wh.ID},
|
||||
)
|
||||
@@ -440,9 +411,7 @@ func TestHandleSourceDelete_RemovesConfigAndEventDatabase(
|
||||
h.HandleSourceDelete().ServeHTTP(w, req)
|
||||
|
||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||
assert.Equal(
|
||||
t, "/hooks?notice=webhook-deleted", w.Header().Get("Location"),
|
||||
)
|
||||
assert.Equal(t, "/sources", w.Header().Get("Location"))
|
||||
|
||||
assert.Equal(
|
||||
t, int64(0),
|
||||
@@ -466,17 +435,68 @@ func TestHandleSourceDelete_RemovesConfigAndEventDatabase(
|
||||
)
|
||||
}
|
||||
|
||||
// TestHandleTargetDelete_EvictsThatTarget proves that deleting a
|
||||
// database target releases that target's archive writer and no
|
||||
// other: the webhook's other database target keeps its own.
|
||||
func TestHandleTargetDelete_EvictsThatTarget(t *testing.T) {
|
||||
// TestHandleTargetDelete_EvictsWhenLastDatabaseTargetGone
|
||||
// proves that removing the last database target releases the
|
||||
// archive writer.
|
||||
func TestHandleTargetDelete_EvictsWhenLastDatabaseTargetGone(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
ev *recordingArchives
|
||||
ev *recordingEvictor
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &ev)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
wh := seedWebhook(t, db)
|
||||
tgt := seedTarget(
|
||||
t, db, wh.ID, database.TargetTypeDatabase,
|
||||
)
|
||||
|
||||
cookies := authenticatedCookies(
|
||||
t, sess, deleteTestUserID, deleteTestUsername,
|
||||
)
|
||||
|
||||
req := postRequest(
|
||||
"/source/"+wh.ID+"/targets/"+tgt.ID+"/delete",
|
||||
cookies,
|
||||
map[string]string{
|
||||
paramSourceID: wh.ID,
|
||||
paramTargetID: tgt.ID,
|
||||
},
|
||||
)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
h.HandleTargetDelete().ServeHTTP(w, req)
|
||||
|
||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||
assert.Equal(
|
||||
t, []string{wh.ID}, ev.Evicted(),
|
||||
"removing the last database target should evict",
|
||||
)
|
||||
}
|
||||
|
||||
// TestHandleTargetDelete_KeepsWriterWhenDatabaseTargetRemains
|
||||
// proves that deleting one of several database targets leaves
|
||||
// the still-needed archive writer alone: the surviving target
|
||||
// keeps archiving to the same file, so the writer must stay.
|
||||
func TestHandleTargetDelete_KeepsWriterWhenDatabaseTargetRemains(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
ev *recordingEvictor
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &ev)
|
||||
@@ -495,7 +515,7 @@ func TestHandleTargetDelete_EvictsThatTarget(t *testing.T) {
|
||||
)
|
||||
|
||||
req := postRequest(
|
||||
"/hook/"+wh.ID+"/targets/"+doomed.ID+"/delete",
|
||||
"/source/"+wh.ID+"/targets/"+doomed.ID+"/delete",
|
||||
cookies,
|
||||
map[string]string{
|
||||
paramSourceID: wh.ID,
|
||||
@@ -507,17 +527,17 @@ func TestHandleTargetDelete_EvictsThatTarget(t *testing.T) {
|
||||
h.HandleTargetDelete().ServeHTTP(w, req)
|
||||
|
||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||
assert.Equal(
|
||||
t, []string{doomed.ID}, ev.EvictedTargets(),
|
||||
"deleting a database target should evict its writer",
|
||||
assert.Empty(
|
||||
t, ev.Evicted(),
|
||||
"a second database target still needs the writer",
|
||||
)
|
||||
assert.Empty(t, ev.Evicted(), "the webhook is not deleted")
|
||||
}
|
||||
|
||||
// TestHandleTargetDelete_IgnoresAnotherWebhooksTarget proves that
|
||||
// a target id from the URL that is not a target of the webhook
|
||||
// deletes nothing and so evicts nothing.
|
||||
func TestHandleTargetDelete_IgnoresAnotherWebhooksTarget(
|
||||
// TestHandleTargetDelete_KeepsWriterWhenOtherTypeDeleted proves
|
||||
// that deleting a target of an unrelated type leaves a
|
||||
// still-needed archive writer alone: the webhook's database
|
||||
// target is untouched, so its writer must stay.
|
||||
func TestHandleTargetDelete_KeepsWriterWhenOtherTypeDeleted(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
@@ -526,7 +546,7 @@ func TestHandleTargetDelete_IgnoresAnotherWebhooksTarget(
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
ev *recordingArchives
|
||||
ev *recordingEvictor
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &ev)
|
||||
@@ -535,20 +555,19 @@ func TestHandleTargetDelete_IgnoresAnotherWebhooksTarget(
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
wh := seedWebhook(t, db)
|
||||
elsewhere := seedTarget(
|
||||
t, db, seedWebhook(t, db).ID, database.TargetTypeDatabase,
|
||||
)
|
||||
seedTarget(t, db, wh.ID, database.TargetTypeDatabase)
|
||||
other := seedTarget(t, db, wh.ID, database.TargetTypeLog)
|
||||
|
||||
cookies := authenticatedCookies(
|
||||
t, sess, deleteTestUserID, deleteTestUsername,
|
||||
)
|
||||
|
||||
req := postRequest(
|
||||
"/hook/"+wh.ID+"/targets/"+elsewhere.ID+"/delete",
|
||||
"/source/"+wh.ID+"/targets/"+other.ID+"/delete",
|
||||
cookies,
|
||||
map[string]string{
|
||||
paramSourceID: wh.ID,
|
||||
paramTargetID: elsewhere.ID,
|
||||
paramTargetID: other.ID,
|
||||
},
|
||||
)
|
||||
w := httptest.NewRecorder()
|
||||
@@ -557,7 +576,7 @@ func TestHandleTargetDelete_IgnoresAnotherWebhooksTarget(
|
||||
|
||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||
assert.Empty(
|
||||
t, ev.EvictedTargets(),
|
||||
"another webhook's target must not be evicted",
|
||||
t, ev.Evicted(),
|
||||
"a surviving database target must keep its writer",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -81,7 +81,7 @@ func (f *baseURLFixture) entrypointURL(
|
||||
req := httptest.NewRequestWithContext(
|
||||
context.Background(),
|
||||
http.MethodGet,
|
||||
"/hook/"+f.webhook,
|
||||
"/source/"+f.webhook,
|
||||
nil,
|
||||
)
|
||||
req.Host = host
|
||||
@@ -213,7 +213,7 @@ func TestSourceDetailBaseURL_ForwardedProtoSpellings(t *testing.T) {
|
||||
|
||||
assert.Equal(
|
||||
t,
|
||||
tc.scheme+"://"+host+"/h/"+fixture.path,
|
||||
tc.scheme+"://"+host+"/webhook/"+fixture.path,
|
||||
fixture.entrypointURL(
|
||||
t, host, forwardedProto(tc.header),
|
||||
),
|
||||
@@ -244,7 +244,7 @@ func TestSourceDetailBaseURL_DirectTLSBeatsPlaintextHeader(
|
||||
|
||||
assert.Equal(
|
||||
t,
|
||||
"https://"+host+"/h/"+fixture.path,
|
||||
"https://"+host+"/webhook/"+fixture.path,
|
||||
got,
|
||||
"a connection this process terminated with TLS "+
|
||||
"outranks a header claiming plaintext",
|
||||
@@ -272,7 +272,7 @@ func TestSourceDetailBaseURL_KeepsHostAuthority(t *testing.T) {
|
||||
|
||||
assert.Equal(
|
||||
t,
|
||||
"https://"+host+"/h/"+fixture.path,
|
||||
"https://"+host+"/webhook/"+fixture.path,
|
||||
fixture.entrypointURL(
|
||||
t, host, forwardedProto("HTTPS"),
|
||||
),
|
||||
|
||||
@@ -82,7 +82,7 @@ func serveSourceDetailPage(
|
||||
req := httptest.NewRequestWithContext(
|
||||
context.Background(),
|
||||
http.MethodGet,
|
||||
"/hook/"+webhookID,
|
||||
"/source/"+webhookID,
|
||||
nil,
|
||||
)
|
||||
|
||||
@@ -241,37 +241,3 @@ func TestHandleSourceDetail_RendersNamedTargetFields(
|
||||
assert.Contains(t, body, "(unavailable)")
|
||||
assert.NotContains(t, body, "beak")
|
||||
}
|
||||
|
||||
// TestHandleSourceDetail_FitsWideAndNarrowWindows pins the webhook
|
||||
// page's maximum width at 108rem (1728 px), half again the 72rem of
|
||||
// max-w-6xl that the webhook list and the event log use, so an
|
||||
// entrypoint URL fits on one line in a 1920-pixel window; and the
|
||||
// wrapping of its title row, so the buttons beside the title do not
|
||||
// push a phone-width window into scrolling sideways.
|
||||
func TestHandleSourceDetail_FitsWideAndNarrowWindows(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
wh := seedWebhook(t, db)
|
||||
|
||||
body := renderSourceDetailPage(t, h, sess, wh.ID)
|
||||
|
||||
assert.Contains(
|
||||
t, body,
|
||||
`<div class="mx-auto px-6 py-8" style="max-width: 108rem"`,
|
||||
)
|
||||
assert.Contains(
|
||||
t, body,
|
||||
`<div class="flex flex-wrap justify-between items-center gap-2 mt-2">`,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1,467 +0,0 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/handlers"
|
||||
"sneak.berlin/go/webhooker/internal/logger"
|
||||
"sneak.berlin/go/webhooker/internal/session"
|
||||
)
|
||||
|
||||
// failedHighlight is how the list marks a number of failed deliveries
|
||||
// that is not zero.
|
||||
const failedHighlight = `class="font-medium text-red-600"`
|
||||
|
||||
// listWebhook adds a webhook with the given name, owned by the test
|
||||
// user.
|
||||
func listWebhook(
|
||||
t *testing.T, db *database.Database, name string,
|
||||
) *database.Webhook {
|
||||
t.Helper()
|
||||
|
||||
wh := &database.Webhook{UserID: deleteTestUserID, Name: name}
|
||||
require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error)
|
||||
|
||||
return wh
|
||||
}
|
||||
|
||||
// addEntrypoints adds the given number of entrypoints, all active or
|
||||
// all inactive, to a webhook and returns their paths.
|
||||
func addEntrypoints(
|
||||
t *testing.T, db *database.Database, webhookID string,
|
||||
count int, active bool,
|
||||
) []string {
|
||||
t.Helper()
|
||||
|
||||
paths := make([]string, count)
|
||||
for i := range paths {
|
||||
paths[i] = statsEntrypoint(t, db, webhookID, active)
|
||||
}
|
||||
|
||||
return paths
|
||||
}
|
||||
|
||||
// addTargets adds the given number of targets, all active or all
|
||||
// inactive, to a webhook and returns them.
|
||||
func addTargets(
|
||||
t *testing.T, db *database.Database, webhookID string,
|
||||
count int, active bool,
|
||||
) []*database.Target {
|
||||
t.Helper()
|
||||
|
||||
targets := make([]*database.Target, count)
|
||||
for i := range targets {
|
||||
targets[i] = seedTarget(t, db, webhookID, database.TargetTypeLog)
|
||||
require.NoError(t, db.DB().Model(targets[i]).
|
||||
Update("active", active).Error)
|
||||
}
|
||||
|
||||
return targets
|
||||
}
|
||||
|
||||
// renderWebhookList runs the real webhook list handler as the test user
|
||||
// and returns the rendered page.
|
||||
func renderWebhookList(
|
||||
t *testing.T, h *handlers.Handlers, sess *session.Session,
|
||||
) string {
|
||||
t.Helper()
|
||||
|
||||
cookies := authenticatedCookies(
|
||||
t, sess, deleteTestUserID, deleteTestUsername,
|
||||
)
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
h.HandleSourceList().ServeHTTP(
|
||||
w, getRequest(t, "/hooks", cookies, nil),
|
||||
)
|
||||
require.Equal(t, http.StatusOK, w.Code)
|
||||
|
||||
return w.Body.String()
|
||||
}
|
||||
|
||||
// listCard returns one webhook's entry in a rendered webhook list, its
|
||||
// markup as rendered and its text with the markup taken out and each
|
||||
// run of space made one space.
|
||||
func listCard(t *testing.T, page, webhookID string) (string, string) {
|
||||
t.Helper()
|
||||
|
||||
_, card, found := strings.Cut(page, `href="/hook/`+webhookID+`"`)
|
||||
require.True(t, found, "the list has no entry for %s", webhookID)
|
||||
|
||||
card, _, _ = strings.Cut(card, "</a>")
|
||||
text := regexp.MustCompile(`<[^>]*>`).ReplaceAllString(card, " ")
|
||||
|
||||
return card, strings.Join(strings.Fields(text), " ")
|
||||
}
|
||||
|
||||
// receiveEvents posts the given number of events to an entrypoint
|
||||
// through the real receiver, and returns the webhook's event database
|
||||
// and its events, oldest first.
|
||||
func receiveEvents(
|
||||
t *testing.T,
|
||||
h *handlers.Handlers,
|
||||
dbMgr *database.WebhookDBManager,
|
||||
webhookID, path string,
|
||||
count int,
|
||||
) (*gorm.DB, []database.Event) {
|
||||
t.Helper()
|
||||
|
||||
router := receiverRouter(h)
|
||||
|
||||
for range count {
|
||||
require.Equal(t, http.StatusOK, postReceiver(t, router, path))
|
||||
}
|
||||
|
||||
webhookDB, err := dbMgr.GetDB(webhookID)
|
||||
require.NoError(t, err)
|
||||
|
||||
events := listEvents(t, webhookDB)
|
||||
require.Len(t, events, count)
|
||||
|
||||
return webhookDB, events
|
||||
}
|
||||
|
||||
// seedFailingWebhook adds a webhook with six entrypoints, two of them
|
||||
// inactive, and seven targets, five of them inactive. Four events reach
|
||||
// its two active targets, arriving 31, 5, 4 and 3 hours ago, and its
|
||||
// event totals row records the last one. Three deliveries failed in the
|
||||
// last 24 hours, two to the first target and one to the second, one
|
||||
// failed 30 hours ago, two were delivered, and two are still pending.
|
||||
// It returns the webhook and when its last event arrived.
|
||||
func seedFailingWebhook(
|
||||
t *testing.T,
|
||||
h *handlers.Handlers,
|
||||
db *database.Database,
|
||||
dbMgr *database.WebhookDBManager,
|
||||
) (*database.Webhook, time.Time) {
|
||||
t.Helper()
|
||||
|
||||
wh := listWebhook(t, db, "failing")
|
||||
paths := addEntrypoints(t, db, wh.ID, 4, true)
|
||||
addEntrypoints(t, db, wh.ID, 2, false)
|
||||
|
||||
active := addTargets(t, db, wh.ID, 2, true)
|
||||
first, second := active[0], active[1]
|
||||
|
||||
addTargets(t, db, wh.ID, 5, false)
|
||||
|
||||
webhookDB, events := receiveEvents(t, h, dbMgr, wh.ID, paths[0], 4)
|
||||
now := time.Now()
|
||||
lastEventAt := now.Add(-3 * time.Hour)
|
||||
|
||||
statsAge(t, webhookDB, events[0].ID, now.Add(-31*time.Hour))
|
||||
statsAge(t, webhookDB, events[1].ID, now.Add(-5*time.Hour))
|
||||
statsAge(t, webhookDB, events[2].ID, now.Add(-4*time.Hour))
|
||||
statsAge(t, webhookDB, events[3].ID, lastEventAt)
|
||||
require.NoError(t, database.AddEventTotals(webhookDB,
|
||||
database.EventTotals{LastEventAt: &lastEventAt}))
|
||||
|
||||
statsFinish(t, webhookDB,
|
||||
statsDelivery(t, webhookDB, events[0].ID, first.ID),
|
||||
database.DeliveryStatusFailed, now.Add(-30*time.Hour))
|
||||
statsFinish(t, webhookDB,
|
||||
statsDelivery(t, webhookDB, events[0].ID, second.ID),
|
||||
database.DeliveryStatusDelivered, now.Add(-30*time.Hour))
|
||||
statsFinish(t, webhookDB,
|
||||
statsDelivery(t, webhookDB, events[1].ID, first.ID),
|
||||
database.DeliveryStatusFailed, now.Add(-time.Hour))
|
||||
statsFinish(t, webhookDB,
|
||||
statsDelivery(t, webhookDB, events[2].ID, first.ID),
|
||||
database.DeliveryStatusFailed, now.Add(-time.Minute))
|
||||
statsFinish(t, webhookDB,
|
||||
statsDelivery(t, webhookDB, events[2].ID, second.ID),
|
||||
database.DeliveryStatusFailed, now.Add(-time.Minute))
|
||||
statsFinish(t, webhookDB,
|
||||
statsDelivery(t, webhookDB, events[3].ID, second.ID),
|
||||
database.DeliveryStatusDelivered, now.Add(-time.Minute))
|
||||
|
||||
return wh, lastEventAt
|
||||
}
|
||||
|
||||
// seedHealthyWebhook adds a webhook with four entrypoints and two
|
||||
// targets, all active, and three events, arriving 8, 7 and 6 hours ago
|
||||
// and each delivered to both targets. Its event totals row records the
|
||||
// last event. It returns the webhook and when its last event arrived.
|
||||
func seedHealthyWebhook(
|
||||
t *testing.T,
|
||||
h *handlers.Handlers,
|
||||
db *database.Database,
|
||||
dbMgr *database.WebhookDBManager,
|
||||
) (*database.Webhook, time.Time) {
|
||||
t.Helper()
|
||||
|
||||
wh := listWebhook(t, db, "healthy")
|
||||
paths := addEntrypoints(t, db, wh.ID, 4, true)
|
||||
targets := addTargets(t, db, wh.ID, 2, true)
|
||||
|
||||
webhookDB, events := receiveEvents(t, h, dbMgr, wh.ID, paths[0], 3)
|
||||
now := time.Now()
|
||||
lastEventAt := now.Add(-6 * time.Hour)
|
||||
|
||||
statsAge(t, webhookDB, events[0].ID, now.Add(-8*time.Hour))
|
||||
statsAge(t, webhookDB, events[1].ID, now.Add(-7*time.Hour))
|
||||
statsAge(t, webhookDB, events[2].ID, lastEventAt)
|
||||
require.NoError(t, database.AddEventTotals(webhookDB,
|
||||
database.EventTotals{LastEventAt: &lastEventAt}))
|
||||
|
||||
for _, ev := range events {
|
||||
for _, target := range targets {
|
||||
statsFinish(t, webhookDB,
|
||||
statsDelivery(t, webhookDB, ev.ID, target.ID),
|
||||
database.DeliveryStatusDelivered, now)
|
||||
}
|
||||
}
|
||||
|
||||
return wh, lastEventAt
|
||||
}
|
||||
|
||||
// lastEventText is how the list shows when the last event arrived.
|
||||
func lastEventText(at time.Time) string {
|
||||
return at.UTC().Format("2006-01-02 15:04:05 UTC")
|
||||
}
|
||||
|
||||
// TestSourceList_ShowsActivityOfEachWebhook checks the figures the list
|
||||
// shows for a webhook with recent failures, a healthy one, a new one
|
||||
// that has received no event, and one without an event database.
|
||||
func TestSourceList_ShowsActivityOfEachWebhook(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
dbMgr *database.WebhookDBManager
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
failing, failingLastEvent := seedFailingWebhook(t, h, db, dbMgr)
|
||||
healthy, healthyLastEvent := seedHealthyWebhook(t, h, db, dbMgr)
|
||||
|
||||
// Creating a webhook creates its event database.
|
||||
fresh := listWebhook(t, db, "fresh")
|
||||
require.NoError(t, dbMgr.CreateDB(fresh.ID))
|
||||
addEntrypoints(t, db, fresh.ID, 2, true)
|
||||
addTargets(t, db, fresh.ID, 3, true)
|
||||
|
||||
quiet := listWebhook(t, db, "quiet")
|
||||
addEntrypoints(t, db, quiet.ID, 2, true)
|
||||
addTargets(t, db, quiet.ID, 3, true)
|
||||
|
||||
page := renderWebhookList(t, h, sess)
|
||||
|
||||
card, text := listCard(t, page, failing.ID)
|
||||
assert.Contains(t, text, "6 entrypoints, 2 inactive")
|
||||
assert.Contains(t, text, "7 targets, 5 inactive")
|
||||
assert.Contains(t, text, "4 events within retention")
|
||||
assert.Contains(t, text, "Last event "+lastEventText(failingLastEvent))
|
||||
assert.Contains(t, card,
|
||||
failedHighlight+">3 failed deliveries in the last 24 hours<")
|
||||
|
||||
card, text = listCard(t, page, healthy.ID)
|
||||
assert.Contains(t, text, "4 entrypoints")
|
||||
assert.Contains(t, text, "2 targets")
|
||||
assert.Contains(t, text, "3 events within retention")
|
||||
assert.Contains(t, text, "Last event "+lastEventText(healthyLastEvent))
|
||||
assert.Contains(t, text, "0 failed deliveries in the last 24 hours")
|
||||
assert.NotContains(t, text, "inactive")
|
||||
assert.NotContains(t, card, failedHighlight)
|
||||
|
||||
card, text = listCard(t, page, fresh.ID)
|
||||
assert.Contains(t, text, "2 entrypoints")
|
||||
assert.Contains(t, text, "3 targets")
|
||||
assert.Contains(t, text, "0 events within retention")
|
||||
assert.Contains(t, text, "No events yet")
|
||||
assert.Contains(t, text, "0 failed deliveries in the last 24 hours")
|
||||
assert.NotContains(t, card, failedHighlight)
|
||||
|
||||
card, text = listCard(t, page, quiet.ID)
|
||||
assert.Contains(t, text, "2 entrypoints")
|
||||
assert.Contains(t, text, "3 targets")
|
||||
assert.Contains(t, text, "0 events within retention")
|
||||
assert.Contains(t, text, "No events yet")
|
||||
assert.Contains(t, text, "0 failed deliveries in the last 24 hours")
|
||||
assert.NotContains(t, card, failedHighlight)
|
||||
assert.False(t, dbMgr.DBExists(quiet.ID),
|
||||
"showing the list must not create an event database")
|
||||
}
|
||||
|
||||
// TestSourceList_CountsOnlyEventsWithinRetention checks that once
|
||||
// retention has removed one of a webhook's three events, the list
|
||||
// counts the two still stored.
|
||||
func TestSourceList_CountsOnlyEventsWithinRetention(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
dbMgr *database.WebhookDBManager
|
||||
log *logger.Logger
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &dbMgr, &log)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
wh := &database.Webhook{
|
||||
UserID: deleteTestUserID, Name: "pruned", RetentionDays: 14,
|
||||
}
|
||||
require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error)
|
||||
|
||||
paths := addEntrypoints(t, db, wh.ID, 3, true)
|
||||
addTargets(t, db, wh.ID, 4, true)
|
||||
|
||||
webhookDB, events := receiveEvents(t, h, dbMgr, wh.ID, paths[0], 3)
|
||||
|
||||
statsAge(t, webhookDB, events[0].ID, time.Now().Add(-15*24*time.Hour))
|
||||
statsPrune(t, db, dbMgr, log, webhookDB)
|
||||
require.Len(t, listEvents(t, webhookDB), 2)
|
||||
|
||||
_, text := listCard(t, renderWebhookList(t, h, sess), wh.ID)
|
||||
assert.Contains(t, text, "3 entrypoints")
|
||||
assert.Contains(t, text, "4 targets")
|
||||
assert.Contains(t, text, "2 events within retention")
|
||||
}
|
||||
|
||||
// TestSourceList_LastEventSurvivesPruningEveryEvent checks that once
|
||||
// retention has removed every event of a webhook, the list still shows
|
||||
// when the last one arrived rather than "No events yet".
|
||||
func TestSourceList_LastEventSurvivesPruningEveryEvent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
dbMgr *database.WebhookDBManager
|
||||
log *logger.Logger
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &dbMgr, &log)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
wh := &database.Webhook{
|
||||
UserID: deleteTestUserID, Name: "emptied", RetentionDays: 1,
|
||||
}
|
||||
require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error)
|
||||
|
||||
paths := addEntrypoints(t, db, wh.ID, 2, true)
|
||||
addTargets(t, db, wh.ID, 3, true)
|
||||
|
||||
webhookDB, events := receiveEvents(t, h, dbMgr, wh.ID, paths[0], 1)
|
||||
lastEventAt := time.Now().Add(-50 * time.Hour)
|
||||
|
||||
statsAge(t, webhookDB, events[0].ID, lastEventAt)
|
||||
require.NoError(t, database.AddEventTotals(webhookDB,
|
||||
database.EventTotals{LastEventAt: &lastEventAt}))
|
||||
statsPrune(t, db, dbMgr, log, webhookDB)
|
||||
require.Empty(t, listEvents(t, webhookDB))
|
||||
|
||||
_, text := listCard(t, renderWebhookList(t, h, sess), wh.ID)
|
||||
assert.Contains(t, text, "0 events within retention")
|
||||
assert.Contains(t, text, "Last event "+lastEventText(lastEventAt))
|
||||
assert.NotContains(t, text, "No events yet")
|
||||
}
|
||||
|
||||
// TestSourceList_CountsOfOneInSingular checks that a webhook with one
|
||||
// entrypoint, one target, one event within retention and one failed
|
||||
// delivery in the last 24 hours has each written in the singular.
|
||||
func TestSourceList_CountsOfOneInSingular(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
dbMgr *database.WebhookDBManager
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
wh := listWebhook(t, db, "single")
|
||||
paths := addEntrypoints(t, db, wh.ID, 1, true)
|
||||
targets := addTargets(t, db, wh.ID, 1, true)
|
||||
|
||||
webhookDB, events := receiveEvents(t, h, dbMgr, wh.ID, paths[0], 1)
|
||||
now := time.Now()
|
||||
lastEventAt := now.Add(-9 * time.Hour)
|
||||
|
||||
statsAge(t, webhookDB, events[0].ID, lastEventAt)
|
||||
require.NoError(t, database.AddEventTotals(webhookDB,
|
||||
database.EventTotals{LastEventAt: &lastEventAt}))
|
||||
statsFinish(t, webhookDB,
|
||||
statsDelivery(t, webhookDB, events[0].ID, targets[0].ID),
|
||||
database.DeliveryStatusFailed, now.Add(-time.Hour))
|
||||
|
||||
card, text := listCard(t, renderWebhookList(t, h, sess), wh.ID)
|
||||
assert.Contains(t, card, ">1 entrypoint<")
|
||||
assert.Contains(t, card, ">1 target<")
|
||||
assert.Contains(t, card, ">1 event within retention<")
|
||||
assert.Contains(t, text, "Last event "+lastEventText(lastEventAt))
|
||||
assert.Contains(t, card,
|
||||
failedHighlight+">1 failed delivery in the last 24 hours<")
|
||||
}
|
||||
|
||||
// TestSourceList_UnreadableEventDatabase checks that a webhook whose
|
||||
// event database cannot be read says so in its entry instead of
|
||||
// showing zeros, and that the rest of the list is still shown.
|
||||
func TestSourceList_UnreadableEventDatabase(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
dbMgr *database.WebhookDBManager
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
broken := listWebhook(t, db, "broken")
|
||||
addEntrypoints(t, db, broken.ID, 2, true)
|
||||
addTargets(t, db, broken.ID, 3, true)
|
||||
|
||||
brokenDB, err := dbMgr.GetDB(broken.ID)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t,
|
||||
brokenDB.Migrator().DropTable(&database.EventTotals{}))
|
||||
|
||||
quiet := listWebhook(t, db, "quiet")
|
||||
addEntrypoints(t, db, quiet.ID, 2, true)
|
||||
addTargets(t, db, quiet.ID, 3, true)
|
||||
|
||||
page := renderWebhookList(t, h, sess)
|
||||
|
||||
_, text := listCard(t, page, broken.ID)
|
||||
assert.Contains(t, text, "2 entrypoints")
|
||||
assert.Contains(t, text, "3 targets")
|
||||
assert.Contains(t, text, "The event figures could not be read.")
|
||||
assert.NotContains(t, text, "events")
|
||||
assert.NotContains(t, text, "failed")
|
||||
|
||||
_, text = listCard(t, page, quiet.ID)
|
||||
assert.Contains(t, text, "No events yet")
|
||||
}
|
||||
@@ -28,7 +28,7 @@ func deleteTargetThroughHandler(
|
||||
t.Helper()
|
||||
|
||||
req := postRequest(
|
||||
"/hook/"+webhookID+"/targets/"+targetID+"/delete",
|
||||
"/source/"+webhookID+"/targets/"+targetID+"/delete",
|
||||
authenticatedCookies(
|
||||
t, sess, deleteTestUserID, deleteTestUsername,
|
||||
),
|
||||
|
||||
@@ -84,7 +84,7 @@ func renderSourceLogsPageWithQuery(
|
||||
req := httptest.NewRequestWithContext(
|
||||
context.Background(),
|
||||
http.MethodGet,
|
||||
"/hook/"+webhookID+"/events"+query,
|
||||
"/source/"+webhookID+"/logs"+query,
|
||||
nil,
|
||||
)
|
||||
|
||||
|
||||
@@ -3,12 +3,10 @@ package handlers
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi"
|
||||
"github.com/google/uuid"
|
||||
@@ -22,20 +20,9 @@ import (
|
||||
type WebhookListItem struct {
|
||||
database.Webhook
|
||||
|
||||
EntrypointCount int
|
||||
InactiveEntrypointCount int
|
||||
TargetCount int
|
||||
InactiveTargetCount int
|
||||
|
||||
// EventCount is how many events the webhook holds, LastEventAt
|
||||
// when the newest arrived (nil before the first), and
|
||||
// FailedLast24Hours how many of its deliveries failed in the last
|
||||
// 24 hours. When the webhook's event database could not be read,
|
||||
// EventsUnreadable is set and these three are not known.
|
||||
EventCount int64
|
||||
LastEventAt *time.Time
|
||||
FailedLast24Hours int64
|
||||
EventsUnreadable bool
|
||||
EntrypointCount int64
|
||||
TargetCount int64
|
||||
EventCount int64
|
||||
}
|
||||
|
||||
// errMissingURL signals that a required URL was not provided.
|
||||
@@ -162,17 +149,18 @@ func (h *Handlers) HandleSourceList() http.HandlerFunc {
|
||||
"user_id = ?", userID,
|
||||
).Order("created_at DESC").Find(&webhooks).Error
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to list webhooks", err)
|
||||
h.log.Error(
|
||||
"failed to list webhooks", "error", err,
|
||||
)
|
||||
http.Error(
|
||||
w, "Internal server error",
|
||||
http.StatusInternalServerError,
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
items, err := h.buildWebhookListItems(webhooks)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to list webhooks", err)
|
||||
|
||||
return
|
||||
}
|
||||
items := h.buildWebhookListItems(webhooks)
|
||||
|
||||
data := map[string]any{
|
||||
"Webhooks": items,
|
||||
@@ -182,115 +170,36 @@ func (h *Handlers) HandleSourceList() http.HandlerFunc {
|
||||
}
|
||||
}
|
||||
|
||||
// buildWebhookListItems builds the list's entry for each webhook. It
|
||||
// fails when the main database cannot be read. A webhook whose event
|
||||
// database cannot be read is marked on its own entry, and the error is
|
||||
// logged.
|
||||
// buildWebhookListItems builds list items with counts.
|
||||
func (h *Handlers) buildWebhookListItems(
|
||||
webhooks []database.Webhook,
|
||||
) ([]WebhookListItem, error) {
|
||||
) []WebhookListItem {
|
||||
items := make([]WebhookListItem, len(webhooks))
|
||||
since := time.Now().Add(-longWindow)
|
||||
|
||||
for i := range webhooks {
|
||||
item := &items[i]
|
||||
item.Webhook = webhooks[i]
|
||||
items[i].Webhook = webhooks[i]
|
||||
|
||||
var err error
|
||||
h.db.DB().Model(&database.Entrypoint{}).Where(
|
||||
"webhook_id = ?", webhooks[i].ID,
|
||||
).Count(&items[i].EntrypointCount)
|
||||
|
||||
item.EntrypointCount, item.InactiveEntrypointCount, err =
|
||||
h.countWithInactive(&database.Entrypoint{}, item.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
h.db.DB().Model(&database.Target{}).Where(
|
||||
"webhook_id = ?", webhooks[i].ID,
|
||||
).Count(&items[i].TargetCount)
|
||||
|
||||
item.TargetCount, item.InactiveTargetCount, err =
|
||||
h.countWithInactive(&database.Target{}, item.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Opening an event database that does not exist would create
|
||||
// it, and it would hold nothing to count.
|
||||
if !h.dbMgr.DBExists(item.ID) {
|
||||
continue
|
||||
}
|
||||
|
||||
err = h.readListEventFigures(item, since)
|
||||
if err != nil {
|
||||
h.log.Error(
|
||||
"failed to read webhook list figures",
|
||||
"webhook_id", item.ID,
|
||||
"error", err,
|
||||
if h.dbMgr.DBExists(webhooks[i].ID) {
|
||||
webhookDB, err := h.dbMgr.GetDB(
|
||||
webhooks[i].ID,
|
||||
)
|
||||
|
||||
item.EventsUnreadable = true
|
||||
if err == nil {
|
||||
webhookDB.Model(
|
||||
&database.Event{},
|
||||
).Count(&items[i].EventCount)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return items, nil
|
||||
}
|
||||
|
||||
// countWithInactive returns how many entrypoints or targets, as model
|
||||
// says, a webhook has, and how many of them are inactive.
|
||||
func (h *Handlers) countWithInactive(
|
||||
model any, webhookID string,
|
||||
) (int, int, error) {
|
||||
var active []bool
|
||||
|
||||
err := h.db.DB().Model(model).
|
||||
Where("webhook_id = ?", webhookID).
|
||||
Pluck("active", &active).Error
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf(
|
||||
"reading active flags of webhook %s: %w", webhookID, err,
|
||||
)
|
||||
}
|
||||
|
||||
inactive := 0
|
||||
|
||||
for _, a := range active {
|
||||
if !a {
|
||||
inactive++
|
||||
}
|
||||
}
|
||||
|
||||
return len(active), inactive, nil
|
||||
}
|
||||
|
||||
// readListEventFigures fills in the figures the list shows from the
|
||||
// webhook's event database, with the statistics pane's own queries:
|
||||
// the event count and last arrival from the event totals row, and the
|
||||
// deliveries that failed since the given time from the deliveries'
|
||||
// status index.
|
||||
func (h *Handlers) readListEventFigures(
|
||||
item *WebhookListItem, since time.Time,
|
||||
) error {
|
||||
webhookDB, err := h.dbMgr.GetDB(item.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var totals database.EventTotals
|
||||
|
||||
err = webhookDB.Take(&totals).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading event totals: %w", err)
|
||||
}
|
||||
|
||||
item.EventCount = totals.Events - totals.EventsRemoved
|
||||
item.LastEventAt = totals.LastEventAt
|
||||
|
||||
byTarget, err := finishedByTarget(webhookDB, since)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, f := range byTarget {
|
||||
item.FailedLast24Hours += f.Failed
|
||||
}
|
||||
|
||||
return nil
|
||||
return items
|
||||
}
|
||||
|
||||
// HandleSourceCreate shows the form to create a new webhook.
|
||||
@@ -340,7 +249,9 @@ func (h *Handlers) HandleSourceCreateSubmit() http.HandlerFunc {
|
||||
// middleware, which runs before CSRF parses the form.
|
||||
err := r.ParseForm()
|
||||
if err != nil {
|
||||
h.renderError(w, r, http.StatusBadRequest)
|
||||
http.Error(
|
||||
w, "Bad request", http.StatusBadRequest,
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -350,12 +261,12 @@ func (h *Handlers) HandleSourceCreateSubmit() http.HandlerFunc {
|
||||
retentionStr := r.PostFormValue("retention_days")
|
||||
|
||||
if name == "" {
|
||||
h.renderTemplateStatus(
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
h.renderTemplate(
|
||||
w, r, "sources_new.html",
|
||||
newSourceFormData(
|
||||
"Name is required", name, description,
|
||||
),
|
||||
http.StatusBadRequest,
|
||||
)
|
||||
|
||||
return
|
||||
@@ -365,13 +276,13 @@ func (h *Handlers) HandleSourceCreateSubmit() http.HandlerFunc {
|
||||
retentionStr, database.DefaultRetentionDays,
|
||||
)
|
||||
if retErr != nil {
|
||||
h.renderTemplateStatus(
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
h.renderTemplate(
|
||||
w, r, "sources_new.html",
|
||||
newSourceFormData(
|
||||
retentionErrorMessage(retErr),
|
||||
name, description,
|
||||
),
|
||||
http.StatusBadRequest,
|
||||
)
|
||||
|
||||
return
|
||||
@@ -400,7 +311,7 @@ func (h *Handlers) createWebhookWithEntrypoint(
|
||||
|
||||
err := h.commitWebhook(webhook)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to create webhook", err)
|
||||
h.serverError(w, "failed to create webhook", err)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -419,8 +330,7 @@ func (h *Handlers) createWebhookWithEntrypoint(
|
||||
)
|
||||
|
||||
http.Redirect(
|
||||
w, r, withNotice("/hook/"+webhook.ID, webhookCreated),
|
||||
http.StatusSeeOther,
|
||||
w, r, "/source/"+webhook.ID, http.StatusSeeOther,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -478,7 +388,7 @@ func (h *Handlers) HandleSourceDetail() http.HandlerFunc {
|
||||
"id = ? AND user_id = ?", sourceID, userID,
|
||||
).First(&webhook).Error
|
||||
if err != nil {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
http.NotFound(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -510,7 +420,7 @@ func (h *Handlers) renderSourceDetail(
|
||||
if h.dbMgr.DBExists(webhook.ID) {
|
||||
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to get webhook database", err)
|
||||
h.serverError(w, "failed to get webhook database", err)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -519,7 +429,7 @@ func (h *Handlers) renderSourceDetail(
|
||||
webhookDB, webhook.ID, singleHTTPTargetID(targets),
|
||||
)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to load recent events", err)
|
||||
h.serverError(w, "failed to load recent events", err)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -547,7 +457,6 @@ func (h *Handlers) renderSourceDetail(
|
||||
"Targets": delivery.NewTargetViews(targets),
|
||||
"Events": events,
|
||||
"BaseURL": baseURL,
|
||||
"Stats": h.loadWebhookStats(webhook.ID, entrypoints, targets),
|
||||
}
|
||||
|
||||
h.renderTemplate(w, r, "source_detail.html", data)
|
||||
@@ -573,7 +482,7 @@ func (h *Handlers) HandleSourceEdit() http.HandlerFunc {
|
||||
"id = ? AND user_id = ?", sourceID, userID,
|
||||
).First(&webhook).Error
|
||||
if err != nil {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
http.NotFound(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -602,16 +511,13 @@ func (h *Handlers) HandleSourceEditSubmit() http.HandlerFunc {
|
||||
|
||||
sourceID := chi.URLParam(r, "sourceID")
|
||||
|
||||
h.renameMu.Lock()
|
||||
defer h.renameMu.Unlock()
|
||||
|
||||
var webhook database.Webhook
|
||||
|
||||
err := h.db.DB().Where(
|
||||
"id = ? AND user_id = ?", sourceID, userID,
|
||||
).First(&webhook).Error
|
||||
if err != nil {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
http.NotFound(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -620,7 +526,9 @@ func (h *Handlers) HandleSourceEditSubmit() http.HandlerFunc {
|
||||
// middleware, which runs before CSRF parses the form.
|
||||
err = r.ParseForm()
|
||||
if err != nil {
|
||||
h.renderError(w, r, http.StatusBadRequest)
|
||||
http.Error(
|
||||
w, "Bad request", http.StatusBadRequest,
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -644,12 +552,12 @@ func (h *Handlers) applyWebhookEdit(
|
||||
tmplKeyError: "Name is required",
|
||||
}
|
||||
|
||||
h.renderTemplateStatus(w, r, "source_edit.html", data, http.StatusBadRequest)
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
h.renderTemplate(w, r, "source_edit.html", data)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
oldName := webhook.Name
|
||||
webhook.Name = name
|
||||
webhook.Description = r.PostFormValue("description")
|
||||
|
||||
@@ -664,56 +572,23 @@ func (h *Handlers) applyWebhookEdit(
|
||||
tmplKeyError: retentionErrorMessage(retErr),
|
||||
}
|
||||
|
||||
h.renderTemplateStatus(w, r, "source_edit.html", data, http.StatusBadRequest)
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
h.renderTemplate(w, r, "source_edit.html", data)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
webhook.RetentionDays = retentionDays
|
||||
|
||||
// A new name renames the archive files before it is saved (see
|
||||
// delivery.Engine.Rename). If either step fails, the same targets'
|
||||
// archives go back to the name that is still stored, without
|
||||
// reading the main database again.
|
||||
targets, err := h.renameWebhookArchives(
|
||||
webhook.ID, oldName, webhook.Name,
|
||||
)
|
||||
if err == nil {
|
||||
err = h.db.DB().Save(webhook).Error
|
||||
}
|
||||
|
||||
err := h.db.DB().Save(webhook).Error
|
||||
if err != nil {
|
||||
restoreErr := h.renameArchives(targets, oldName)
|
||||
if restoreErr != nil {
|
||||
h.log.Error(
|
||||
"failed to rename archives back",
|
||||
"webhook_id", webhook.ID,
|
||||
"error", restoreErr,
|
||||
)
|
||||
}
|
||||
|
||||
if errors.Is(err, delivery.ErrArchiveNameTaken) {
|
||||
data := map[string]any{
|
||||
tmplKeyWebhook: webhook,
|
||||
tmplKeyError: "Not saved: " + err.Error() +
|
||||
". Move that archive out of the data directory, " +
|
||||
"its .db together with any -wal and -shm beside " +
|
||||
"it, then save again.",
|
||||
}
|
||||
|
||||
h.renderTemplateStatus(w, r, "source_edit.html", data, http.StatusConflict)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
h.serverError(w, r, "failed to update webhook", err)
|
||||
h.serverError(w, "failed to update webhook", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
http.Redirect(
|
||||
w, r, withNotice("/hook/"+webhook.ID, webhookSaved),
|
||||
http.StatusSeeOther,
|
||||
w, r, "/source/"+webhook.ID, http.StatusSeeOther,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -737,7 +612,7 @@ func (h *Handlers) HandleSourceDelete() http.HandlerFunc {
|
||||
"id = ? AND user_id = ?", sourceID, userID,
|
||||
).First(&webhook).Error
|
||||
if err != nil {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
http.NotFound(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -764,7 +639,7 @@ func (h *Handlers) deleteWebhookResources(
|
||||
// be removed by hand; deleted history cannot be recovered.
|
||||
err := h.commitWebhookDeletion(&webhook)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to delete webhook", err)
|
||||
h.serverError(w, "failed to delete webhook", err)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -790,15 +665,13 @@ func (h *Handlers) deleteWebhookResources(
|
||||
// redirecting as though everything succeeded: the file
|
||||
// needs removing by hand, and the logged error names it.
|
||||
h.serverError(
|
||||
w, r, "failed to delete webhook event database", err,
|
||||
w, "failed to delete webhook event database", err,
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
http.Redirect(
|
||||
w, r, withNotice("/hooks", webhookDeleted), http.StatusSeeOther,
|
||||
)
|
||||
http.Redirect(w, r, "/sources", http.StatusSeeOther)
|
||||
}
|
||||
|
||||
// commitWebhookDeletion soft-deletes a webhook's entrypoints,
|
||||
@@ -843,11 +716,11 @@ func (h *Handlers) commitWebhookDeletion(
|
||||
return tx.Commit().Error
|
||||
}
|
||||
|
||||
// evictArchiveWriter asks the delivery engine to drop the cached
|
||||
// archive writers of a webhook's database targets, closing their
|
||||
// archive file handles.
|
||||
// evictArchiveWriter asks the delivery engine to drop its
|
||||
// cached archive writer for a webhook, closing the archive file
|
||||
// handle.
|
||||
//
|
||||
// The archive database files are NOT deleted. Unlike the event
|
||||
// The archive database file is NOT deleted. Unlike the event
|
||||
// database — which is per-webhook working storage and is
|
||||
// hard-deleted with the webhook — an archive is explicitly
|
||||
// long-term storage that an operator may want to keep or move
|
||||
@@ -855,72 +728,50 @@ func (h *Handlers) commitWebhookDeletion(
|
||||
// deleting a webhook would be a surprising and unrecoverable
|
||||
// data loss, so the file is left for the operator to handle.
|
||||
func (h *Handlers) evictArchiveWriter(webhookID string) {
|
||||
if h.archives == nil {
|
||||
if h.evictor == nil {
|
||||
return
|
||||
}
|
||||
|
||||
h.archives.EvictWebhook(webhookID)
|
||||
h.evictor.EvictWebhook(webhookID)
|
||||
}
|
||||
|
||||
// evictTargetArchiveWriter is evictArchiveWriter for one deleted
|
||||
// target, and leaves its archive file on disk for the same reason.
|
||||
// A target that is not a database target has no writer, and
|
||||
// evicting it does nothing.
|
||||
func (h *Handlers) evictTargetArchiveWriter(targetID string) {
|
||||
if h.archives == nil {
|
||||
return
|
||||
}
|
||||
|
||||
h.archives.EvictTarget(targetID)
|
||||
}
|
||||
|
||||
// renameWebhookArchives renames the archive file of every database
|
||||
// target of a webhook from the webhook name oldName to newName,
|
||||
// keeping each target's own name. It does nothing when the name is
|
||||
// unchanged. It returns the targets it read, so that a failed edit can
|
||||
// move those same archives back with renameArchives.
|
||||
func (h *Handlers) renameWebhookArchives(
|
||||
webhookID, oldName, newName string,
|
||||
) ([]database.Target, error) {
|
||||
if h.archives == nil || oldName == newName {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var targets []database.Target
|
||||
// evictArchiveWriterIfUnused releases a webhook's archive
|
||||
// writer once the webhook has no database target left to feed
|
||||
// it.
|
||||
//
|
||||
// It is called after any child resource of a webhook is
|
||||
// deleted, and is correct without knowing which kind was: it
|
||||
// evicts only when no database target remains, so deleting one
|
||||
// of several database targets — or deleting an unrelated
|
||||
// target type — leaves a still-needed writer alone. When no
|
||||
// database target ever existed there is no writer and eviction
|
||||
// is a no-op. Soft-deleted targets are excluded by GORM's
|
||||
// default scope, so the row just deleted is not counted.
|
||||
func (h *Handlers) evictArchiveWriterIfUnused(webhookID string) {
|
||||
var remaining int64
|
||||
|
||||
err := h.db.DB().
|
||||
Model(&database.Target{}).
|
||||
Where(
|
||||
"webhook_id = ? AND type = ?",
|
||||
webhookID, database.TargetTypeDatabase,
|
||||
).
|
||||
Find(&targets).Error
|
||||
Count(&remaining).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return targets, h.renameArchives(targets, newName)
|
||||
}
|
||||
|
||||
// renameArchives renames the archive file of each of the given
|
||||
// database targets to the webhook name webhookName, keeping each
|
||||
// target's own name. It tries every target even after one fails, so
|
||||
// that moving the archives back after a failed edit leaves none under
|
||||
// the new name, and returns every failure joined.
|
||||
func (h *Handlers) renameArchives(
|
||||
targets []database.Target, webhookName string,
|
||||
) error {
|
||||
var errs []error
|
||||
|
||||
for i := range targets {
|
||||
err := h.archives.Rename(
|
||||
targets[i].ID, webhookName, targets[i].Name,
|
||||
h.log.Error(
|
||||
"failed to count remaining database targets",
|
||||
"webhook_id", webhookID,
|
||||
"error", err,
|
||||
)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
return errors.Join(errs...)
|
||||
if remaining > 0 {
|
||||
return
|
||||
}
|
||||
|
||||
h.evictArchiveWriter(webhookID)
|
||||
}
|
||||
|
||||
// ownedWebhook resolves the request's sourceID parameter to a
|
||||
@@ -958,7 +809,7 @@ func (h *Handlers) ownedWebhook(
|
||||
"id = ? AND user_id = ?", sourceID, userID,
|
||||
).First(&webhook).Error
|
||||
if err != nil {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
http.NotFound(w, r)
|
||||
|
||||
return database.Webhook{}, false
|
||||
}
|
||||
@@ -980,7 +831,7 @@ func (h *Handlers) HandleSourceLogs() http.HandlerFunc {
|
||||
// Without the map every delivery renders through a
|
||||
// zero redactor, so failing the page is the only
|
||||
// safe answer.
|
||||
h.serverError(w, r, "failed to load targets", err)
|
||||
h.serverError(w, "failed to load targets", err)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -988,7 +839,7 @@ func (h *Handlers) HandleSourceLogs() http.HandlerFunc {
|
||||
page := h.parsePage(r)
|
||||
|
||||
evts, total, ok := h.loadEventsWithDeliveries(
|
||||
w, r, webhook, targets, page,
|
||||
w, webhook, targets, page,
|
||||
)
|
||||
if !ok {
|
||||
return
|
||||
@@ -999,16 +850,31 @@ func (h *Handlers) HandleSourceLogs() http.HandlerFunc {
|
||||
totalPages++
|
||||
}
|
||||
|
||||
// The banner a replay or resubmit POST redirected back
|
||||
// with. The message comes from a fixed set keyed by the
|
||||
// outcome code, never from the query string itself.
|
||||
replayMsg, replayOK := replayOutcome(
|
||||
r.URL.Query().Get(replayOutcomeParam),
|
||||
)
|
||||
|
||||
resubmitMsg, resubmitOK := resubmitOutcome(
|
||||
r.URL.Query().Get(resubmitOutcomeParam),
|
||||
)
|
||||
|
||||
data := map[string]any{
|
||||
tmplKeyWebhook: &webhook,
|
||||
"Events": evts,
|
||||
"Page": page,
|
||||
"TotalPages": totalPages,
|
||||
"TotalEvents": total,
|
||||
"HasPrev": page > 1,
|
||||
"HasNext": page < totalPages,
|
||||
"PrevPage": page - 1,
|
||||
"NextPage": page + 1,
|
||||
tmplKeyWebhook: &webhook,
|
||||
"Events": evts,
|
||||
"ReplayMessage": replayMsg,
|
||||
"ReplayQueued": replayOK,
|
||||
"ResubmitMessage": resubmitMsg,
|
||||
"ResubmitQueued": resubmitOK,
|
||||
"Page": page,
|
||||
"TotalPages": totalPages,
|
||||
"TotalEvents": total,
|
||||
"HasPrev": page > 1,
|
||||
"HasNext": page < totalPages,
|
||||
"PrevPage": page - 1,
|
||||
"NextPage": page + 1,
|
||||
}
|
||||
|
||||
h.renderTemplate(w, r, "source_logs.html", data)
|
||||
@@ -1083,7 +949,6 @@ func (h *Handlers) parsePage(r *http.Request) int {
|
||||
// caller must then render nothing further.
|
||||
func (h *Handlers) loadEventsWithDeliveries(
|
||||
w http.ResponseWriter,
|
||||
r *http.Request,
|
||||
webhook database.Webhook,
|
||||
targetMap map[string]eventLogTarget,
|
||||
page int,
|
||||
@@ -1097,7 +962,7 @@ func (h *Handlers) loadEventsWithDeliveries(
|
||||
webhookDB, err := h.dbMgr.GetDB(webhook.ID)
|
||||
if err != nil {
|
||||
h.serverError(
|
||||
w, r, "failed to get webhook database", err,
|
||||
w, "failed to get webhook database", err,
|
||||
)
|
||||
|
||||
return nil, 0, false
|
||||
@@ -1134,7 +999,7 @@ func (h *Handlers) loadEventsWithDeliveries(
|
||||
)
|
||||
if err != nil {
|
||||
h.serverError(
|
||||
w, r, "failed to load delivery attempts", err,
|
||||
w, "failed to load delivery attempts", err,
|
||||
)
|
||||
|
||||
return nil, 0, false
|
||||
@@ -1143,7 +1008,7 @@ func (h *Handlers) loadEventsWithDeliveries(
|
||||
resubmits, err := resubmitCounts(webhookDB, eventIDs)
|
||||
if err != nil {
|
||||
h.serverError(
|
||||
w, r, "failed to count event resubmissions", err,
|
||||
w, "failed to count event resubmissions", err,
|
||||
)
|
||||
|
||||
return nil, 0, false
|
||||
@@ -1366,7 +1231,7 @@ func (h *Handlers) HandleEntrypointCreate() http.HandlerFunc {
|
||||
"id = ? AND user_id = ?", sourceID, userID,
|
||||
).First(&webhook).Error
|
||||
if err != nil {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
http.NotFound(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -1375,7 +1240,9 @@ func (h *Handlers) HandleEntrypointCreate() http.HandlerFunc {
|
||||
// middleware, which runs before CSRF parses the form.
|
||||
err = r.ParseForm()
|
||||
if err != nil {
|
||||
h.renderError(w, r, http.StatusBadRequest)
|
||||
http.Error(
|
||||
w, "Bad request", http.StatusBadRequest,
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -1391,14 +1258,13 @@ func (h *Handlers) HandleEntrypointCreate() http.HandlerFunc {
|
||||
|
||||
err = h.db.DB().Create(entrypoint).Error
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to create entrypoint", err)
|
||||
h.serverError(w, "failed to create entrypoint", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
http.Redirect(
|
||||
w, r, withNotice("/hook/"+webhook.ID, entrypointAdded),
|
||||
http.StatusSeeOther,
|
||||
w, r, "/source/"+webhook.ID, http.StatusSeeOther,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1417,16 +1283,13 @@ func (h *Handlers) HandleTargetCreate() http.HandlerFunc {
|
||||
|
||||
sourceID := chi.URLParam(r, "sourceID")
|
||||
|
||||
h.renameMu.Lock()
|
||||
defer h.renameMu.Unlock()
|
||||
|
||||
var webhook database.Webhook
|
||||
|
||||
err := h.db.DB().Where(
|
||||
"id = ? AND user_id = ?", sourceID, userID,
|
||||
).First(&webhook).Error
|
||||
if err != nil {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
http.NotFound(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -1435,7 +1298,9 @@ func (h *Handlers) HandleTargetCreate() http.HandlerFunc {
|
||||
// middleware, which runs before CSRF parses the form.
|
||||
err = r.ParseForm()
|
||||
if err != nil {
|
||||
h.renderError(w, r, http.StatusBadRequest)
|
||||
http.Error(
|
||||
w, "Bad request", http.StatusBadRequest,
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -1455,7 +1320,7 @@ func (h *Handlers) processTargetCreate(
|
||||
//
|
||||
// Every field here is read with PostFormValue, not FormValue.
|
||||
// FormValue falls back to the query string, which would let
|
||||
// `POST /hook/{id}/targets?url=https://hooks.slack.com/...`
|
||||
// `POST /source/{id}/targets?url=https://hooks.slack.com/...`
|
||||
// configure a target from a value the request line carries — and
|
||||
// the request line, unlike the body, is what logs, proxies,
|
||||
// Referer headers and error trackers record.
|
||||
@@ -1506,14 +1371,13 @@ func (h *Handlers) processTargetCreate(
|
||||
|
||||
err = h.db.DB().Create(target).Error
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to create target", err)
|
||||
h.serverError(w, "failed to create target", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
http.Redirect(
|
||||
w, r, withNotice("/hook/"+webhook.ID, targetAdded),
|
||||
http.StatusSeeOther,
|
||||
w, r, "/source/"+webhook.ID, http.StatusSeeOther,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1571,7 +1435,7 @@ type targetFormInput struct {
|
||||
//
|
||||
// Every field is read with PostFormValue, not FormValue. FormValue
|
||||
// falls back to the query string, which would let
|
||||
// `POST /hook/{id}/targets?url=https://hooks.slack.com/...`
|
||||
// `POST /source/{id}/targets?url=https://hooks.slack.com/...`
|
||||
// configure a target from a value the request line carries — and the
|
||||
// request line, unlike the body, is what logs, proxies, Referer
|
||||
// headers and error trackers record. The headers field is under the
|
||||
@@ -1601,7 +1465,7 @@ func (h *Handlers) buildTargetConfig(
|
||||
case database.TargetTypeSlack:
|
||||
return h.buildSlackTargetConfig(w, r, in.URL)
|
||||
case database.TargetTypeDatabase:
|
||||
return h.buildDatabaseTargetConfig(w, r, in.Expiry)
|
||||
return h.buildDatabaseTargetConfig(w, in.Expiry)
|
||||
case database.TargetTypeLog:
|
||||
return "", nil
|
||||
default:
|
||||
@@ -1651,7 +1515,7 @@ func (h *Handlers) buildHTTPTargetConfig(
|
||||
return "", err
|
||||
}
|
||||
|
||||
return h.marshalTargetConfig(w, r, delivery.HTTPTargetConfig{
|
||||
return marshalTargetConfig(w, delivery.HTTPTargetConfig{
|
||||
URL: in.URL,
|
||||
Headers: headers,
|
||||
Timeout: timeout,
|
||||
@@ -1673,7 +1537,7 @@ func (h *Handlers) buildSlackTargetConfig(
|
||||
return "", err
|
||||
}
|
||||
|
||||
return h.marshalTargetConfig(w, r, delivery.SlackTargetConfig{
|
||||
return marshalTargetConfig(w, delivery.SlackTargetConfig{
|
||||
WebhookURL: targetURL,
|
||||
})
|
||||
}
|
||||
@@ -1717,12 +1581,11 @@ func (h *Handlers) validateTargetURL(
|
||||
msg := "Invalid target URL: " + err.Error()
|
||||
|
||||
// Only a private or reserved address's refusal says how
|
||||
// to allow it. Other refusals never do: link-local, the
|
||||
// unspecified addresses and the unconditional metadata
|
||||
// addresses cannot be opened, and the default
|
||||
// blocklist's public addresses, which listing does open,
|
||||
// hand out credentials.
|
||||
if errors.Is(err, delivery.ErrBlockedPrivateOrReservedIP) {
|
||||
// to allow it. Metadata refusals never do: link-local and
|
||||
// the other unconditional metadata addresses cannot be
|
||||
// opened, and Azure's WireServer, which listing does
|
||||
// open, hands out VM credentials.
|
||||
if errors.Is(err, delivery.ErrBlockedIP) {
|
||||
msg += ". Private and reserved addresses are refused " +
|
||||
"by default; the server's ALLOWED_EGRESS_CIDRS " +
|
||||
"setting allows named networks (see \"Allowing " +
|
||||
@@ -1739,14 +1602,16 @@ func (h *Handlers) validateTargetURL(
|
||||
|
||||
// marshalTargetConfig serialises a target configuration for storage,
|
||||
// writing a 500 itself if it cannot.
|
||||
func (h *Handlers) marshalTargetConfig(
|
||||
func marshalTargetConfig(
|
||||
w http.ResponseWriter,
|
||||
r *http.Request,
|
||||
cfg any,
|
||||
) (string, error) {
|
||||
configBytes, err := json.Marshal(cfg)
|
||||
if err != nil {
|
||||
h.serverError(w, r, "failed to encode target config", err)
|
||||
http.Error(
|
||||
w, "Internal server error",
|
||||
http.StatusInternalServerError,
|
||||
)
|
||||
|
||||
return "", err
|
||||
}
|
||||
@@ -1762,7 +1627,6 @@ func (h *Handlers) marshalTargetConfig(
|
||||
// expiry yields an empty config (the keep-forever default).
|
||||
func (h *Handlers) buildDatabaseTargetConfig(
|
||||
w http.ResponseWriter,
|
||||
r *http.Request,
|
||||
expiry string,
|
||||
) (string, error) {
|
||||
expiry = strings.TrimSpace(expiry)
|
||||
@@ -1781,8 +1645,8 @@ func (h *Handlers) buildDatabaseTargetConfig(
|
||||
return "", err
|
||||
}
|
||||
|
||||
return h.marshalTargetConfig(
|
||||
w, r, map[string]any{"expiry": expiry},
|
||||
return marshalTargetConfig(
|
||||
w, map[string]any{"expiry": expiry},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1792,33 +1656,30 @@ func (h *Handlers) HandleEntrypointDelete() http.HandlerFunc {
|
||||
"entrypointID", &database.Entrypoint{},
|
||||
"failed to delete entrypoint",
|
||||
nil,
|
||||
entrypointDeleted,
|
||||
)
|
||||
}
|
||||
|
||||
// HandleTargetDelete handles deleting a target. A deleted
|
||||
// database target's archive writer is evicted and its handle
|
||||
// closed; the archive file is left on disk.
|
||||
// HandleTargetDelete handles deleting a target. Deleting the
|
||||
// last database target of a webhook leaves its archive writer
|
||||
// with nothing to write, so the writer is evicted and its
|
||||
// handle closed; the archive file is left on disk.
|
||||
func (h *Handlers) HandleTargetDelete() http.HandlerFunc {
|
||||
return h.deleteChildResource(
|
||||
"targetID", &database.Target{},
|
||||
"failed to delete target",
|
||||
h.evictTargetArchiveWriter,
|
||||
targetDeleted,
|
||||
h.evictArchiveWriterIfUnused,
|
||||
)
|
||||
}
|
||||
|
||||
// deleteChildResource returns a handler that deletes a child
|
||||
// resource (entrypoint or target) belonging to a webhook. The
|
||||
// optional afterDelete hook runs with the child's id once the
|
||||
// delete has removed it, before the redirect, which carries done as
|
||||
// its notice.
|
||||
// optional afterDelete hook runs with the webhook's id once the
|
||||
// delete has succeeded, before the redirect.
|
||||
func (h *Handlers) deleteChildResource(
|
||||
idParam string,
|
||||
model any,
|
||||
errMsg string,
|
||||
afterDelete func(childID string),
|
||||
done noticeCode,
|
||||
afterDelete func(webhookID string),
|
||||
) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
userID, ok := h.getUserID(r)
|
||||
@@ -1839,7 +1700,7 @@ func (h *Handlers) deleteChildResource(
|
||||
"id = ? AND user_id = ?", sourceID, userID,
|
||||
).First(&webhook).Error
|
||||
if err != nil {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
http.NotFound(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -1849,20 +1710,22 @@ func (h *Handlers) deleteChildResource(
|
||||
childID, webhook.ID,
|
||||
).Delete(model)
|
||||
if result.Error != nil {
|
||||
h.serverError(w, r, errMsg, result.Error)
|
||||
h.log.Error(errMsg, "error", result.Error)
|
||||
http.Error(
|
||||
w, "Internal server error",
|
||||
http.StatusInternalServerError,
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// Only for a row this webhook really had: the id came from
|
||||
// the URL and may name another webhook's child.
|
||||
if afterDelete != nil && result.RowsAffected > 0 {
|
||||
afterDelete(childID)
|
||||
if afterDelete != nil {
|
||||
afterDelete(webhook.ID)
|
||||
}
|
||||
|
||||
http.Redirect(
|
||||
w, r,
|
||||
withNotice("/hook/"+webhook.ID, done),
|
||||
"/source/"+webhook.ID,
|
||||
http.StatusSeeOther,
|
||||
)
|
||||
}
|
||||
@@ -1873,7 +1736,7 @@ func (h *Handlers) deleteChildResource(
|
||||
func (h *Handlers) HandleEntrypointToggle() http.HandlerFunc {
|
||||
return h.toggleChildResource(
|
||||
"entrypointID",
|
||||
func(webhookID, childID string) (bool, error) {
|
||||
func(webhookID, childID string) error {
|
||||
var ep database.Entrypoint
|
||||
|
||||
err := h.db.DB().Where(
|
||||
@@ -1881,15 +1744,14 @@ func (h *Handlers) HandleEntrypointToggle() http.HandlerFunc {
|
||||
childID, webhookID,
|
||||
).First(&ep).Error
|
||||
if err != nil {
|
||||
return false, err
|
||||
return err
|
||||
}
|
||||
|
||||
ep.Active = !ep.Active
|
||||
|
||||
return ep.Active, h.db.DB().Save(&ep).Error
|
||||
return h.db.DB().Save(&ep).Error
|
||||
},
|
||||
"failed to toggle entrypoint",
|
||||
entrypointActivated, entrypointDeactivated,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1897,7 +1759,7 @@ func (h *Handlers) HandleEntrypointToggle() http.HandlerFunc {
|
||||
func (h *Handlers) HandleTargetToggle() http.HandlerFunc {
|
||||
return h.toggleChildResource(
|
||||
"targetID",
|
||||
func(webhookID, childID string) (bool, error) {
|
||||
func(webhookID, childID string) error {
|
||||
var tgt database.Target
|
||||
|
||||
err := h.db.DB().Where(
|
||||
@@ -1905,31 +1767,23 @@ func (h *Handlers) HandleTargetToggle() http.HandlerFunc {
|
||||
childID, webhookID,
|
||||
).First(&tgt).Error
|
||||
if err != nil {
|
||||
return false, err
|
||||
return err
|
||||
}
|
||||
|
||||
// Only the active column: saving the whole row would
|
||||
// write back the name and settings read above over an
|
||||
// edit saved since.
|
||||
active := !tgt.Active
|
||||
tgt.Active = !tgt.Active
|
||||
|
||||
return active, h.db.DB().Model(&tgt).
|
||||
Update("active", active).Error
|
||||
return h.db.DB().Save(&tgt).Error
|
||||
},
|
||||
"failed to toggle target",
|
||||
targetActivated, targetDeactivated,
|
||||
)
|
||||
}
|
||||
|
||||
// toggleChildResource returns a handler that toggles the active
|
||||
// state of a child resource belonging to a webhook. toggleFn returns
|
||||
// the new state, and the redirect carries activated or deactivated as
|
||||
// its notice to match.
|
||||
// state of a child resource belonging to a webhook.
|
||||
func (h *Handlers) toggleChildResource(
|
||||
idParam string,
|
||||
toggleFn func(webhookID, childID string) (bool, error),
|
||||
toggleFn func(webhookID, childID string) error,
|
||||
errMsg string,
|
||||
activated, deactivated noticeCode,
|
||||
) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
userID, ok := h.getUserID(r)
|
||||
@@ -1950,26 +1804,25 @@ func (h *Handlers) toggleChildResource(
|
||||
"id = ? AND user_id = ?", sourceID, userID,
|
||||
).First(&webhook).Error
|
||||
if err != nil {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
http.NotFound(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
active, err := toggleFn(webhook.ID, childID)
|
||||
err = toggleFn(webhook.ID, childID)
|
||||
if err != nil {
|
||||
h.serverError(w, r, errMsg, err)
|
||||
h.log.Error(errMsg, "error", err)
|
||||
http.Error(
|
||||
w, "Internal server error",
|
||||
http.StatusInternalServerError,
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
done := deactivated
|
||||
if active {
|
||||
done = activated
|
||||
}
|
||||
|
||||
http.Redirect(
|
||||
w, r,
|
||||
withNotice("/hook/"+webhook.ID, done),
|
||||
"/source/"+webhook.ID,
|
||||
http.StatusSeeOther,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -2,20 +2,16 @@ package handlers_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/handlers"
|
||||
@@ -109,7 +105,7 @@ func submitCreate(
|
||||
form.Set("retention_days", *retention)
|
||||
}
|
||||
|
||||
req := formRequest("/hooks/new", cookies, form, nil)
|
||||
req := formRequest("/sources/new", cookies, form, nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
h.HandleSourceCreateSubmit().ServeHTTP(w, req)
|
||||
@@ -191,7 +187,6 @@ func storedRetentionDays(
|
||||
type sourceTestEnv struct {
|
||||
handlers *handlers.Handlers
|
||||
db *database.Database
|
||||
archives *recordingArchives
|
||||
cookies []*http.Cookie
|
||||
}
|
||||
|
||||
@@ -204,9 +199,7 @@ func setupSourceTest(t *testing.T) *sourceTestEnv {
|
||||
|
||||
var db *database.Database
|
||||
|
||||
var archives *recordingArchives
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &archives)
|
||||
app := newTestApp(t, &h, &sess, &db)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
@@ -214,7 +207,6 @@ func setupSourceTest(t *testing.T) *sourceTestEnv {
|
||||
return &sourceTestEnv{
|
||||
handlers: h,
|
||||
db: db,
|
||||
archives: archives,
|
||||
cookies: authenticatedCookies(
|
||||
t, sess, sourceTestUserID, "sourceuser",
|
||||
),
|
||||
@@ -273,7 +265,7 @@ func TestHandleSourceCreate_PrefillsDefaultFromConstant(t *testing.T) {
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
env.handlers.HandleSourceCreate().ServeHTTP(
|
||||
w, getRequest(t, "/hooks/new", env.cookies, nil),
|
||||
w, getRequest(t, "/sources/new", env.cookies, nil),
|
||||
)
|
||||
|
||||
require.Equal(t, http.StatusOK, w.Code)
|
||||
@@ -410,7 +402,7 @@ func TestHandleSourceCreateSubmit_RejectedFormKeepsUserInput(
|
||||
form.Set("description", description)
|
||||
form.Set("retention_days", "nonsense")
|
||||
|
||||
req := formRequest("/hooks/new", env.cookies, form, nil)
|
||||
req := formRequest("/sources/new", env.cookies, form, nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
env.handlers.HandleSourceCreateSubmit().ServeHTTP(w, req)
|
||||
@@ -438,7 +430,7 @@ func submitEdit(
|
||||
form.Set("retention_days", retention)
|
||||
|
||||
req := formRequest(
|
||||
"/hook/"+wh.ID+"/edit",
|
||||
"/source/"+wh.ID+"/edit",
|
||||
env.cookies,
|
||||
form,
|
||||
map[string]string{sourceIDParam: wh.ID},
|
||||
@@ -506,301 +498,6 @@ func TestHandleSourceEditSubmit_EmptyRetentionLeavesValueUnchanged(
|
||||
assert.Equal(t, 7, storedRetentionDays(t, env.db, wh.ID))
|
||||
}
|
||||
|
||||
// renamedWebhookName is the name the rename tests give a webhook.
|
||||
const renamedWebhookName = "Renamed"
|
||||
|
||||
// TestHandleSourceEditSubmit_RenamesArchives proves that a save
|
||||
// that keeps the webhook's name renames nothing, and that renaming a
|
||||
// webhook renames the archive of each of its database targets and
|
||||
// asks nothing of its other targets.
|
||||
func TestHandleSourceEditSubmit_RenamesArchives(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
wh := seedWebhookWithRetention(t, env.db, 7)
|
||||
first := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
||||
second := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
||||
seedTarget(t, env.db, wh.ID, database.TargetTypeLog)
|
||||
|
||||
w := submitEdit(t, env, wh, "")
|
||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||
assert.Empty(t, env.archives.Renames())
|
||||
|
||||
wh.Name = renamedWebhookName
|
||||
|
||||
w = submitEdit(t, env, wh, "")
|
||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||
|
||||
assert.ElementsMatch(
|
||||
t,
|
||||
[]archiveRename{
|
||||
{first.ID, renamedWebhookName, first.Name},
|
||||
{second.ID, renamedWebhookName, second.Name},
|
||||
},
|
||||
env.archives.Renames(),
|
||||
)
|
||||
}
|
||||
|
||||
// TestHandleSourceEditSubmit_FailedRenameKeepsTheName proves that a
|
||||
// webhook whose archive cannot be renamed keeps its stored name, so
|
||||
// the name on disk and the name in the UI do not part, and that the
|
||||
// handler puts back what it may already have moved.
|
||||
func TestHandleSourceEditSubmit_FailedRenameKeepsTheName(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
wh := seedWebhookWithRetention(t, env.db, 7)
|
||||
tgt := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
||||
|
||||
env.archives.FailRenames(tgt.ID, errInjectedRename)
|
||||
|
||||
oldName := wh.Name
|
||||
wh.Name = renamedWebhookName
|
||||
|
||||
w := submitEdit(t, env, wh, "")
|
||||
require.Equal(t, http.StatusInternalServerError, w.Code)
|
||||
|
||||
var stored database.Webhook
|
||||
|
||||
require.NoError(
|
||||
t, env.db.DB().First(&stored, "id = ?", wh.ID).Error,
|
||||
)
|
||||
assert.Equal(t, oldName, stored.Name)
|
||||
|
||||
assert.Equal(
|
||||
t,
|
||||
[]archiveRename{
|
||||
{tgt.ID, renamedWebhookName, tgt.Name},
|
||||
{tgt.ID, oldName, tgt.Name},
|
||||
},
|
||||
env.archives.Renames(),
|
||||
)
|
||||
}
|
||||
|
||||
// TestHandleSourceEditSubmit_FailedSaveRenamesBack proves that when
|
||||
// the archive is renamed but the new name cannot be saved, the
|
||||
// archive is renamed back to the stored name and the stored name
|
||||
// stays.
|
||||
func TestHandleSourceEditSubmit_FailedSaveRenamesBack(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
wh := seedWebhookWithRetention(t, env.db, 7)
|
||||
tgt := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
||||
|
||||
failSaveOnTable(t, env.db, "webhooks")
|
||||
|
||||
oldName := wh.Name
|
||||
wh.Name = renamedWebhookName
|
||||
|
||||
w := submitEdit(t, env, wh, "")
|
||||
require.Equal(t, http.StatusInternalServerError, w.Code)
|
||||
|
||||
var stored database.Webhook
|
||||
|
||||
require.NoError(
|
||||
t, env.db.DB().First(&stored, "id = ?", wh.ID).Error,
|
||||
)
|
||||
assert.Equal(t, oldName, stored.Name)
|
||||
|
||||
assert.Equal(
|
||||
t,
|
||||
[]archiveRename{
|
||||
{tgt.ID, renamedWebhookName, tgt.Name},
|
||||
{tgt.ID, oldName, tgt.Name},
|
||||
},
|
||||
env.archives.Renames(),
|
||||
)
|
||||
}
|
||||
|
||||
// errInjectedRead is the failure a test makes reads of the main
|
||||
// database report.
|
||||
var errInjectedRead = errors.New("injected read failure")
|
||||
|
||||
// TestHandleSourceEditSubmit_FailedSaveRenamesBackWithoutReading
|
||||
// proves that when the save fails and every later read of the main
|
||||
// database fails too, each archive the rename moved is still renamed
|
||||
// back: the move back needs no second read of the webhook's targets.
|
||||
func TestHandleSourceEditSubmit_FailedSaveRenamesBackWithoutReading(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
wh := seedWebhookWithRetention(t, env.db, 7)
|
||||
first := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
||||
second := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
||||
|
||||
var saveFailed atomic.Bool
|
||||
|
||||
require.NoError(t, env.db.DB().Callback().Update().
|
||||
Before("gorm:update").
|
||||
Register("test:fail_save", func(tx *gorm.DB) {
|
||||
saveFailed.Store(true)
|
||||
|
||||
_ = tx.AddError(errInjectedSave)
|
||||
}),
|
||||
)
|
||||
require.NoError(t, env.db.DB().Callback().Query().
|
||||
Before("gorm:query").
|
||||
Register("test:fail_reads_after_save", func(tx *gorm.DB) {
|
||||
if saveFailed.Load() {
|
||||
_ = tx.AddError(errInjectedRead)
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
oldName := wh.Name
|
||||
wh.Name = renamedWebhookName
|
||||
|
||||
w := submitEdit(t, env, wh, "")
|
||||
require.Equal(t, http.StatusInternalServerError, w.Code)
|
||||
|
||||
assert.Equal(
|
||||
t,
|
||||
[]archiveRename{
|
||||
{first.ID, renamedWebhookName, first.Name},
|
||||
{second.ID, renamedWebhookName, second.Name},
|
||||
{first.ID, oldName, first.Name},
|
||||
{second.ID, oldName, second.Name},
|
||||
},
|
||||
env.archives.Renames(),
|
||||
)
|
||||
}
|
||||
|
||||
// TestHandleSourceEditSubmit_EditsDoNotInterleave proves that a second
|
||||
// webhook edit submitted while the first is inside its archive rename
|
||||
// does not run until the first is saved, so afterwards the stored
|
||||
// names are the ones the archive was last renamed to. The stand-in's
|
||||
// last rename is the name the file has on disk.
|
||||
func TestHandleSourceEditSubmit_EditsDoNotInterleave(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
wh := seedWebhookWithRetention(t, env.db, 7)
|
||||
tgt := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
||||
|
||||
entered, release := env.archives.BlockNextRename()
|
||||
|
||||
firstEdit, secondEdit := wh, wh
|
||||
firstEdit.Name = "First"
|
||||
secondEdit.Name = "Second"
|
||||
|
||||
firstCode := make(chan int, 1)
|
||||
|
||||
go func() { firstCode <- submitEdit(t, env, firstEdit, "").Code }()
|
||||
|
||||
<-entered
|
||||
|
||||
secondCode := make(chan int, 1)
|
||||
|
||||
go func() { secondCode <- submitEdit(t, env, secondEdit, "").Code }()
|
||||
|
||||
// Were the edits not ordered, the second would run to its end in
|
||||
// this time, while the first is still inside its rename.
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
release()
|
||||
|
||||
assert.Equal(t, http.StatusSeeOther, <-firstCode)
|
||||
assert.Equal(t, http.StatusSeeOther, <-secondCode)
|
||||
|
||||
var (
|
||||
storedWebhook database.Webhook
|
||||
storedTarget database.Target
|
||||
)
|
||||
|
||||
require.NoError(
|
||||
t, env.db.DB().First(&storedWebhook, "id = ?", wh.ID).Error,
|
||||
)
|
||||
require.NoError(
|
||||
t, env.db.DB().First(&storedTarget, "id = ?", tgt.ID).Error,
|
||||
)
|
||||
|
||||
renames := env.archives.Renames()
|
||||
require.NotEmpty(t, renames)
|
||||
assert.Equal(
|
||||
t,
|
||||
archiveRename{tgt.ID, storedWebhook.Name, storedTarget.Name},
|
||||
renames[len(renames)-1],
|
||||
)
|
||||
}
|
||||
|
||||
// TestHandleSourceEditSubmit_FailedRenameRenamesTheOthersBack proves
|
||||
// that when a webhook has three database targets and only the middle
|
||||
// one's archive cannot be renamed, the stored name stays and both
|
||||
// others are renamed back, the last one included: the move back does
|
||||
// not stop at the target it cannot rename. The handler reaches the
|
||||
// targets in the order they were created, which the exact sequence
|
||||
// below pins, so the refused target always comes before the last.
|
||||
func TestHandleSourceEditSubmit_FailedRenameRenamesTheOthersBack(
|
||||
t *testing.T,
|
||||
) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
wh := seedWebhookWithRetention(t, env.db, 7)
|
||||
first := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
||||
middle := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
||||
last := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
||||
|
||||
env.archives.FailRenames(middle.ID, errNameTaken)
|
||||
|
||||
oldName := wh.Name
|
||||
wh.Name = renamedWebhookName
|
||||
|
||||
w := submitEdit(t, env, wh, "")
|
||||
require.Equal(t, http.StatusConflict, w.Code)
|
||||
|
||||
var stored database.Webhook
|
||||
|
||||
require.NoError(
|
||||
t, env.db.DB().First(&stored, "id = ?", wh.ID).Error,
|
||||
)
|
||||
assert.Equal(t, oldName, stored.Name)
|
||||
|
||||
assert.Equal(
|
||||
t,
|
||||
[]archiveRename{
|
||||
{first.ID, renamedWebhookName, first.Name},
|
||||
{middle.ID, renamedWebhookName, middle.Name},
|
||||
{last.ID, renamedWebhookName, last.Name},
|
||||
{first.ID, oldName, first.Name},
|
||||
{middle.ID, oldName, middle.Name},
|
||||
{last.ID, oldName, last.Name},
|
||||
},
|
||||
env.archives.Renames(),
|
||||
)
|
||||
}
|
||||
|
||||
// TestHandleSourceEditSubmit_ArchiveNameTaken proves that when a file
|
||||
// already has an archive's new name, the edit is refused with an
|
||||
// error naming that file, and the webhook keeps its stored name.
|
||||
func TestHandleSourceEditSubmit_ArchiveNameTaken(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
wh := seedWebhookWithRetention(t, env.db, 7)
|
||||
tgt := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
||||
|
||||
env.archives.FailRenames(tgt.ID, errNameTaken)
|
||||
|
||||
oldName := wh.Name
|
||||
wh.Name = renamedWebhookName
|
||||
|
||||
w := submitEdit(t, env, wh, "")
|
||||
require.Equal(t, http.StatusConflict, w.Code)
|
||||
assert.Contains(t, w.Body.String(), "archive-taken.db")
|
||||
|
||||
var stored database.Webhook
|
||||
|
||||
require.NoError(
|
||||
t, env.db.DB().First(&stored, "id = ?", wh.ID).Error,
|
||||
)
|
||||
assert.Equal(t, oldName, stored.Name)
|
||||
}
|
||||
|
||||
// TestSourceEditForm_ForeverWebhookRoundTrips walks the exact path that
|
||||
// the removed max="365" cap used to break: render the edit form for a
|
||||
// retain-forever webhook, confirm the pre-filled sentinel is not capped
|
||||
@@ -815,7 +512,7 @@ func TestSourceEditForm_ForeverWebhookRoundTrips(t *testing.T) {
|
||||
)
|
||||
|
||||
req := getRequest(
|
||||
t, "/hook/"+wh.ID+"/edit", env.cookies,
|
||||
t, "/source/"+wh.ID+"/edit", env.cookies,
|
||||
map[string]string{sourceIDParam: wh.ID},
|
||||
)
|
||||
w := httptest.NewRecorder()
|
||||
@@ -870,7 +567,7 @@ func TestSourceListAndDetail_ShowForeverNotTheSentinelNumber(
|
||||
|
||||
listW := httptest.NewRecorder()
|
||||
env.handlers.HandleSourceList().ServeHTTP(
|
||||
listW, getRequest(t, "/hooks", env.cookies, nil),
|
||||
listW, getRequest(t, "/sources", env.cookies, nil),
|
||||
)
|
||||
|
||||
require.Equal(t, http.StatusOK, listW.Code)
|
||||
@@ -881,7 +578,7 @@ func TestSourceListAndDetail_ShowForeverNotTheSentinelNumber(
|
||||
env.handlers.HandleSourceDetail().ServeHTTP(
|
||||
detailW,
|
||||
getRequest(
|
||||
t, "/hook/"+wh.ID, env.cookies,
|
||||
t, "/source/"+wh.ID, env.cookies,
|
||||
map[string]string{sourceIDParam: wh.ID},
|
||||
),
|
||||
)
|
||||
|
||||
@@ -76,11 +76,11 @@ func postTargetCreate(
|
||||
router := chi.NewRouter()
|
||||
router.Use(mw.Logging())
|
||||
router.Post(
|
||||
"/hook/{sourceID}/targets",
|
||||
"/source/{sourceID}/targets",
|
||||
env.handlers.HandleTargetCreate(),
|
||||
)
|
||||
|
||||
target := "/hook/" + webhookID + "/targets"
|
||||
target := "/source/" + webhookID + "/targets"
|
||||
if query != "" {
|
||||
target += "?" + query
|
||||
}
|
||||
@@ -114,7 +114,7 @@ func postTargetCreate(
|
||||
// regression test for the ingress leak. r.FormValue falls back to the
|
||||
// query string when a field is absent from the POST body, so
|
||||
//
|
||||
// POST /hook/{id}/targets?url=https://hooks.slack.com/services/...
|
||||
// POST /source/{id}/targets?url=https://hooks.slack.com/services/...
|
||||
//
|
||||
// with an empty url field used to create a working target from a value
|
||||
// carried on the request line — where logs, proxies, Referer headers
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi"
|
||||
@@ -48,7 +47,7 @@ type targetEditView struct {
|
||||
//
|
||||
// This page is the one place the full destination URL and header
|
||||
// values are shown. It is reachable only through the
|
||||
// /hook/{sourceID} route group, which supplies RequireAuth and
|
||||
// /source/{sourceID} route group, which supplies RequireAuth and
|
||||
// NoCache, and only for a target of a webhook the session's user
|
||||
// owns; masking (delivery.TargetView) is unchanged everywhere else.
|
||||
func (h *Handlers) HandleTargetEdit() http.HandlerFunc {
|
||||
@@ -80,9 +79,6 @@ func (h *Handlers) HandleTargetEdit() http.HandlerFunc {
|
||||
// HandleTargetEditSubmit handles the target edit form submission.
|
||||
func (h *Handlers) HandleTargetEditSubmit() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
h.renameMu.Lock()
|
||||
defer h.renameMu.Unlock()
|
||||
|
||||
webhook, target, ok := h.ownedTarget(w, r)
|
||||
if !ok {
|
||||
return
|
||||
@@ -92,7 +88,9 @@ func (h *Handlers) HandleTargetEditSubmit() http.HandlerFunc {
|
||||
// middleware, which runs before CSRF parses the form.
|
||||
err := r.ParseForm()
|
||||
if err != nil {
|
||||
h.renderError(w, r, http.StatusBadRequest)
|
||||
http.Error(
|
||||
w, "Bad request", http.StatusBadRequest,
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -154,69 +152,21 @@ func (h *Handlers) applyTargetEdit(
|
||||
target.MaxRetries = retries
|
||||
}
|
||||
|
||||
oldName := target.Name
|
||||
target.Name = name
|
||||
target.Config = configJSON
|
||||
|
||||
// A new name renames the archive file before it is saved (see
|
||||
// delivery.Engine.Rename). If either step fails, it goes back to
|
||||
// the name that is still stored.
|
||||
err = h.renameTargetArchive(target, webhook.Name, oldName, name)
|
||||
if err == nil {
|
||||
err = h.db.DB().Save(target).Error
|
||||
}
|
||||
|
||||
err = h.db.DB().Save(target).Error
|
||||
if err != nil {
|
||||
restoreErr := h.renameTargetArchive(
|
||||
target, webhook.Name, name, oldName,
|
||||
)
|
||||
if restoreErr != nil {
|
||||
h.log.Error(
|
||||
"failed to rename archive back",
|
||||
"target_id", target.ID,
|
||||
"error", restoreErr,
|
||||
)
|
||||
}
|
||||
|
||||
if errors.Is(err, delivery.ErrArchiveNameTaken) {
|
||||
http.Error(
|
||||
w,
|
||||
"Not saved: "+err.Error()+
|
||||
". Move that archive out of the data directory, "+
|
||||
"its .db together with any -wal and -shm beside "+
|
||||
"it, then save again.",
|
||||
http.StatusConflict,
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
h.serverError(w, r, "failed to update target", err)
|
||||
h.serverError(w, "failed to update target", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
http.Redirect(
|
||||
w, r, withNotice("/hook/"+webhook.ID, targetSaved),
|
||||
http.StatusSeeOther,
|
||||
w, r, "/source/"+webhook.ID, http.StatusSeeOther,
|
||||
)
|
||||
}
|
||||
|
||||
// renameTargetArchive renames a database target's archive file from
|
||||
// the target name oldName to newName. It does nothing when the name
|
||||
// is unchanged; other target types have no archive.
|
||||
func (h *Handlers) renameTargetArchive(
|
||||
target *database.Target,
|
||||
webhookName, oldName, newName string,
|
||||
) error {
|
||||
if h.archives == nil || oldName == newName ||
|
||||
target.Type != database.TargetTypeDatabase {
|
||||
return nil
|
||||
}
|
||||
|
||||
return h.archives.Rename(target.ID, webhookName, newName)
|
||||
}
|
||||
|
||||
// renderTargetEdit renders the target edit page with an optional
|
||||
// error message.
|
||||
func (h *Handlers) renderTargetEdit(
|
||||
@@ -270,7 +220,7 @@ func (h *Handlers) ownedTarget(
|
||||
chi.URLParam(r, "targetID"), webhook.ID,
|
||||
).First(&target).Error
|
||||
if err != nil {
|
||||
h.renderError(w, r, http.StatusNotFound)
|
||||
http.NotFound(w, r)
|
||||
|
||||
return database.Webhook{}, nil, false
|
||||
}
|
||||
|
||||
@@ -42,15 +42,15 @@ const (
|
||||
func targetRouter(env *sourceTestEnv) *chi.Mux {
|
||||
router := chi.NewRouter()
|
||||
router.Post(
|
||||
"/hook/{sourceID}/targets",
|
||||
"/source/{sourceID}/targets",
|
||||
env.handlers.HandleTargetCreate(),
|
||||
)
|
||||
router.Get(
|
||||
"/hook/{sourceID}/targets/{targetID}/edit",
|
||||
"/source/{sourceID}/targets/{targetID}/edit",
|
||||
env.handlers.HandleTargetEdit(),
|
||||
)
|
||||
router.Post(
|
||||
"/hook/{sourceID}/targets/{targetID}/edit",
|
||||
"/source/{sourceID}/targets/{targetID}/edit",
|
||||
env.handlers.HandleTargetEditSubmit(),
|
||||
)
|
||||
|
||||
@@ -117,7 +117,7 @@ func seedHTTPTarget(
|
||||
|
||||
w := serveTarget(
|
||||
env, http.MethodPost,
|
||||
"/hook/"+webhook.ID+"/targets", form,
|
||||
"/source/"+webhook.ID+"/targets", form,
|
||||
)
|
||||
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
||||
|
||||
@@ -188,7 +188,7 @@ func submitTargetEdit(
|
||||
) *httptest.ResponseRecorder {
|
||||
return serveTarget(
|
||||
env, http.MethodPost,
|
||||
"/hook/"+webhookID+"/targets/"+targetID+"/edit",
|
||||
"/source/"+webhookID+"/targets/"+targetID+"/edit",
|
||||
form,
|
||||
)
|
||||
}
|
||||
@@ -401,7 +401,7 @@ func TestHandleTargetEdit_PrefillsTheStoredValuesUnmasked(
|
||||
|
||||
w := serveTarget(
|
||||
env, http.MethodGet,
|
||||
"/hook/"+webhook.ID+"/targets/"+target.ID+"/edit",
|
||||
"/source/"+webhook.ID+"/targets/"+target.ID+"/edit",
|
||||
nil,
|
||||
)
|
||||
require.Equal(t, http.StatusOK, w.Code)
|
||||
@@ -508,7 +508,7 @@ func assertEditIgnoresQueryString(
|
||||
|
||||
w := serveTarget(
|
||||
env, http.MethodPost,
|
||||
"/hook/"+webhook.ID+"/targets/"+target.ID+
|
||||
"/source/"+webhook.ID+"/targets/"+target.ID+
|
||||
"/edit?url="+url.QueryEscape(editReplacedURL)+
|
||||
"&headers="+url.QueryEscape(editAuthHeader),
|
||||
form,
|
||||
@@ -592,7 +592,7 @@ func assertTargetOfAnotherWebhook404s(
|
||||
|
||||
get := serveTarget(
|
||||
env, http.MethodGet,
|
||||
"/hook/"+mine.ID+"/targets/"+target.ID+"/edit", nil,
|
||||
"/source/"+mine.ID+"/targets/"+target.ID+"/edit", nil,
|
||||
)
|
||||
assert.Equal(t, http.StatusNotFound, get.Code)
|
||||
|
||||
@@ -630,105 +630,8 @@ func assertWebhookOfAnotherUser404s(
|
||||
|
||||
w := serveTarget(
|
||||
env, http.MethodGet,
|
||||
"/hook/"+other.ID+"/targets/"+target.ID+"/edit", nil,
|
||||
"/source/"+other.ID+"/targets/"+target.ID+"/edit", nil,
|
||||
)
|
||||
|
||||
assert.Equal(t, http.StatusNotFound, w.Code)
|
||||
}
|
||||
|
||||
// renamedTargetName is the name the rename tests give a target.
|
||||
const renamedTargetName = "Long Term"
|
||||
|
||||
// TestHandleTargetEditSubmit_RenamesArchive proves that renaming a
|
||||
// database target renames its archive, that a save that keeps the
|
||||
// name renames nothing, that a target of another type has no archive
|
||||
// to rename, and that a target whose archive cannot be renamed keeps
|
||||
// its stored name. When a file already has the archive's new name,
|
||||
// the edit is refused with an error naming that file.
|
||||
func TestHandleTargetEditSubmit_RenamesArchive(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
wh := seedWebhookWithRetention(t, env.db, 7)
|
||||
archive := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
||||
rename := url.Values{"name": {renamedTargetName}}
|
||||
|
||||
w := submitTargetEdit(env, wh.ID, archive.ID, rename)
|
||||
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
||||
assert.Equal(
|
||||
t,
|
||||
[]archiveRename{{archive.ID, wh.Name, renamedTargetName}},
|
||||
env.archives.Renames(),
|
||||
)
|
||||
|
||||
w = submitTargetEdit(env, wh.ID, archive.ID, rename)
|
||||
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
||||
assert.Len(
|
||||
t, env.archives.Renames(), 1,
|
||||
"a save that keeps the name renames nothing",
|
||||
)
|
||||
|
||||
httpWebhook, httpTarget := seedHTTPTarget(t, env, "", "")
|
||||
|
||||
w = submitTargetEdit(
|
||||
env, httpWebhook.ID, httpTarget.ID,
|
||||
editForm(editOriginalURL, "", ""),
|
||||
)
|
||||
require.Equal(t, http.StatusSeeOther, w.Code, w.Body.String())
|
||||
assert.Len(
|
||||
t, env.archives.Renames(), 1,
|
||||
"an HTTP target has no archive to rename",
|
||||
)
|
||||
|
||||
again := url.Values{"name": {"Again"}}
|
||||
|
||||
env.archives.FailRenames(archive.ID, errInjectedRename)
|
||||
|
||||
w = submitTargetEdit(env, wh.ID, archive.ID, again)
|
||||
require.Equal(t, http.StatusInternalServerError, w.Code)
|
||||
assert.Equal(
|
||||
t, renamedTargetName, storedTarget(t, env, archive.ID).Name,
|
||||
"a target whose archive was not renamed keeps its name",
|
||||
)
|
||||
|
||||
env.archives.FailRenames(archive.ID, errNameTaken)
|
||||
|
||||
w = submitTargetEdit(env, wh.ID, archive.ID, again)
|
||||
require.Equal(t, http.StatusConflict, w.Code)
|
||||
assert.Contains(t, w.Body.String(), "archive-taken.db")
|
||||
assert.Equal(
|
||||
t, renamedTargetName, storedTarget(t, env, archive.ID).Name,
|
||||
)
|
||||
}
|
||||
|
||||
// TestHandleTargetEditSubmit_FailedSaveRenamesBack proves that when a
|
||||
// database target's archive is renamed but the new name cannot be
|
||||
// saved, the archive is renamed back to the stored name and the
|
||||
// stored name stays.
|
||||
func TestHandleTargetEditSubmit_FailedSaveRenamesBack(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
wh := seedWebhookWithRetention(t, env.db, 7)
|
||||
archive := seedTarget(t, env.db, wh.ID, database.TargetTypeDatabase)
|
||||
|
||||
failSaveOnTable(t, env.db, "targets")
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("name", renamedTargetName)
|
||||
|
||||
w := submitTargetEdit(env, wh.ID, archive.ID, form)
|
||||
require.Equal(t, http.StatusInternalServerError, w.Code)
|
||||
assert.Equal(
|
||||
t, archive.Name, storedTarget(t, env, archive.ID).Name,
|
||||
)
|
||||
|
||||
assert.Equal(
|
||||
t,
|
||||
[]archiveRename{
|
||||
{archive.ID, wh.Name, renamedTargetName},
|
||||
{archive.ID, wh.Name, archive.Name},
|
||||
},
|
||||
env.archives.Renames(),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@ func TestTargetRefusal_PrivateDestinationSaysHowToAllowIt(
|
||||
t.Parallel()
|
||||
|
||||
webhook := seedWebhookWithRetention(t, env.db, 30)
|
||||
targetsPath := "/hook/" + webhook.ID + "/targets"
|
||||
targetsPath := "/source/" + webhook.ID + "/targets"
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("name", "private")
|
||||
@@ -105,7 +105,7 @@ func TestTargetRefusal_MetadataDestinationDoesNotSayHowToAllowIt(
|
||||
|
||||
w := serveTarget(
|
||||
env, http.MethodPost,
|
||||
"/hook/"+webhook.ID+"/targets", form,
|
||||
"/source/"+webhook.ID+"/targets", form,
|
||||
)
|
||||
assert.Equal(t, http.StatusBadRequest, w.Code)
|
||||
assert.NotContains(
|
||||
|
||||
@@ -102,7 +102,7 @@ func createWithRetries(
|
||||
|
||||
w := serveTarget(
|
||||
env, http.MethodPost,
|
||||
"/hook/"+webhook.ID+"/targets",
|
||||
"/source/"+webhook.ID+"/targets",
|
||||
createRetriesForm(retries),
|
||||
)
|
||||
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// TestHandleTargetToggle_DoesNotUndoAnEdit proves that a toggle which
|
||||
// loaded the target before an edit of it was saved does not write the
|
||||
// old name and settings back over the edit. The edit is submitted from
|
||||
// a callback on the toggle's own read of the target, so it is saved
|
||||
// after that read and before the toggle writes.
|
||||
func TestHandleTargetToggle_DoesNotUndoAnEdit(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
env := setupSourceTest(t)
|
||||
wh, tgt := seedHTTPTarget(t, env, "", "")
|
||||
require.True(t, tgt.Active)
|
||||
|
||||
var (
|
||||
edited bool
|
||||
editCode int
|
||||
)
|
||||
|
||||
require.NoError(t, env.db.DB().Callback().Query().
|
||||
After("gorm:query").
|
||||
Register("test:edit_after_toggle_read", func(tx *gorm.DB) {
|
||||
// The edit reads the target too; only the toggle's read,
|
||||
// the first, submits it.
|
||||
if tx.Statement.Table != "targets" || edited {
|
||||
return
|
||||
}
|
||||
|
||||
edited = true
|
||||
editCode = submitTargetEdit(
|
||||
env, wh.ID, tgt.ID,
|
||||
editForm(editReplacedURL, "", ""),
|
||||
).Code
|
||||
}),
|
||||
)
|
||||
|
||||
req := postRequest(
|
||||
"/hook/"+wh.ID+"/targets/"+tgt.ID+"/toggle",
|
||||
env.cookies,
|
||||
map[string]string{paramSourceID: wh.ID, paramTargetID: tgt.ID},
|
||||
)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
env.handlers.HandleTargetToggle().ServeHTTP(w, req)
|
||||
|
||||
require.Equal(t, http.StatusSeeOther, w.Code)
|
||||
require.Equal(t, http.StatusSeeOther, editCode)
|
||||
|
||||
stored := storedTarget(t, env, tgt.ID)
|
||||
assert.False(t, stored.Active)
|
||||
assert.Equal(t, "edited-name", stored.Name)
|
||||
assert.Equal(t, 5, stored.MaxRetries)
|
||||
assert.Equal(
|
||||
t, editReplacedURL, storedHTTPConfig(t, env, tgt.ID).URL,
|
||||
)
|
||||
}
|
||||
@@ -54,7 +54,8 @@ func renderPage(
|
||||
}
|
||||
|
||||
// TestNavbarUsesWebhookTerminology pins the user-visible navigation
|
||||
// label to "Webhooks" and its link to the webhook list at /hooks.
|
||||
// label to "Webhooks". The /sources route is deliberately unchanged, so
|
||||
// the assertion targets the link text rather than the href.
|
||||
func TestNavbarUsesWebhookTerminology(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -94,11 +95,15 @@ func TestNavbarUsesWebhookTerminology(t *testing.T) {
|
||||
t, body, ">Sources<",
|
||||
"no user-visible element may still be labelled Sources",
|
||||
)
|
||||
assert.Contains(t, body, `href="/hooks"`)
|
||||
assert.Contains(
|
||||
t, body, `href="/sources"`,
|
||||
"the /sources route itself must not change",
|
||||
)
|
||||
}
|
||||
|
||||
// TestEditPageUsesWebhookTerminology pins the edit page's heading and
|
||||
// its back link to the webhook page at /hook/{id}.
|
||||
// its back link. The link's href still points at /source/{id}, which is
|
||||
// intentional: only user-visible copy changes.
|
||||
func TestEditPageUsesWebhookTerminology(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -125,57 +130,7 @@ func TestEditPageUsesWebhookTerminology(t *testing.T) {
|
||||
|
||||
assert.Contains(t, body, "Edit Webhook")
|
||||
assert.NotContains(t, body, ">Sources<")
|
||||
assert.Contains(t, body, `href="/hook/wh-1"`)
|
||||
}
|
||||
|
||||
// TestEventLogPageIsCalledFullEventLog pins the one name the event log
|
||||
// page at /hook/{id}/events goes by: both links to it on the webhook
|
||||
// page, and its own heading, read "Full Event Log".
|
||||
func TestEventLogPageIsCalledFullEventLog(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var h *handlers.Handlers
|
||||
|
||||
var sess *session.Session
|
||||
|
||||
app := newTestApp(t, &h, &sess)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
// A pointer, as in the handlers: source_detail.html calls
|
||||
// Webhook.RetentionLabel, a pointer method. Both pages only range
|
||||
// over their lists, and a list left out renders as empty, so the
|
||||
// lists are left out.
|
||||
webhook := &database.Webhook{Name: "wh", RetentionDays: 14}
|
||||
webhook.ID = testWebhookID
|
||||
|
||||
detailBody := renderPage(
|
||||
t, h, sess, "source_detail.html", map[string]any{
|
||||
dataKeyWebhook: webhook,
|
||||
},
|
||||
)
|
||||
|
||||
assert.Contains(
|
||||
t, detailBody,
|
||||
`<a href="/hook/wh-1/events" class="btn-secondary">Full Event Log</a>`,
|
||||
"the button at the top of the webhook page",
|
||||
)
|
||||
assert.Contains(
|
||||
t, detailBody,
|
||||
`<a href="/hook/wh-1/events" class="btn-text text-sm">Full Event Log</a>`,
|
||||
"the link under recent events",
|
||||
)
|
||||
|
||||
logBody := renderPage(t, h, sess, "source_logs.html", map[string]any{
|
||||
dataKeyWebhook: webhook,
|
||||
"TotalEvents": int64(0),
|
||||
})
|
||||
|
||||
assert.Contains(
|
||||
t, logBody,
|
||||
`<h1 class="text-2xl font-medium text-gray-900">Full Event Log</h1>`,
|
||||
)
|
||||
assert.Contains(t, body, `href="/source/wh-1"`)
|
||||
}
|
||||
|
||||
// TestCreateFormRetentionCopyMatchesBehaviour pins the create form's
|
||||
@@ -328,7 +283,7 @@ func TestEntrypointCopyButtonIsProgressiveEnhancement(t *testing.T) {
|
||||
t, body,
|
||||
`<code id="entrypoint-url-ep-1"`,
|
||||
)
|
||||
assert.Contains(t, body, "https://hooks.example.com/h/abc123")
|
||||
assert.Contains(t, body, "https://hooks.example.com/webhook/abc123")
|
||||
assert.Contains(
|
||||
t, body,
|
||||
`hidden data-copy-target="entrypoint-url-ep-1"`,
|
||||
|
||||
@@ -88,14 +88,14 @@ func (h *Handlers) processWebhookRequest(
|
||||
|
||||
headersJSON, err := json.Marshal(r.Header)
|
||||
if err != nil {
|
||||
h.receiverError(w, "failed to serialize headers", err)
|
||||
h.serverError(w, "failed to serialize headers", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
targets, err := h.loadActiveTargets(entrypoint.WebhookID)
|
||||
if err != nil {
|
||||
h.receiverError(w, "failed to query targets", err)
|
||||
h.serverError(w, "failed to query targets", err)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -131,7 +131,7 @@ func (h *Handlers) lookupEntrypoint(
|
||||
"path = ?", entrypointUUID,
|
||||
).First(&entrypoint)
|
||||
if result.Error != nil {
|
||||
// The receiver is unauthenticated and /h/{uuid}
|
||||
// The receiver is unauthenticated and /webhook/{uuid}
|
||||
// matches any single segment, so this value is entirely
|
||||
// client-chosen on exactly the branch where the lookup
|
||||
// failed. DEBUG is off by default; the cap is what keeps
|
||||
@@ -150,9 +150,7 @@ func (h *Handlers) lookupEntrypoint(
|
||||
return entrypoint, true
|
||||
}
|
||||
|
||||
// readWebhookBody reads and validates the request body size. This is
|
||||
// the receiver's only body cap: /h/{uuid} has no MaxBodySize
|
||||
// middleware (see Server.setupWebhookRoutes).
|
||||
// readWebhookBody reads and validates the request body size.
|
||||
func (h *Handlers) readWebhookBody(
|
||||
w http.ResponseWriter,
|
||||
r *http.Request,
|
||||
@@ -198,7 +196,7 @@ func (h *Handlers) createAndDeliverEvent(
|
||||
targets,
|
||||
)
|
||||
if err != nil {
|
||||
h.receiverError(w, "failed to store webhook event", err)
|
||||
h.serverError(w, "failed to store webhook event", err)
|
||||
|
||||
return
|
||||
}
|
||||
@@ -206,19 +204,6 @@ func (h *Handlers) createAndDeliverEvent(
|
||||
h.finishWebhookResponse(w, event, entrypoint, tasks)
|
||||
}
|
||||
|
||||
// receiverError logs an error and answers the sender with a plain-text
|
||||
// 500. The receiver's answers are for programs, so it never sends the
|
||||
// error page the web UI uses.
|
||||
func (h *Handlers) receiverError(
|
||||
w http.ResponseWriter, msg string, err error,
|
||||
) {
|
||||
h.log.Error(msg, "error", err)
|
||||
http.Error(
|
||||
w, "Internal server error",
|
||||
http.StatusInternalServerError,
|
||||
)
|
||||
}
|
||||
|
||||
// eventSource carries the fields a new event is built from. The
|
||||
// receiver fills it from the live request; the resubmit handler fills
|
||||
// it from a stored event. Both then go through createAndFanOut, so an
|
||||
@@ -268,12 +253,11 @@ func requestEventSource(
|
||||
}
|
||||
}
|
||||
|
||||
// createAndFanOut writes the event and one pending delivery per target,
|
||||
// and adds them to the webhook's running totals, in a single
|
||||
// transaction, then hands the tasks to the delivery engine. It is the
|
||||
// only path by which an event and its deliveries are created, so a
|
||||
// resubmitted event is retried, SSRF-guarded and circuit-broken
|
||||
// exactly as a received one is.
|
||||
// createAndFanOut writes the event and one pending delivery per target
|
||||
// in a single transaction, then hands the tasks to the delivery
|
||||
// engine. It is the only path by which an event and its deliveries are
|
||||
// created, so a resubmitted event is retried, SSRF-guarded and
|
||||
// circuit-broken exactly as a received one is.
|
||||
//
|
||||
// The tasks are returned as well as queued, so a caller can report how
|
||||
// many targets the event went to.
|
||||
@@ -313,15 +297,6 @@ func (h *Handlers) createAndFanOut(
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
err = database.AddEventTotals(tx, database.EventTotals{
|
||||
Events: 1, LastEventAt: &event.CreatedAt,
|
||||
})
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
err = tx.Commit().Error
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf(
|
||||
@@ -380,9 +355,8 @@ func (h *Handlers) finishWebhookResponse(
|
||||
}
|
||||
|
||||
// buildDeliveryTasks creates one pending delivery per target in the
|
||||
// transaction, adds each to its target's totals, and returns the tasks
|
||||
// for the delivery engine. The caller owns the transaction and rolls
|
||||
// it back on error.
|
||||
// transaction and returns the tasks for the delivery engine. The
|
||||
// caller owns the transaction and rolls it back on error.
|
||||
func buildDeliveryTasks(
|
||||
tx *gorm.DB,
|
||||
event *database.Event,
|
||||
@@ -406,13 +380,6 @@ func buildDeliveryTasks(
|
||||
)
|
||||
}
|
||||
|
||||
err = database.AddTargetTotals(tx, database.TargetTotals{
|
||||
TargetID: targets[i].ID, Deliveries: 1,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
tasks = append(tasks, delivery.Task{
|
||||
DeliveryID: dlv.ID,
|
||||
EventID: event.ID,
|
||||
|
||||
@@ -1,259 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
)
|
||||
|
||||
// The spans of the two recent windows the statistics pane reports on:
|
||||
// the last 10 minutes and the last 24 hours.
|
||||
const (
|
||||
shortWindow = 10 * time.Minute
|
||||
longWindow = 24 * time.Hour
|
||||
)
|
||||
|
||||
// percent turns a fraction into a percentage.
|
||||
const percent = 100
|
||||
|
||||
// WebhookStats holds the figures in the statistics pane at the top of
|
||||
// the webhook page.
|
||||
type WebhookStats struct {
|
||||
Entrypoints int
|
||||
ActiveEntrypoints int
|
||||
Targets int
|
||||
ActiveTargets int
|
||||
|
||||
// Lifetime counts every event, delivery and failure the webhook
|
||||
// has had, and WithinRetention those still stored.
|
||||
Lifetime Counts
|
||||
WithinRetention Counts
|
||||
|
||||
// InProgress counts the deliveries still pending or retrying.
|
||||
InProgress int64
|
||||
|
||||
// LastEventAt is when the newest event arrived, or nil when none
|
||||
// has. Retention does not change it.
|
||||
LastEventAt *time.Time
|
||||
|
||||
Last10Minutes RecentWindow
|
||||
Last24Hours RecentWindow
|
||||
}
|
||||
|
||||
// Counts holds a number of events, of deliveries and of failed
|
||||
// deliveries.
|
||||
type Counts struct {
|
||||
Events int64
|
||||
Deliveries int64
|
||||
Failures int64
|
||||
}
|
||||
|
||||
// RecentWindow holds what happened in one recent window: the events
|
||||
// received in it, and the deliveries that became delivered or failed in
|
||||
// it.
|
||||
type RecentWindow struct {
|
||||
Events int64
|
||||
Delivered int64
|
||||
Failed int64
|
||||
}
|
||||
|
||||
// TargetFinished is how many of one target's deliveries became
|
||||
// delivered, and how many failed, in a recent window.
|
||||
type TargetFinished struct {
|
||||
TargetID string
|
||||
Delivered int64
|
||||
Failed int64
|
||||
}
|
||||
|
||||
// FailurePercent is the share of the deliveries finished in the window
|
||||
// that failed, or a dash when none finished. Deliveries still pending
|
||||
// or retrying are not counted either way.
|
||||
func (w RecentWindow) FailurePercent() string {
|
||||
finished := w.Delivered + w.Failed
|
||||
if finished == 0 {
|
||||
return "—"
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
"%.1f%%", percent*float64(w.Failed)/float64(finished),
|
||||
)
|
||||
}
|
||||
|
||||
// loadWebhookStats gathers the figures for the statistics pane from the
|
||||
// webhook's entrypoints and targets, as the page has already loaded
|
||||
// them, and from its event database. It returns nil, and logs why, when
|
||||
// the event database cannot be read.
|
||||
func (h *Handlers) loadWebhookStats(
|
||||
webhookID string,
|
||||
entrypoints []database.Entrypoint,
|
||||
targets []database.Target,
|
||||
) *WebhookStats {
|
||||
stats := &WebhookStats{
|
||||
Entrypoints: len(entrypoints),
|
||||
Targets: len(targets),
|
||||
}
|
||||
|
||||
for i := range entrypoints {
|
||||
if entrypoints[i].Active {
|
||||
stats.ActiveEntrypoints++
|
||||
}
|
||||
}
|
||||
|
||||
for i := range targets {
|
||||
if targets[i].Active {
|
||||
stats.ActiveTargets++
|
||||
}
|
||||
}
|
||||
|
||||
// Opening an event database that does not exist would create it,
|
||||
// and it would hold nothing to count.
|
||||
if !h.dbMgr.DBExists(webhookID) {
|
||||
return stats
|
||||
}
|
||||
|
||||
webhookDB, err := h.dbMgr.GetDB(webhookID)
|
||||
if err == nil {
|
||||
err = readEventStats(webhookDB, time.Now(), stats)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
h.log.Error(
|
||||
"failed to read webhook statistics",
|
||||
"webhook_id", webhookID,
|
||||
"error", err,
|
||||
)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
// readEventStats fills in the figures that come from the webhook's
|
||||
// event database. None of them reads every stored row: the totals are
|
||||
// one row for the events and one per target for the deliveries, and
|
||||
// every other figure is read from an index, over only the rows it
|
||||
// counts.
|
||||
func readEventStats(
|
||||
db *gorm.DB, now time.Time, stats *WebhookStats,
|
||||
) error {
|
||||
err := readTotals(db, stats)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = db.Model(&database.Delivery{}).
|
||||
Where("status IN ?", []database.DeliveryStatus{
|
||||
database.DeliveryStatusPending,
|
||||
database.DeliveryStatusRetrying,
|
||||
}).
|
||||
Count(&stats.InProgress).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf("counting deliveries in progress: %w", err)
|
||||
}
|
||||
|
||||
stats.Last10Minutes, err = readRecentWindow(
|
||||
db, now.Add(-shortWindow),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
stats.Last24Hours, err = readRecentWindow(
|
||||
db, now.Add(-longWindow),
|
||||
)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// readTotals fills in the lifetime and within-retention figures, and
|
||||
// when the last event arrived, from the running totals: the events'
|
||||
// row, and the targets' rows summed.
|
||||
func readTotals(db *gorm.DB, stats *WebhookStats) error {
|
||||
var events database.EventTotals
|
||||
|
||||
err := db.Take(&events).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading event totals: %w", err)
|
||||
}
|
||||
|
||||
var targets []database.TargetTotals
|
||||
|
||||
err = db.Find(&targets).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading target totals: %w", err)
|
||||
}
|
||||
|
||||
stats.Lifetime.Events = events.Events
|
||||
stats.WithinRetention.Events = events.Events - events.EventsRemoved
|
||||
stats.LastEventAt = events.LastEventAt
|
||||
|
||||
for _, t := range targets {
|
||||
stats.Lifetime.Deliveries += t.Deliveries
|
||||
stats.Lifetime.Failures += t.Failed
|
||||
stats.WithinRetention.Deliveries += t.Deliveries - t.DeliveriesRemoved
|
||||
stats.WithinRetention.Failures += t.Failed - t.FailedRemoved
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// readRecentWindow counts the events received, and the deliveries that
|
||||
// became delivered or failed, since the given time.
|
||||
func readRecentWindow(
|
||||
db *gorm.DB, since time.Time,
|
||||
) (RecentWindow, error) {
|
||||
var w RecentWindow
|
||||
|
||||
err := db.Model(&database.Event{}).
|
||||
Where("created_at >= ?", since).
|
||||
Count(&w.Events).Error
|
||||
if err != nil {
|
||||
return w, fmt.Errorf("counting recent events: %w", err)
|
||||
}
|
||||
|
||||
byTarget, err := finishedByTarget(db, since)
|
||||
if err != nil {
|
||||
return w, err
|
||||
}
|
||||
|
||||
for _, f := range byTarget {
|
||||
w.Delivered += f.Delivered
|
||||
w.Failed += f.Failed
|
||||
}
|
||||
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// finishedByTarget counts, for each target, the deliveries that became
|
||||
// delivered and those that failed since the given time, in one query
|
||||
// over just that window of the deliveries' status index. A target with
|
||||
// neither is left out.
|
||||
func finishedByTarget(
|
||||
db *gorm.DB, since time.Time,
|
||||
) ([]TargetFinished, error) {
|
||||
var byTarget []TargetFinished
|
||||
|
||||
err := db.Model(&database.Delivery{}).
|
||||
Select("target_id, "+
|
||||
"count(CASE WHEN status = ? THEN 1 END) AS delivered, "+
|
||||
"count(CASE WHEN status = ? THEN 1 END) AS failed",
|
||||
database.DeliveryStatusDelivered,
|
||||
database.DeliveryStatusFailed).
|
||||
Where("status IN ? AND finished_at >= ?",
|
||||
[]database.DeliveryStatus{
|
||||
database.DeliveryStatusDelivered,
|
||||
database.DeliveryStatusFailed,
|
||||
}, since).
|
||||
Group("target_id").
|
||||
Find(&byTarget).Error
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(
|
||||
"counting deliveries finished by target: %w", err,
|
||||
)
|
||||
}
|
||||
|
||||
return byTarget, nil
|
||||
}
|
||||
@@ -1,569 +0,0 @@
|
||||
package handlers_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/fx/fxtest"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
"sneak.berlin/go/webhooker/internal/config"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/handlers"
|
||||
"sneak.berlin/go/webhooker/internal/logger"
|
||||
"sneak.berlin/go/webhooker/internal/session"
|
||||
)
|
||||
|
||||
// statsEntrypoint adds an entrypoint to a webhook and returns its path.
|
||||
func statsEntrypoint(
|
||||
t *testing.T, db *database.Database, webhookID string, active bool,
|
||||
) string {
|
||||
t.Helper()
|
||||
|
||||
ep := &database.Entrypoint{
|
||||
WebhookID: webhookID,
|
||||
Path: uuid.New().String(),
|
||||
}
|
||||
|
||||
require.NoError(t, db.DB().Omit(clause.Associations).Create(ep).Error)
|
||||
require.NoError(t, db.DB().Model(ep).Update("active", active).Error)
|
||||
|
||||
return ep.Path
|
||||
}
|
||||
|
||||
// statsDelivery returns an event's delivery to a target.
|
||||
func statsDelivery(
|
||||
t *testing.T, webhookDB *gorm.DB, eventID, targetID string,
|
||||
) database.Delivery {
|
||||
t.Helper()
|
||||
|
||||
var d database.Delivery
|
||||
|
||||
require.NoError(t, webhookDB.Where(
|
||||
"event_id = ? AND target_id = ?", eventID, targetID,
|
||||
).First(&d).Error)
|
||||
|
||||
return d
|
||||
}
|
||||
|
||||
// statsFinish settles a delivery as the delivery engine does: its
|
||||
// final status and the time it finished, and one more on its target's
|
||||
// delivered or failed total, in one transaction.
|
||||
func statsFinish(
|
||||
t *testing.T,
|
||||
webhookDB *gorm.DB,
|
||||
d database.Delivery,
|
||||
status database.DeliveryStatus,
|
||||
at time.Time,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
add := database.TargetTotals{TargetID: d.TargetID, Delivered: 1}
|
||||
if status == database.DeliveryStatusFailed {
|
||||
add = database.TargetTotals{TargetID: d.TargetID, Failed: 1}
|
||||
}
|
||||
|
||||
require.NoError(t, webhookDB.Transaction(func(tx *gorm.DB) error {
|
||||
err := tx.Model(&database.Delivery{}).
|
||||
Where("id = ?", d.ID).
|
||||
Updates(map[string]any{"status": status, "finished_at": at}).
|
||||
Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return database.AddTargetTotals(tx, add)
|
||||
}))
|
||||
}
|
||||
|
||||
// statsAge moves an event's arrival back to the given time.
|
||||
func statsAge(
|
||||
t *testing.T, webhookDB *gorm.DB, eventID string, at time.Time,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
require.NoError(t, webhookDB.Model(&database.Event{}).
|
||||
Where("id = ?", eventID).
|
||||
Update("created_at", at).Error)
|
||||
}
|
||||
|
||||
// statsTargetTotals reads a webhook database's target totals, keyed by
|
||||
// target.
|
||||
func statsTargetTotals(
|
||||
t *testing.T, webhookDB *gorm.DB,
|
||||
) map[string]database.TargetTotals {
|
||||
t.Helper()
|
||||
|
||||
var rows []database.TargetTotals
|
||||
|
||||
require.NoError(t, webhookDB.Find(&rows).Error)
|
||||
|
||||
byTarget := make(map[string]database.TargetTotals, len(rows))
|
||||
for _, row := range rows {
|
||||
byTarget[row.TargetID] = row
|
||||
}
|
||||
|
||||
return byTarget
|
||||
}
|
||||
|
||||
// statsHistory is the webhook seedStatsHistory builds: its event
|
||||
// database, its newest event, and its two active targets.
|
||||
type statsHistory struct {
|
||||
webhook *database.Webhook
|
||||
webhookDB *gorm.DB
|
||||
newest database.Event
|
||||
first, second string
|
||||
}
|
||||
|
||||
// seedStatsHistory builds the webhook the statistics test checks: 14
|
||||
// days of retention, twelve entrypoints (one inactive) and six targets
|
||||
// (four inactive). Ten events arrive through the receiver, and so each
|
||||
// has a delivery to the two active targets. The oldest event is past
|
||||
// retention, the next 30 hours old, the next six hours old, the other
|
||||
// seven just in. Six deliveries are settled as the delivery engine
|
||||
// would, two of them inside a recent window though their event arrived
|
||||
// before it. The newest event's delivery to the second target is
|
||||
// retrying, the rest are left pending, and a replay adds a pending
|
||||
// delivery to the oldest event. Once retention has removed the oldest
|
||||
// event, every figure in the pane differs from every other.
|
||||
func seedStatsHistory(
|
||||
t *testing.T,
|
||||
h *handlers.Handlers,
|
||||
sess *session.Session,
|
||||
db *database.Database,
|
||||
dbMgr *database.WebhookDBManager,
|
||||
) statsHistory {
|
||||
t.Helper()
|
||||
|
||||
wh := &database.Webhook{UserID: deleteTestUserID, Name: "stats", RetentionDays: 14}
|
||||
require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error)
|
||||
|
||||
path := statsEntrypoint(t, db, wh.ID, true)
|
||||
for range 10 {
|
||||
statsEntrypoint(t, db, wh.ID, true)
|
||||
}
|
||||
|
||||
statsEntrypoint(t, db, wh.ID, false)
|
||||
|
||||
first := seedConfiguredTarget(
|
||||
t, db, wh.ID, database.TargetTypeHTTP,
|
||||
`{"url":"`+replayTargetURL+`"}`,
|
||||
)
|
||||
second := seedTarget(t, db, wh.ID, database.TargetTypeLog)
|
||||
|
||||
for range 4 {
|
||||
inactive := seedTarget(t, db, wh.ID, database.TargetTypeLog)
|
||||
require.NoError(t, db.DB().Model(inactive).
|
||||
Update("active", false).Error)
|
||||
}
|
||||
|
||||
router := receiverRouter(h)
|
||||
|
||||
for range 10 {
|
||||
require.Equal(t, http.StatusOK, postReceiver(t, router, path))
|
||||
}
|
||||
|
||||
webhookDB, err := dbMgr.GetDB(wh.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
events := listEvents(t, webhookDB)
|
||||
require.Len(t, events, 10)
|
||||
|
||||
oldest, yesterday, middle, newest := events[0], events[1], events[2], events[9]
|
||||
now := time.Now()
|
||||
|
||||
statsAge(t, webhookDB, oldest.ID, now.Add(-15*24*time.Hour))
|
||||
statsAge(t, webhookDB, yesterday.ID, now.Add(-30*time.Hour))
|
||||
statsAge(t, webhookDB, middle.ID, now.Add(-6*time.Hour))
|
||||
|
||||
oldestFailure := statsDelivery(t, webhookDB, oldest.ID, first.ID)
|
||||
statsFinish(t, webhookDB, oldestFailure,
|
||||
database.DeliveryStatusFailed, now.Add(-14*24*time.Hour))
|
||||
statsFinish(t, webhookDB,
|
||||
statsDelivery(t, webhookDB, yesterday.ID, first.ID),
|
||||
database.DeliveryStatusFailed, now.Add(-29*time.Hour))
|
||||
statsFinish(t, webhookDB,
|
||||
statsDelivery(t, webhookDB, yesterday.ID, second.ID),
|
||||
database.DeliveryStatusFailed, now.Add(-23*time.Hour))
|
||||
statsFinish(t, webhookDB,
|
||||
statsDelivery(t, webhookDB, middle.ID, second.ID),
|
||||
database.DeliveryStatusFailed, now.Add(-5*time.Hour))
|
||||
statsFinish(t, webhookDB,
|
||||
statsDelivery(t, webhookDB, middle.ID, first.ID),
|
||||
database.DeliveryStatusFailed, now.Add(-time.Minute))
|
||||
statsFinish(t, webhookDB,
|
||||
statsDelivery(t, webhookDB, newest.ID, first.ID),
|
||||
database.DeliveryStatusDelivered, now.Add(-2*time.Minute))
|
||||
|
||||
retrying := statsDelivery(t, webhookDB, newest.ID, second.ID)
|
||||
require.NoError(t, webhookDB.Model(&retrying).
|
||||
Update("status", database.DeliveryStatusRetrying).Error)
|
||||
|
||||
require.Equal(t, http.StatusSeeOther,
|
||||
postReplay(t, h, sess, wh.ID, oldestFailure.ID).Code)
|
||||
|
||||
return statsHistory{
|
||||
webhook: wh, webhookDB: webhookDB, newest: newest,
|
||||
first: first.ID, second: second.ID,
|
||||
}
|
||||
}
|
||||
|
||||
// statsPrune runs the real retention reaper until it has removed one
|
||||
// event from the webhook's database, then stops it.
|
||||
func statsPrune(
|
||||
t *testing.T,
|
||||
db *database.Database,
|
||||
dbMgr *database.WebhookDBManager,
|
||||
log *logger.Logger,
|
||||
webhookDB *gorm.DB,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
lc := fxtest.NewLifecycle(t)
|
||||
database.NewRetentionReaper(lc, database.RetentionReaperParams{
|
||||
Config: &config.Config{
|
||||
RetentionSweepInterval: 10 * time.Millisecond,
|
||||
},
|
||||
Database: db,
|
||||
DBManager: dbMgr,
|
||||
Logger: log,
|
||||
})
|
||||
|
||||
lc.RequireStart()
|
||||
|
||||
require.Eventually(t, func() bool {
|
||||
var totals database.EventTotals
|
||||
|
||||
err := webhookDB.Take(&totals).Error
|
||||
|
||||
return err == nil && totals.EventsRemoved == 1
|
||||
}, 10*time.Second, 10*time.Millisecond)
|
||||
|
||||
lc.RequireStop()
|
||||
}
|
||||
|
||||
// statsPane returns the text of the statistics pane in a rendered
|
||||
// webhook page, everything from its heading to the next heading on the
|
||||
// page, with the markup taken out and each run of space made one
|
||||
// space. A table then reads header by header and row by row, each
|
||||
// row's label followed by its figures in column order.
|
||||
func statsPane(t *testing.T, page string) string {
|
||||
t.Helper()
|
||||
|
||||
_, pane, found := strings.Cut(page, ">Statistics</h2>")
|
||||
require.True(t, found, "the page has no statistics pane")
|
||||
|
||||
pane, _, _ = strings.Cut(pane, "<h2")
|
||||
pane = regexp.MustCompile(`<[^>]*>`).ReplaceAllString(pane, " ")
|
||||
|
||||
return strings.Join(strings.Fields(pane), " ")
|
||||
}
|
||||
|
||||
// assertStatsTargets checks, for the history seedStatsHistory builds,
|
||||
// each target's totals and its deliveries finished in the last 24
|
||||
// hours. The first target has ten deliveries and the replay, the
|
||||
// second ten; the inactive targets have none and so no row.
|
||||
func assertStatsTargets(t *testing.T, hist statsHistory) {
|
||||
t.Helper()
|
||||
|
||||
first, second := hist.first, hist.second
|
||||
|
||||
assert.Equal(t, map[string]database.TargetTotals{
|
||||
first: {TargetID: first, Deliveries: 11, Delivered: 1, Failed: 3},
|
||||
second: {TargetID: second, Deliveries: 10, Failed: 2},
|
||||
}, statsTargetTotals(t, hist.webhookDB))
|
||||
|
||||
lastDay, err := handlers.FinishedByTargetForTest(
|
||||
hist.webhookDB, time.Now().Add(-24*time.Hour),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
assert.ElementsMatch(t, []handlers.TargetFinished{
|
||||
{TargetID: first, Delivered: 1, Failed: 1},
|
||||
{TargetID: second, Failed: 2},
|
||||
}, lastDay)
|
||||
}
|
||||
|
||||
// assertStatsPaneAfterPrune checks the rendered statistics pane for the
|
||||
// history seedStatsHistory builds, once retention has removed the
|
||||
// oldest event: each figure after its label, in its column.
|
||||
func assertStatsPaneAfterPrune(
|
||||
t *testing.T,
|
||||
h *handlers.Handlers,
|
||||
sess *session.Session,
|
||||
hist statsHistory,
|
||||
) {
|
||||
t.Helper()
|
||||
|
||||
pane := statsPane(t, renderSourceDetailPage(t, h, sess, hist.webhook.ID))
|
||||
lastEvent := hist.newest.CreatedAt.UTC().Format("2006-01-02 15:04:05 UTC")
|
||||
|
||||
assert.Contains(t, pane, "Entrypoints 12 (11 active) "+
|
||||
"Targets 6 (2 active) "+
|
||||
"Deliveries in progress 13 "+
|
||||
"Last event "+lastEvent+" "+
|
||||
"Retention 14 days")
|
||||
assert.Contains(t, pane, "Lifetime Within retention "+
|
||||
"Events 10 9 "+
|
||||
"Deliveries 21 18 "+
|
||||
"Failures 5 4")
|
||||
assert.Contains(t, pane, "Last 10 minutes Last 24 hours "+
|
||||
"Events 7 8 "+
|
||||
"Failures 1 3 "+
|
||||
"Failure percentage 50.0% 75.0%")
|
||||
}
|
||||
|
||||
// TestWebhookStats_EveryFigureAcrossRetentionPrune checks every figure
|
||||
// the statistics pane shows for the history seedStatsHistory builds,
|
||||
// and each target's totals and recent figures, before and after the
|
||||
// real retention reaper removes the oldest event.
|
||||
func TestWebhookStats_EveryFigureAcrossRetentionPrune(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
dbMgr *database.WebhookDBManager
|
||||
log *logger.Logger
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &dbMgr, &log)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
hist := seedStatsHistory(t, h, sess, db, dbMgr)
|
||||
first, second := hist.first, hist.second
|
||||
|
||||
stats := h.WebhookStatsForTest(hist.webhook.ID)
|
||||
require.NotNil(t, stats)
|
||||
|
||||
assert.Equal(t, 12, stats.Entrypoints)
|
||||
assert.Equal(t, 11, stats.ActiveEntrypoints)
|
||||
assert.Equal(t, 6, stats.Targets)
|
||||
assert.Equal(t, 2, stats.ActiveTargets)
|
||||
assert.Equal(t, handlers.Counts{Events: 10, Deliveries: 21, Failures: 5},
|
||||
stats.Lifetime)
|
||||
assert.Equal(t, stats.Lifetime, stats.WithinRetention)
|
||||
assert.Equal(t, int64(15), stats.InProgress)
|
||||
require.NotNil(t, stats.LastEventAt)
|
||||
assert.True(t, hist.newest.CreatedAt.Equal(*stats.LastEventAt))
|
||||
assert.Equal(t, handlers.RecentWindow{
|
||||
Events: 7, Delivered: 1, Failed: 1,
|
||||
}, stats.Last10Minutes)
|
||||
assert.Equal(t, handlers.RecentWindow{
|
||||
Events: 8, Delivered: 1, Failed: 3,
|
||||
}, stats.Last24Hours)
|
||||
assert.Equal(t, "50.0%", stats.Last10Minutes.FailurePercent())
|
||||
assert.Equal(t, "75.0%", stats.Last24Hours.FailurePercent())
|
||||
|
||||
assertStatsTargets(t, hist)
|
||||
|
||||
// Retention removes the oldest event with its three deliveries:
|
||||
// the first target's failed one and the pending replay, and the
|
||||
// second target's pending one.
|
||||
statsPrune(t, db, dbMgr, log, hist.webhookDB)
|
||||
|
||||
after := h.WebhookStatsForTest(hist.webhook.ID)
|
||||
require.NotNil(t, after)
|
||||
|
||||
assert.Equal(t, stats.Lifetime, after.Lifetime)
|
||||
assert.Equal(t, handlers.Counts{Events: 9, Deliveries: 18, Failures: 4},
|
||||
after.WithinRetention)
|
||||
assert.Equal(t, int64(13), after.InProgress)
|
||||
assert.Equal(t, stats.LastEventAt, after.LastEventAt)
|
||||
assert.Equal(t, stats.Last10Minutes, after.Last10Minutes)
|
||||
assert.Equal(t, stats.Last24Hours, after.Last24Hours)
|
||||
|
||||
assert.Equal(t, map[string]database.TargetTotals{
|
||||
first: {
|
||||
TargetID: first, Deliveries: 11, Delivered: 1, Failed: 3,
|
||||
DeliveriesRemoved: 2, FailedRemoved: 1,
|
||||
},
|
||||
second: {
|
||||
TargetID: second, Deliveries: 10, Failed: 2,
|
||||
DeliveriesRemoved: 1,
|
||||
},
|
||||
}, statsTargetTotals(t, hist.webhookDB))
|
||||
|
||||
assertStatsPaneAfterPrune(t, h, sess, hist)
|
||||
}
|
||||
|
||||
// TestWebhookStats_LastEventSurvivesPruningEveryEvent checks that once
|
||||
// retention has removed every event, the pane still shows when the last
|
||||
// one arrived rather than "none".
|
||||
func TestWebhookStats_LastEventSurvivesPruningEveryEvent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
dbMgr *database.WebhookDBManager
|
||||
log *logger.Logger
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &dbMgr, &log)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
wh := &database.Webhook{
|
||||
UserID: deleteTestUserID, Name: "pruned", RetentionDays: 1,
|
||||
}
|
||||
require.NoError(t, db.DB().Omit(clause.Associations).Create(wh).Error)
|
||||
|
||||
path := statsEntrypoint(t, db, wh.ID, true)
|
||||
require.Equal(t, http.StatusOK,
|
||||
postReceiver(t, receiverRouter(h), path))
|
||||
|
||||
webhookDB, err := dbMgr.GetDB(wh.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
events := listEvents(t, webhookDB)
|
||||
require.Len(t, events, 1)
|
||||
|
||||
arrived := events[0].CreatedAt
|
||||
|
||||
statsAge(t, webhookDB, events[0].ID, time.Now().Add(-50*time.Hour))
|
||||
statsPrune(t, db, dbMgr, log, webhookDB)
|
||||
require.Empty(t, listEvents(t, webhookDB))
|
||||
|
||||
stats := h.WebhookStatsForTest(wh.ID)
|
||||
require.NotNil(t, stats)
|
||||
require.NotNil(t, stats.LastEventAt)
|
||||
assert.True(t, arrived.Equal(*stats.LastEventAt))
|
||||
|
||||
pane := statsPane(t, renderSourceDetailPage(t, h, sess, wh.ID))
|
||||
assert.Contains(t, pane,
|
||||
"Last event "+arrived.UTC().Format("2006-01-02 15:04:05 UTC"))
|
||||
}
|
||||
|
||||
// TestWebhookStats_LastEventInUTC checks that the pane shows when the
|
||||
// last event arrived in UTC, as the event list does, when the time was
|
||||
// stored in another zone, as it is on a host whose local time is not
|
||||
// UTC.
|
||||
func TestWebhookStats_LastEventInUTC(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
dbMgr *database.WebhookDBManager
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
wh := seedWebhook(t, db)
|
||||
|
||||
webhookDB, err := dbMgr.GetDB(wh.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
arrived := time.Date(2026, time.March, 4, 22, 30, 0, 0,
|
||||
time.FixedZone("EST", -5*60*60))
|
||||
require.NoError(t, database.AddEventTotals(webhookDB,
|
||||
database.EventTotals{Events: 1, LastEventAt: &arrived}))
|
||||
|
||||
pane := statsPane(t, renderSourceDetailPage(t, h, sess, wh.ID))
|
||||
assert.Contains(t, pane, "Last event 2026-03-05 03:30:00 UTC")
|
||||
}
|
||||
|
||||
// TestWebhookStats_PaneShowsRetentionPeriod checks that the statistics
|
||||
// pane itself, not only the line at the foot of the page, shows the
|
||||
// webhook's retention period, for a finite one and for forever.
|
||||
func TestWebhookStats_PaneShowsRetentionPeriod(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
tests := []struct {
|
||||
retentionDays int
|
||||
want string
|
||||
}{
|
||||
{30, "30 days"},
|
||||
{database.RetentionForeverDays, "forever"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
wh := &database.Webhook{
|
||||
UserID: deleteTestUserID,
|
||||
Name: "retention",
|
||||
RetentionDays: tt.retentionDays,
|
||||
}
|
||||
require.NoError(t,
|
||||
db.DB().Omit(clause.Associations).Create(wh).Error)
|
||||
|
||||
pane := statsPane(t, renderSourceDetailPage(t, h, sess, wh.ID))
|
||||
assert.Contains(t, pane, "Retention "+tt.want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWebhookStats_WebhookWithNoEvents covers a webhook whose event
|
||||
// database has never been opened: every count is zero, the
|
||||
// percentages are a dash, and showing the page does not create the
|
||||
// database.
|
||||
func TestWebhookStats_WebhookWithNoEvents(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var (
|
||||
h *handlers.Handlers
|
||||
sess *session.Session
|
||||
db *database.Database
|
||||
dbMgr *database.WebhookDBManager
|
||||
)
|
||||
|
||||
app := newTestApp(t, &h, &sess, &db, &dbMgr)
|
||||
app.RequireStart()
|
||||
|
||||
t.Cleanup(app.RequireStop)
|
||||
|
||||
wh := seedWebhook(t, db)
|
||||
|
||||
assert.Equal(t, &handlers.WebhookStats{}, h.WebhookStatsForTest(wh.ID))
|
||||
assert.Equal(t, "—", handlers.RecentWindow{}.FailurePercent())
|
||||
|
||||
pane := statsPane(t, renderSourceDetailPage(t, h, sess, wh.ID))
|
||||
assert.Contains(t, pane, "Last event none")
|
||||
assert.Contains(t, pane, "Failure percentage — —")
|
||||
assert.False(t, dbMgr.DBExists(wh.ID))
|
||||
}
|
||||
|
||||
// TestRecentWindow_FailurePercent pins the percentage: failed
|
||||
// deliveries out of all that finished in the window.
|
||||
func TestRecentWindow_FailurePercent(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tests := []struct {
|
||||
window handlers.RecentWindow
|
||||
want string
|
||||
}{
|
||||
{handlers.RecentWindow{}, "—"},
|
||||
{handlers.RecentWindow{Events: 4}, "—"},
|
||||
{handlers.RecentWindow{Delivered: 3, Failed: 1}, "25.0%"},
|
||||
{handlers.RecentWindow{Failed: 2}, "100.0%"},
|
||||
{handlers.RecentWindow{Delivered: 2}, "0.0%"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
assert.Equal(t, tt.want, tt.window.FailurePercent(), tt.window)
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"go.uber.org/fx"
|
||||
"sneak.berlin/go/webhooker/internal/config"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
"sneak.berlin/go/webhooker/internal/globals"
|
||||
"sneak.berlin/go/webhooker/internal/logger"
|
||||
@@ -17,6 +18,7 @@ type HealthcheckParams struct {
|
||||
fx.In
|
||||
|
||||
Globals *globals.Globals
|
||||
Config *config.Config
|
||||
Logger *logger.Logger
|
||||
Database *database.Database
|
||||
}
|
||||
@@ -62,6 +64,7 @@ func (s *Healthcheck) Healthcheck() *Response {
|
||||
UptimeHuman: s.uptime().String(),
|
||||
Appname: s.params.Globals.Appname,
|
||||
Version: s.params.Globals.Version,
|
||||
Maintenance: s.params.Config.MaintenanceMode,
|
||||
}
|
||||
|
||||
return resp
|
||||
@@ -75,6 +78,7 @@ type Response struct {
|
||||
UptimeHuman string `json:"uptimeHuman"`
|
||||
Version string `json:"version"`
|
||||
Appname string `json:"appname"`
|
||||
Maintenance bool `json:"maintenanceMode"`
|
||||
}
|
||||
|
||||
func (s *Healthcheck) uptime() time.Duration {
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"io"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"unicode/utf8"
|
||||
|
||||
@@ -19,11 +18,15 @@ import (
|
||||
// width.
|
||||
const budget = 64
|
||||
|
||||
// batchRunes is how many consecutive code points the charge test logs
|
||||
// in one value. Logging every code point on its own line is too slow
|
||||
// for the suite under the race detector; 4,096 at a time is 272 lines
|
||||
// per handler for all of Unicode.
|
||||
const batchRunes = 4096
|
||||
// sampleRunes is how many runes wide the values in the charge test
|
||||
// are. The handlers add a constant per field — a pair of quotes when
|
||||
// the value needs quoting — so the per-rune charge is only visible
|
||||
// once it is amortised over a run of them.
|
||||
const sampleRunes = 64
|
||||
|
||||
// quotingSlack is that constant: the pair of quotes a handler adds to
|
||||
// a value that needs them and omits from one that does not.
|
||||
const quotingSlack = 2
|
||||
|
||||
// newHandlers are the two handlers internal/logger can install. Time
|
||||
// is dropped so a line's width is a function of its value alone —
|
||||
@@ -63,42 +66,46 @@ func renderedWidth(
|
||||
return buf.Len()
|
||||
}
|
||||
|
||||
// emittedBytes is what a handler writes for the runes of s alone: the
|
||||
// width of a line carrying s twice, less that of a line carrying it
|
||||
// once. Both values hold the same runes, so the text handler quotes
|
||||
// both or neither, and the quotes cancel along with everything else on
|
||||
// the line.
|
||||
func emittedBytes(
|
||||
newHandler func(io.Writer) slog.Handler,
|
||||
s string,
|
||||
) int {
|
||||
return renderedWidth(newHandler, s+s) - renderedWidth(newHandler, s)
|
||||
}
|
||||
// chargeTestRunes is the set of code points the charge test measures:
|
||||
// every rune in the first two planes' worth of the BMP that the
|
||||
// handlers are most likely to treat specially, the separators that
|
||||
// only slog's JSON handler escapes, and a stratified sample across
|
||||
// the rest of Unicode so the astral charge is exercised on more than
|
||||
// one hand-picked rune.
|
||||
func chargeTestRunes() []rune {
|
||||
const (
|
||||
denseCeiling = 0x800
|
||||
stride = 1021
|
||||
surrogateLo = 0xD800
|
||||
surrogateHi = 0xDFFF
|
||||
)
|
||||
|
||||
// firstUndercharged returns the first rune in s that the handler
|
||||
// writes in more bytes than EncodedBytes charges for it, and how many
|
||||
// runes in s are undercharged that way. It measures one rune per line,
|
||||
// so the charge test calls it only on a batch that has already failed,
|
||||
// to name the code points rather than just their range.
|
||||
func firstUndercharged(
|
||||
newHandler func(io.Writer) slog.Handler,
|
||||
s string,
|
||||
) (rune, int) {
|
||||
first, count := rune(-1), 0
|
||||
var runes []rune
|
||||
|
||||
for _, r := range s {
|
||||
if emittedBytes(newHandler, string(r)) <= logfield.EncodedBytes(r) {
|
||||
continue
|
||||
keep := func(r rune) {
|
||||
if r >= surrogateLo && r <= surrogateHi {
|
||||
return
|
||||
}
|
||||
|
||||
if count == 0 {
|
||||
first = r
|
||||
}
|
||||
|
||||
count++
|
||||
runes = append(runes, r)
|
||||
}
|
||||
|
||||
return first, count
|
||||
for r := range rune(denseCeiling) {
|
||||
keep(r)
|
||||
}
|
||||
|
||||
for _, r := range []rune{
|
||||
0x2028, 0x2029, 0x200B, 0x4E00, 0xE000, 0xFFFD,
|
||||
0x1000C, 0x1F600, 0xE0001, 0x10FFFF,
|
||||
} {
|
||||
keep(r)
|
||||
}
|
||||
|
||||
for r := rune(denseCeiling); r <= utf8.MaxRune; r += stride {
|
||||
keep(r)
|
||||
}
|
||||
|
||||
return runes
|
||||
}
|
||||
|
||||
// TestEncodedBytes_ChargesAtLeastWhatTheHandlersEmit is the property
|
||||
@@ -107,70 +114,33 @@ func firstUndercharged(
|
||||
// how a stated ceiling becomes false without any test noticing, so
|
||||
// the charge is measured against what the handlers actually write
|
||||
// rather than against the escaping rules as read.
|
||||
//
|
||||
// Every code point is covered, batchRunes at a time: each batch's
|
||||
// summed charge must cover what the handler writes for the whole
|
||||
// batch. A sum can hide an undercharged rune behind overcharged ones
|
||||
// in the same batch, which is the price of fitting all of Unicode
|
||||
// into the suite.
|
||||
func TestEncodedBytes_ChargesAtLeastWhatTheHandlersEmit(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var batches []string
|
||||
|
||||
for lo := rune(0); lo <= utf8.MaxRune; lo += batchRunes {
|
||||
var batch strings.Builder
|
||||
|
||||
for r := lo; r < lo+batchRunes; r++ {
|
||||
// Surrogate halves are not runes a string can carry.
|
||||
if utf8.ValidRune(r) {
|
||||
batch.WriteRune(r)
|
||||
}
|
||||
}
|
||||
|
||||
batches = append(batches, batch.String())
|
||||
}
|
||||
|
||||
// What EncodedBytes charges for each batch. Under -race -cover this
|
||||
// takes longer than logging the batches, so it is worked out once,
|
||||
// by whichever handler finishes logging first, while the other is
|
||||
// still logging.
|
||||
charged := sync.OnceValue(func() []int {
|
||||
costs := make([]int, len(batches))
|
||||
|
||||
for i, batch := range batches {
|
||||
for _, r := range batch {
|
||||
costs[i] += logfield.EncodedBytes(r)
|
||||
}
|
||||
}
|
||||
|
||||
return costs
|
||||
})
|
||||
|
||||
for name, newHandler := range newHandlers() {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
emitted := make([]int, len(batches))
|
||||
for i, batch := range batches {
|
||||
emitted[i] = emittedBytes(newHandler, batch)
|
||||
}
|
||||
// 'a' is a printable ASCII rune, charged exactly one
|
||||
// byte, so it is the zero point the other runes are
|
||||
// measured against.
|
||||
base := renderedWidth(
|
||||
newHandler, strings.Repeat("a", sampleRunes),
|
||||
)
|
||||
|
||||
for i, cost := range charged() {
|
||||
if emitted[i] <= cost {
|
||||
continue
|
||||
}
|
||||
|
||||
lo := rune(i * batchRunes)
|
||||
first, count := firstUndercharged(
|
||||
newHandler, batches[i],
|
||||
for _, r := range chargeTestRunes() {
|
||||
got := renderedWidth(
|
||||
newHandler,
|
||||
strings.Repeat(string(r), sampleRunes),
|
||||
)
|
||||
t.Errorf(
|
||||
"U+%04X to U+%04X emit %d bytes but are "+
|
||||
"charged %d; %d of them cost more than "+
|
||||
"EncodedBytes charges, the first U+%04X",
|
||||
lo, lo+batchRunes-1, emitted[i], cost,
|
||||
count, first,
|
||||
charged := sampleRunes *
|
||||
(logfield.EncodedBytes(r) - 1)
|
||||
|
||||
require.LessOrEqual(
|
||||
t, got-base, charged+quotingSlack,
|
||||
"U+%04X costs more on the line than "+
|
||||
"EncodedBytes charges for it",
|
||||
r,
|
||||
)
|
||||
}
|
||||
})
|
||||
@@ -231,7 +201,7 @@ func TestTruncate_LeavesShortValuesAlone(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, s := range []string{
|
||||
"", "GET", "/hook/abc/edit", "Mozilla/5.0 (X11)",
|
||||
"", "GET", "/source/abc/edit", "Mozilla/5.0 (X11)",
|
||||
} {
|
||||
assert.Equal(t, s, logfield.Truncate(s, budget))
|
||||
}
|
||||
|
||||
+20
-28
@@ -3,18 +3,17 @@
|
||||
// deliveries are attempted, how they end, how long they take, how
|
||||
// deep the queues are, and how many circuit breakers are open.
|
||||
//
|
||||
// It also builds the registry the authenticated /metrics route
|
||||
// serves. In production, these collectors, the inbound HTTP metrics
|
||||
// recorded in internal/middleware, and the Go runtime and process
|
||||
// collectors all register on that one registry, never on Prometheus's
|
||||
// global default.
|
||||
// The inbound HTTP metrics come from the go-http-metrics recorder in
|
||||
// internal/middleware and land on prometheus.DefaultRegisterer. These
|
||||
// collectors register there too, so both surfaces are gathered by the
|
||||
// one promhttp handler mounted on the authenticated /metrics route.
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/collectors"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
"sneak.berlin/go/webhooker/internal/database"
|
||||
)
|
||||
@@ -58,31 +57,25 @@ var knownTargetTypes = []database.TargetType{
|
||||
database.TargetTypeSlack,
|
||||
}
|
||||
|
||||
// NewRegistry returns the registry /metrics serves, carrying the Go
|
||||
// runtime and process collectors that Prometheus's global default
|
||||
// registry carries, so the go_* and process_* series stay in the
|
||||
// scrape.
|
||||
// defaultSet is the process-wide metric set, registered on the same
|
||||
// registry the HTTP middleware and the /metrics handler already use.
|
||||
// It is built on first use rather than in an init so that a test
|
||||
// binary that never touches metrics never registers them.
|
||||
//
|
||||
// A registry of its own, rather than the global default, is what lets
|
||||
// two dependency graphs in one process — two tests, say — each
|
||||
// register their collectors without the second registration
|
||||
// panicking.
|
||||
func NewRegistry() *prometheus.Registry {
|
||||
reg := prometheus.NewRegistry()
|
||||
reg.MustRegister(
|
||||
collectors.NewGoCollector(),
|
||||
collectors.NewProcessCollector(
|
||||
collectors.ProcessCollectorOpts{},
|
||||
),
|
||||
)
|
||||
//nolint:gochecknoglobals // one process-wide registration, by design
|
||||
var defaultSet = sync.OnceValue(func() *Set {
|
||||
return New(prometheus.DefaultRegisterer)
|
||||
})
|
||||
|
||||
return reg
|
||||
// Default returns the process-wide metric set.
|
||||
func Default() *Set {
|
||||
return defaultSet()
|
||||
}
|
||||
|
||||
// Set is one registered group of webhooker's delivery collectors.
|
||||
// Production builds one on the registry /metrics serves; tests build
|
||||
// one on a registry of their own so they can gather what their own
|
||||
// deliveries recorded.
|
||||
// Production uses the single Default set; tests build their own
|
||||
// against a private registry so assertions are not disturbed by
|
||||
// deliveries other tests are making concurrently.
|
||||
type Set struct {
|
||||
eventsReceived prometheus.Counter
|
||||
deliveryAttempts *prometheus.CounterVec
|
||||
@@ -100,7 +93,7 @@ type Set struct {
|
||||
// New registers a full set of delivery collectors on reg and returns
|
||||
// it. It panics if reg already holds them, which is the intended
|
||||
// behaviour for a duplicate registration.
|
||||
func New(reg *prometheus.Registry) *Set {
|
||||
func New(reg prometheus.Registerer) *Set {
|
||||
factory := promauto.With(reg)
|
||||
|
||||
s := &Set{
|
||||
@@ -384,7 +377,6 @@ func (s *Set) initSeries() {
|
||||
s.deliveriesFailed.WithLabelValues(label)
|
||||
s.deliveryRetries.WithLabelValues(label)
|
||||
s.deliveryReplays.WithLabelValues(label)
|
||||
s.deliveryDuration.WithLabelValues(label)
|
||||
s.deliveriesPending.WithLabelValues(label)
|
||||
s.deliveriesRetrying.WithLabelValues(label)
|
||||
s.circuitBreakersOpen.WithLabelValues(label)
|
||||
|
||||
@@ -167,7 +167,6 @@ func TestKnownSeriesExistBeforeAnyDelivery(t *testing.T) {
|
||||
"webhooker_deliveries_succeeded_total",
|
||||
"webhooker_deliveries_failed_total",
|
||||
"webhooker_delivery_retries_total",
|
||||
"webhooker_delivery_duration_seconds",
|
||||
"webhooker_circuit_breakers_open",
|
||||
} {
|
||||
assert.ElementsMatch(t,
|
||||
|
||||
@@ -119,7 +119,7 @@ func accessLogRouter(m *middleware.Middleware) *chi.Mux {
|
||||
)
|
||||
|
||||
router.HandleFunc(
|
||||
"/h/{uuid}",
|
||||
"/webhook/{uuid}",
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
// Stands in for the real handler: an unknown entrypoint
|
||||
// UUID 404s, a known one succeeds.
|
||||
@@ -271,11 +271,11 @@ func TestAccessLog_InventedReceiverPathsLogRoutePattern(t *testing.T) {
|
||||
assertFloodIsBounded(
|
||||
t,
|
||||
func(i int) string {
|
||||
return "/h/" + attackerMarker +
|
||||
return "/webhook/" + attackerMarker +
|
||||
strings.Repeat("x", i) + "?q=" + attackerMarker
|
||||
},
|
||||
http.StatusNotFound,
|
||||
"/h/{uuid}",
|
||||
"/webhook/{uuid}",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -346,10 +346,10 @@ type sizeCase struct {
|
||||
func lineSizeCases() map[string]sizeCase {
|
||||
cases := map[string]sizeCase{
|
||||
"oversized path segment": {
|
||||
target: "/h/" + attackerMarker +
|
||||
target: "/webhook/" + attackerMarker +
|
||||
strings.Repeat("x", oversizedSegmentBytes),
|
||||
wantStatus: http.StatusNotFound,
|
||||
wantURL: "/h/{uuid}",
|
||||
wantURL: "/webhook/{uuid}",
|
||||
bound: maxLineBytes,
|
||||
},
|
||||
// /.well-known/healthcheck answers 200 to anyone and has no
|
||||
@@ -605,14 +605,14 @@ func TestAccessLog_SuccessKeepsConcretePathAndRedactsQuery(
|
||||
router := accessLogRouter(m)
|
||||
|
||||
assert.Equal(
|
||||
t, http.StatusOK, get(t, router, "/h/known?src=ci"),
|
||||
t, http.StatusOK, get(t, router, "/webhook/known?src=ci"),
|
||||
)
|
||||
|
||||
// The path resolved against a stored entrypoint, so it stays. The
|
||||
// query never does: see TestAccessLog_UnauthenticatedSuccess...
|
||||
entries := accessLogEntries(t, buf)
|
||||
require.Len(t, entries, 1)
|
||||
assert.Equal(t, "/h/known?(redacted)", entries[0]["url"])
|
||||
assert.Equal(t, "/webhook/known?(redacted)", entries[0]["url"])
|
||||
assert.NotContains(t, buf.String(), "src=ci")
|
||||
}
|
||||
|
||||
@@ -640,7 +640,7 @@ func TestAccessLog_RetainsEveryOtherField(t *testing.T) {
|
||||
assert.Equal(
|
||||
t,
|
||||
http.StatusNotFound,
|
||||
get(t, router, "/h/"+attackerMarker),
|
||||
get(t, router, "/webhook/"+attackerMarker),
|
||||
)
|
||||
|
||||
entries := accessLogEntries(t, buf)
|
||||
|
||||
@@ -19,7 +19,7 @@ func CSRFToken(r *http.Request) string {
|
||||
// key to sign a CSRF cookie and validates a masked token submitted via
|
||||
// the "csrf_token" form field (or the "X-CSRF-Token" header) on
|
||||
// POST/PUT/PATCH/DELETE requests. Requests with an invalid or missing
|
||||
// token are logged and answered by forbidden, which must write the 403.
|
||||
// token receive a 403 Forbidden response.
|
||||
//
|
||||
// The middleware detects the client-facing transport protocol
|
||||
// per-request via reqtls.IsTLS, the single TLS predicate the session
|
||||
@@ -36,14 +36,12 @@ func CSRFToken(r *http.Request) string {
|
||||
// Two gorilla/csrf instances are maintained — one with Secure cookies
|
||||
// (for TLS) and one without (for plaintext HTTP) — because the
|
||||
// csrf.Secure option is set at creation time, not per-request.
|
||||
func (m *Middleware) CSRF(
|
||||
forbidden http.Handler,
|
||||
) func(http.Handler) http.Handler {
|
||||
func (m *Middleware) CSRF() func(http.Handler) http.Handler {
|
||||
csrfErrorHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// CSRF is registered ahead of RequireAuth on every route
|
||||
// group that uses it, so this WARN is reachable by an
|
||||
// unauthenticated client: a POST with no token to
|
||||
// /hook/<any length of any text>/edit lands here. The
|
||||
// /source/<any length of any text>/edit lands here. The
|
||||
// method and path are capped against the same budgets as
|
||||
// the access log. remote_addr is set by net/http from the
|
||||
// accepted connection rather than by the client, and
|
||||
@@ -59,7 +57,7 @@ func (m *Middleware) CSRF(
|
||||
"remote_addr", r.RemoteAddr,
|
||||
"reason", csrf.FailureReason(r),
|
||||
)
|
||||
forbidden.ServeHTTP(w, r)
|
||||
http.Error(w, "Forbidden - invalid CSRF token", http.StatusForbidden)
|
||||
})
|
||||
|
||||
key := m.session.GetKey()
|
||||
|
||||
@@ -18,12 +18,6 @@ import (
|
||||
// csrfCookieName is the gorilla/csrf cookie name.
|
||||
const csrfCookieName = "_gorilla_csrf"
|
||||
|
||||
// forbidden stands in for the error page the server hands CSRF to
|
||||
// answer a refused request with.
|
||||
func forbidden(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
}
|
||||
|
||||
// csrfGetToken performs a GET request through the CSRF middleware
|
||||
// and returns the token and cookies.
|
||||
func csrfGetToken(
|
||||
@@ -104,7 +98,7 @@ func TestCSRF_GETSetsToken(t *testing.T) {
|
||||
|
||||
var gotToken string
|
||||
|
||||
handler := m.CSRF(http.HandlerFunc(forbidden))(http.HandlerFunc(
|
||||
handler := m.CSRF()(http.HandlerFunc(
|
||||
func(_ http.ResponseWriter, r *http.Request) {
|
||||
gotToken = middleware.CSRFToken(r)
|
||||
},
|
||||
@@ -126,7 +120,7 @@ func TestCSRF_POSTWithValidToken(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
m, _ := testMiddleware(t, config.EnvironmentDev)
|
||||
csrfMW := m.CSRF(http.HandlerFunc(forbidden))
|
||||
csrfMW := m.CSRF()
|
||||
|
||||
getReq := httptest.NewRequestWithContext(
|
||||
context.Background(),
|
||||
@@ -158,7 +152,7 @@ func csrfPOSTWithoutTokenTest(
|
||||
t.Helper()
|
||||
|
||||
m, _ := testMiddleware(t, env)
|
||||
csrfMW := m.CSRF(http.HandlerFunc(forbidden))
|
||||
csrfMW := m.CSRF()
|
||||
|
||||
// GET to establish the CSRF cookie
|
||||
getHandler := csrfMW(http.HandlerFunc(
|
||||
@@ -215,7 +209,7 @@ func TestCSRF_POSTWithInvalidToken(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
m, _ := testMiddleware(t, config.EnvironmentDev)
|
||||
csrfMW := m.CSRF(http.HandlerFunc(forbidden))
|
||||
csrfMW := m.CSRF()
|
||||
|
||||
// GET to establish the CSRF cookie
|
||||
getHandler := csrfMW(http.HandlerFunc(
|
||||
@@ -271,7 +265,7 @@ func TestCSRF_GETDoesNotValidate(t *testing.T) {
|
||||
|
||||
var called bool
|
||||
|
||||
handler := m.CSRF(http.HandlerFunc(forbidden))(http.HandlerFunc(
|
||||
handler := m.CSRF()(http.HandlerFunc(
|
||||
func(_ http.ResponseWriter, _ *http.Request) {
|
||||
called = true
|
||||
},
|
||||
@@ -334,7 +328,7 @@ func csrfTookStrictPath(
|
||||
t.Helper()
|
||||
|
||||
m, _ := testMiddleware(t, env)
|
||||
csrfMW := m.CSRF(http.HandlerFunc(forbidden))
|
||||
csrfMW := m.CSRF()
|
||||
|
||||
newReq := func(method string) *http.Request {
|
||||
r := httptest.NewRequestWithContext(
|
||||
@@ -483,7 +477,7 @@ func TestCSRF_ProdMode_PlaintextHTTP_POSTWithValidToken(
|
||||
t.Parallel()
|
||||
|
||||
m, _ := testMiddleware(t, config.EnvironmentProd)
|
||||
csrfMW := m.CSRF(http.HandlerFunc(forbidden))
|
||||
csrfMW := m.CSRF()
|
||||
|
||||
getReq := httptest.NewRequestWithContext(
|
||||
context.Background(),
|
||||
@@ -523,7 +517,7 @@ func TestCSRF_ProdMode_BehindProxy_POSTWithValidToken(
|
||||
t.Parallel()
|
||||
|
||||
m, _ := testMiddleware(t, config.EnvironmentProd)
|
||||
csrfMW := m.CSRF(http.HandlerFunc(forbidden))
|
||||
csrfMW := m.CSRF()
|
||||
|
||||
getReq := httptest.NewRequestWithContext(
|
||||
context.Background(),
|
||||
@@ -568,7 +562,7 @@ func TestCSRF_ProdMode_DirectTLS_POSTWithValidToken(
|
||||
t.Parallel()
|
||||
|
||||
m, _ := testMiddleware(t, config.EnvironmentProd)
|
||||
csrfMW := m.CSRF(http.HandlerFunc(forbidden))
|
||||
csrfMW := m.CSRF()
|
||||
|
||||
getReq := httptest.NewRequestWithContext(
|
||||
context.Background(),
|
||||
|
||||
@@ -10,7 +10,8 @@ import (
|
||||
|
||||
// MetricsMiddlewareForTest builds the metrics recording middleware
|
||||
// against a caller-supplied recorder, so a test can gather from its
|
||||
// own Prometheus registry without building a whole Middleware.
|
||||
// own Prometheus registry rather than the process-wide default one
|
||||
// that Middleware.Metrics uses.
|
||||
func MetricsMiddlewareForTest(
|
||||
rec httpmetrics.Recorder,
|
||||
) func(http.Handler) http.Handler {
|
||||
|
||||
@@ -260,9 +260,7 @@ func logSites() map[string]logSite {
|
||||
) http.Handler {
|
||||
t.Helper()
|
||||
|
||||
return m.CSRF(http.HandlerFunc(forbidden))(
|
||||
unreachable(t),
|
||||
)
|
||||
return m.CSRF()(unreachable(t))
|
||||
},
|
||||
send: postNoToken,
|
||||
wantStatus: http.StatusForbidden,
|
||||
@@ -385,7 +383,7 @@ func TestLogLines_ClientChosenPathDoesNotSizeTheLine(t *testing.T) {
|
||||
t, newHandler,
|
||||
)
|
||||
|
||||
path := "/hook/" +
|
||||
path := "/source/" +
|
||||
oversizedPathSegment(fill) + "/edit"
|
||||
|
||||
assert.Equal(
|
||||
@@ -436,7 +434,7 @@ func TestLoginThrottle_LogLineDoesNotTrackPathSize(t *testing.T) {
|
||||
req := httptest.NewRequestWithContext(
|
||||
context.Background(),
|
||||
http.MethodPost,
|
||||
"/hook/"+
|
||||
"/source/"+
|
||||
oversizedPathSegment(fill)+"/login",
|
||||
nil,
|
||||
)
|
||||
@@ -501,7 +499,7 @@ func TestMaxBodySize_FloodOfOversizePathsDoesNotGrowTheLog(
|
||||
http.StatusRequestEntityTooLarge,
|
||||
postOversize(
|
||||
h,
|
||||
"/hook/"+segment(i)+"/edit",
|
||||
"/source/"+segment(i)+"/edit",
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -108,10 +108,10 @@ type failureWindow struct {
|
||||
//
|
||||
// A limiter that spends budget on arrival cannot protect a
|
||||
// single-admin product: behind the reverse proxy the deployment
|
||||
// requires, when TRUSTED_PROXIES does not cover it, every client
|
||||
// keys on the proxy, so a stranger trickling five POSTs a minute
|
||||
// keeps the one bucket full and the operator's own correct password
|
||||
// is answered 429 forever. There is no second administrative path.
|
||||
// requires, with TRUSTED_PROXIES unset, every client keys on the
|
||||
// proxy, so a stranger trickling five POSTs a minute keeps the one
|
||||
// bucket full and the operator's own correct password is answered 429
|
||||
// forever. There is no second administrative path.
|
||||
//
|
||||
// So budget is spent only by a FAILED verification. A correct
|
||||
// password is never throttled, whatever the counters say, which is
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user